diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..224f845c8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Shell scripts are executed and inspected verbatim on every CI host. +*.sh text eol=lf diff --git a/.github/scripts/priority-completion.test.cjs b/.github/scripts/priority-completion.test.cjs new file mode 100644 index 000000000..bb9a95a09 --- /dev/null +++ b/.github/scripts/priority-completion.test.cjs @@ -0,0 +1,45 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { test } = require('node:test'); + +const workflow = fs.readFileSync(path.join(__dirname, '../workflows/go-ci.yml'), 'utf8'); + +function jobSection(name) { + const start = workflow.indexOf(`\n ${name}:\n`); + assert.notEqual(start, -1, `missing job ${name}`); + const rest = workflow.slice(start + 1); + const next = rest.slice(1).search(/^ [a-zA-Z0-9_-]+:\s*$/m); + return next < 0 ? rest : rest.slice(0, next + 1); +} + +test('translator jobs require policy preflight, not external compatibility', () => { + for (const name of [ + 'fmt', 'build', 'benchmark', + 'stdlib-corpus', 'cross-runtime', 'test', 'arm-scan', 'race', 'coverage', + ]) { + const section = jobSection(name); + assert.match(section, /^ needs: ci_policy$/m, name); + // Native needs success gating must not be bypassed by always()/!cancelled(). + assert.doesNotMatch(section, /^ if:/m, name); + } + assert.doesNotMatch(workflow, /discovered_library|reported_library|testdata\/discovery/); +}); + +test('all translator matrix failures remain failures', () => { + for (const name of ['test', 'race', 'coverage', 'cross-runtime']) { + const job = jobSection(name); + assert.match(job, /^ fail-fast: false$/m); + assert.doesNotMatch(job, /continue-on-error:/); + } +}); + +test('cheap CI policy checks pass before instruction and official corpus jobs', () => { + const policy = jobSection('ci_policy'); + assert.match(policy, /bash scripts\/install-ci-llvm22-ubuntu\.sh/); + assert.match(policy, /node --test \.github\/scripts\/priority-\*\.test\.cjs/); + assert.match(policy, /go test \. -run '\^TestCI' -count=1/); + assert.doesNotMatch(policy, /^ (needs|if):/m); + assert.ok(workflow.indexOf('\n ci_policy:\n') < + workflow.indexOf('\n test:\n')); +}); diff --git a/.github/scripts/priority-cross-runtime.test.cjs b/.github/scripts/priority-cross-runtime.test.cjs new file mode 100644 index 000000000..69d74a4a0 --- /dev/null +++ b/.github/scripts/priority-cross-runtime.test.cjs @@ -0,0 +1,42 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); + +const workflow = fs.readFileSync('.github/workflows/go-ci.yml', 'utf8'); +function job(name) { + const marker = `\n ${name}:\n`; + const start = workflow.indexOf(marker); + assert.notEqual(start, -1, `missing ${name}`); + const body = workflow.slice(start + marker.length); + const end = body.search(/^ [a-zA-Z][a-zA-Z0-9_-]*:\s*$/m); + return end < 0 ? body : body.slice(0, end); +} + +test('required cross-runtime inventory has sixteen exhaustive owners, not a concurrency cap', () => { + const body = job('cross-runtime'); + assert.match(body, /fail-fast: false/); + assert.doesNotMatch(body, /max-parallel:|continue-on-error:/); + assert.match(body, /owner: \[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15\]/); + assert.match(body, /-ci-isolate-root TestCrossLinuxRuntimeMatrix/); + assert.match(body, /-ci-owner \$\{\{ matrix.owner \}\}/); + assert.match(body, /-shards 16 -timeout 20m/); + assert.match(body, /\^\(TestCrossLinuxRuntimeMatrix\|TestARM64Conformance\|TestARM64RawSQSHLURuntimeSemantics\|TestARMIntegerMemoryRuntimeCross\|TestARMRawSaturationRuntimeCross\)/); + assert.match(body, /PLAN9ASM_CROSS_EXEC: '1'/); + assert.match(body, /bash scripts\/install-ci-qemu.sh/); + assert.match(body, /actions\/upload-artifact@v6/); + assert.match(body, /if: \$\{\{ !cancelled\(\) \}\}/); +}); + +test('strict completion runs even after a failed owner and keeps original tool requirements', () => { + const body = job('cross-runtime-verify'); + assert.match(body, /needs: cross-runtime/); + assert.match(body, /if: \$\{\{ !cancelled\(\) \}\}/); + assert.match(body, /-verify-ci-partitions _out\/cross-runtime-partitions/); + assert.match(body, /-shards 16 -timeout 20m/); + assert.match(body, /-ci-isolate-root TestCrossLinuxRuntimeMatrix/); + assert.match(body, /actions\/download-artifact@v8/); + assert.match(body, /gcc-i686-linux-gnu gcc-arm-linux-gnueabihf/); + assert.match(body, /gcc-aarch64-linux-gnu/); + assert.match(body, /bash scripts\/install-ci-qemu.sh/); + assert.doesNotMatch(body, /continue-on-error:|PLAN9ASM_CROSS_EXEC: '0'/); +}); diff --git a/.github/scripts/priority-module-root-parts.test.cjs b/.github/scripts/priority-module-root-parts.test.cjs new file mode 100644 index 000000000..42e8fbb34 --- /dev/null +++ b/.github/scripts/priority-module-root-parts.test.cjs @@ -0,0 +1,188 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const { spawnSync } = require('node:child_process'); +const { test } = require('node:test'); + +const workflow = fs.readFileSync( + path.join(__dirname, '../workflows/go-ci.yml'), 'utf8', +); +const script = fs.readFileSync( + path.join(__dirname, '../../scripts/test-module-ci.sh'), 'utf8', +); + +function job(name) { + const marker = `\n ${name}:\n`; + const start = workflow.indexOf(marker); + assert.notEqual(start, -1, `missing job ${name}`); + const body = workflow.slice(start + marker.length); + const end = body.search(/^ [a-zA-Z][a-zA-Z0-9_-]*:\s*$/m); + return end < 0 ? body : body.slice(0, end); +} + +function step(body, name) { + const marker = ` - name: ${name}\n`; + const start = body.indexOf(marker); + assert.notEqual(start, -1, `missing step ${name}`); + const end = body.indexOf('\n - name:', start + marker.length); + return body.slice(start, end < 0 ? body.length : end); +} + +test('race uses one two-shard wave; ordinary flavors retain eight root roles', () => { + for (const name of ['test', 'race', 'coverage']) { + const count = name === 'race' ? 16 : 8; + const roles = [ + ...Array.from({ length: count }, (_, index) => `root-${index}`), + 'profile', 'other', + ]; + const body = job(name); + const match = body.match(/^ module-role: \[([^\]]+)\]$/m); + assert.ok(match, `${name}: missing explicit module scheduling roles`); + assert.deepEqual(match[1].split(',').map(value => value.trim()), roles); + + const mappings = Array.from(body.matchAll( + /^ - \{module-role: (root-\d+|profile|other), module-owner: ([012]), root-partition: (-?\d+)\}$/gm, + ), ([, role, owner, part]) => [role, Number(owner), Number(part)]); + assert.deepEqual(mappings, [ + ...Array.from({ length: count }, (_, index) => [`root-${index}`, 0, index]), + ['profile', 1, -1], + ['other', 2, -1], + ], `${name}: role mapping repeats or loses package/shard ownership`); + assert.doesNotMatch(body, /^ module-owner: \[/m, + `${name}: Cartesian ownership repeats profile/rest`); + assert.match(body, /^ fail-fast: false$/m); + assert.doesNotMatch(body, /max-parallel:|continue-on-error:/); + } +}); + +test('root job fanout retains original workers, per-command and job budgets', () => { + assert.match(script, /test\) timeout=45m ;;/); + assert.match(script, /race\) timeout=60m; flags=\(-race\); root_partitions=16 ;;/); + assert.match(script, /coverage\) timeout=45m; flags=\(-coverprofile=coverage-root\.out\) ;;/); + assert.match(script, /-shards=32 -workers=2/); + assert.match(script, /GOMAXPROCS=2/); + for (const [name, minutes] of [['test', 60], ['race', 75], ['coverage', 60]]) { + assert.match(job(name), new RegExp(`^ timeout-minutes: ${minutes}$`, 'm')); + } + assert.match(job('module-completion'), /^ timeout-minutes: 30$/m); +}); + +test('executor and completion both request the same explicit root partition count', () => { + assert.match(script, /^root_partitions=8$/m); + assert.match(script, /-module-root-partitions=\$root_partitions/); + assert.match(script, /-module-root-partition=\$4/); + assert.match(script, /\[\[ \$# != 4/); + for (const [name, flavor, title] of [ + ['test', 'test', 'Go test (Ubuntu)'], + ['race', 'race', 'Go test (race)'], + ['coverage', 'coverage', 'Go test with coverage'], + ]) { + assert.match(step(job(name), title), new RegExp( + `bash scripts/test-module-ci\\.sh ${flavor} execute \\$\\{\\{ matrix\\.module-owner \\}\\} \\$\\{\\{ matrix\\.root-partition \\}\\}`, + )); + } + assert.match(step(job('module-completion'), 'Check complete actual module inventory and coverage'), + /bash scripts\/test-module-ci\.sh \$\{\{ matrix\.flavor \}\} complete/); +}); + +test('actual shell dispatch validates flavor-specific part bounds before launching Go', t => { + // This is only an argument-transport fixture, never compiler/test evidence. + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'module-ci-args-')); + t.after(() => fs.rmSync(directory, { recursive: true, force: true })); + fs.writeFileSync(path.join(directory, 'go'), + '#!/usr/bin/env bash\nprintf "%s\\n" SYNTHETIC_GO "$@"\n', { mode: 0o700 }); + const run = (...args) => spawnSync('bash', [ + path.join(__dirname, '../../scripts/test-module-ci.sh'), ...args, + ], { + cwd: directory, + env: { ...process.env, PATH: directory + path.delimiter + process.env.PATH, GOFLAGS: '' }, + encoding: 'utf8', + timeout: 5000, + }); + for (const [flavor, count] of [['test', 8], ['coverage', 8], ['race', 16]]) { + for (const args of [['execute', '0', String(count - 1)], ['execute', '2', '-1'], ['complete']]) { + const actual = run(flavor, ...args); + assert.equal(actual.status, 0, actual.stderr || String(actual.error)); + const fields = actual.stdout.trim().split('\n'); + assert.equal(fields[0], 'SYNTHETIC_GO'); + assert.ok(fields.includes(`-module-root-partitions=${count}`)); + assert.ok(fields.includes('-shards=32')); + assert.ok(fields.includes('-workers=2')); + } + for (const part of [String(count), '-1', '01', '1+1', '999999999999999999999']) { + const actual = run(flavor, 'execute', '0', part); + assert.equal(actual.status, 1, `${flavor}/${part}: ${actual.stderr}`); + assert.doesNotMatch(actual.stdout, /SYNTHETIC_GO/); + } + } +}); + +test('each root part has an exact separate artifact and strict download destination', () => { + for (const name of ['test', 'race', 'coverage']) { + const body = job(name); + assert.match(body, /name: module-(?:test|race|coverage)-.*\$\{\{ matrix\.module-role \}\}/); + assert.match(body, /if-no-files-found: error/); + } + const aggregate = job('module-completion'); + for (let part = 0; part < 16; part++) { + const download = step(aggregate, `Download root part ${part}`); + assert.match(download, new RegExp( + `^ name: module-\\$\\{\\{ matrix\\.flavor \\}\\}-\\$\\{\\{ matrix\\.go-version \\}\\}-root-${part}$`, 'm', + )); + assert.match(download, new RegExp( + `^ path: _out/module-\\$\\{\\{ matrix\\.flavor \\}\\}-owners/owner-0000/part-${String(part).padStart(4, '0')}$`, 'm', + )); + if (part >= 8) { + assert.match(download, /if: \$\{\{ !cancelled\(\) && matrix\.flavor == 'race' \}\}/); + } else { + assert.doesNotMatch(download, /matrix\.flavor ==/); + } + } + assert.match(step(aggregate, 'Download profile owner'), /-profile$/m); + assert.match(step(aggregate, 'Download remaining-package owner'), /-other$/m); + assert.doesNotMatch(aggregate, /merge-multiple:|pattern:/); + assert.match(aggregate, /needs: \[test, race, coverage\]/); + assert.match(aggregate, /^ if: \$\{\{ !cancelled\(\) \}\}$/m); +}); + +test('owner-two side gates run once and Windows remains a complete independent suite', () => { + const body = job('test'); + for (const title of [ + 'Required exact downloaded AMD64 numerical fixture', + 'Go test (cmd/plan9asm, Ubuntu)', + 'Go test (cmd/plan9asmll, Ubuntu)', + ]) { + assert.match(step(body, title), /matrix\.module-owner == 2/); + } + assert.match(body, + /\{os: windows-latest, go-version: '1\.27\.x', module-role: other, module-owner: 2, root-partition: -1, nested_modules: true\}/); + assert.match(step(body, 'Go test with coverage (Windows)'), /go test -count=1 -timeout=30m \.\/\.\.\./); + assert.match(body, + /go-version: \['1\.20\.x', '1\.21\.x', '1\.22\.x', '1\.23\.x', '1\.24\.x', '1\.25\.x', '1\.26\.x', '1\.27\.x'\]/); +}); + +test('selected native CI remains a separate sixteen-owner serial protocol', () => { + const body = job('cross-runtime'); + assert.match(body, /owner: \[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15\]/); + assert.match(body, /-ci-owner \$\{\{ matrix.owner \}\} -shards 16 -timeout 20m/); + assert.doesNotMatch(body, /module-root-partition|module-owner|(?:^|\s)-workers(?:=|\s)/); +}); + +test('Windows CLI suites cannot be suppressed by the root package timeout', () => { + const body = job('windows-cli'); + assert.match(body, /^ needs: ci_policy$/m); + assert.match(body, /^ runs-on: windows-latest$/m); + assert.match(body, /^ fail-fast: false$/m); + assert.match(body, /^ command: \[plan9asm, plan9asmll\]$/m); + assert.doesNotMatch(body, /max-parallel:|continue-on-error:/); + + const execute = step(body, 'Run complete CLI suite with coverage'); + assert.match(execute, /working-directory: cmd\/\$\{\{ matrix\.command \}\}/); + assert.match(execute, + /go test -count=1 -timeout=30m \.\/\.\.\. -coverprofile=\.\.\/\.\.\/coverage-windows-\$\{\{ matrix\.command \}\}\.out/); + assert.doesNotMatch(execute, /-run[=\s]|-skip[=\s]/); + assert.match(step(body, 'Upload Windows CLI coverage to Codecov'), + /files: \.\/coverage-windows-\$\{\{ matrix\.command \}\}\.out/); + assert.doesNotMatch(job('test'), /cd cmd\/plan9asm.*coverage-windows/); +}); diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 39dd98fc6..47aad7807 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -18,8 +18,35 @@ permissions: contents: read jobs: + ci_policy: + name: CI policy preflight + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install LLVM 22 + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version: '1.27.1' + cache: true + + - name: Workflow scheduling regression tests + run: node --test .github/scripts/priority-*.test.cjs + + - name: CI policy tests + run: go test . -run '^TestCI' -count=1 + + fmt: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} steps: - name: Checkout @@ -41,19 +68,16 @@ jobs: fi build: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v7 - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh - name: Setup Go uses: ./.github/actions/setup-go @@ -75,47 +99,20 @@ jobs: - name: ARM64 Plan 9 reference corpus run: scripts/check-arm64-plan9-corpus.sh - reported_library_list: - name: Third-party corpus matrix - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} - outputs: - libraries: ${{ steps.matrix.outputs.libraries }} - - steps: - - name: Checkout - uses: actions/checkout@v7 - - - name: Read third-party library manifest - id: matrix - run: | - manifest=testdata/corpus/reported-libraries.json - if ! jq -e '.libraries | length > 0 and all(.[]; (.id | test("^[a-z0-9]+(-[a-z0-9]+)*$")))' "$manifest" >/dev/null; then - echo "third-party library ids must match ^[a-z0-9]+(-[a-z0-9]+)*$" >&2 - exit 1 - fi - libraries=$(jq -c '[.libraries[].id]' "$manifest") - printf '%s\n' "libraries=$libraries" >> "$GITHUB_OUTPUT" - reported_library_corpus: - name: third-party-corpus (${{ matrix.library }}) - needs: reported_library_list - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} - strategy: - fail-fast: false - matrix: - library: ${{ fromJSON(needs.reported_library_list.outputs.libraries) }} + benchmark: + needs: ci_policy + name: compile benchmark (Go 1.27, LLVM 22) + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 35 steps: - name: Checkout uses: actions/checkout@v7 - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh - name: Setup Go uses: ./.github/actions/setup-go @@ -123,12 +120,26 @@ jobs: go-version: '1.27.x' cache: true - - name: Scan latest module and compile complete asm corpus + - name: Translate and compile standard-library assembly env: - REPORTED_LIBRARY: ${{ matrix.library }} - run: scripts/check-reported-library-corpus.sh "$REPORTED_LIBRARY" + PLAN9ASM_BENCHMARK_MAX_TARGET_SECONDS: '300' + PLAN9ASM_BENCHMARK_MAX_TOTAL_SECONDS: '900' + run: scripts/benchmark-compile.sh + + - name: Upload benchmark report + if: always() + uses: actions/upload-artifact@v6 + with: + name: compile-benchmark + path: _out/compile-benchmark + if-no-files-found: error + + + + stdlib-corpus: + needs: ci_policy name: stdlib-corpus (${{ matrix.os }}, Go ${{ matrix.go-version }}) runs-on: ${{ github.repository_owner == 'xgo-dev' && matrix.runner || matrix.os }} strategy: @@ -156,12 +167,8 @@ jobs: - name: Install LLVM 22 (Ubuntu) if: runner.os == 'Linux' - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh - name: Install LLVM 22 (macOS) if: runner.os == 'macOS' @@ -215,23 +222,37 @@ jobs: scripts/check-go-asm-coverage.sh cross-runtime: + needs: ci_policy name: cross compile, link, and run (Linux) - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + strategy: + # Every selected root is owned exactly once; owner zero isolates the + # giant matrix root. A failure must not cancel other owners. + fail-fast: false + matrix: + owner: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + + env: + GOFLAGS: '' + GOENV: 'off' + GOWORK: 'off' + GOTOOLCHAIN: local + PLAN9ASM_CROSS_EXEC: '1' steps: - name: Checkout uses: actions/checkout@v7 - - name: Install LLVM, cross compilers, and QEMU - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y \ - llvm-22-dev clang-22 lld-22 \ - gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu \ - qemu-user - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + - name: Install LLVM and cross compilers + timeout-minutes: 14 + run: >- + bash scripts/install-ci-llvm22-ubuntu.sh + gcc-i686-linux-gnu gcc-arm-linux-gnueabihf + gcc-aarch64-linux-gnu + + - name: Install pinned QEMU user-mode emulators + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh - name: Setup Go uses: ./.github/actions/setup-go @@ -240,12 +261,78 @@ jobs: cache: true - name: Cross transpile, compile, link, and run - env: - PLAN9ASM_CROSS_EXEC: '1' - run: go test . -run '^(TestCrossLinuxRuntimeMatrix|TestARM64Conformance)' -count=1 -v + run: | + mkdir -p _out + go run ./cmd/plan9asmtest -out _out/cross-runtime-owner \ + -ci-selector '^(TestCrossLinuxRuntimeMatrix|TestARM64Conformance|TestARM64RawSQSHLURuntimeSemantics|TestARMIntegerMemoryRuntimeCross|TestARMRawSaturationRuntimeCross)' \ + -ci-isolate-root TestCrossLinuxRuntimeMatrix \ + -ci-owner ${{ matrix.owner }} -shards 16 -timeout 20m + + - name: Retain every owner receipt, including failures + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: cross-runtime-owner-${{ matrix.owner }} + path: _out/cross-runtime-owner + if-no-files-found: error + + cross-runtime-verify: + name: cross-runtime exhaustive completion + needs: cross-runtime + if: ${{ !cancelled() }} + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + env: + GOFLAGS: '' + GOENV: 'off' + GOWORK: 'off' + GOTOOLCHAIN: local + PLAN9ASM_CROSS_EXEC: '1' + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Install LLVM and cross compilers + timeout-minutes: 14 + run: >- + bash scripts/install-ci-llvm22-ubuntu.sh + gcc-i686-linux-gnu gcc-arm-linux-gnueabihf + gcc-aarch64-linux-gnu + + - name: Install pinned QEMU user-mode emulators + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version: '1.27.x' + cache: true + + - name: Download all owner evidence + uses: actions/download-artifact@v8 + with: + pattern: cross-runtime-owner-* + path: _out/cross-runtime-partitions + + - name: Verify the fresh complete five-prefix inventory + run: | + go run ./cmd/plan9asmtest -out _out/cross-runtime-aggregate \ + -verify-ci-partitions _out/cross-runtime-partitions \ + -ci-selector '^(TestCrossLinuxRuntimeMatrix|TestARM64Conformance|TestARM64RawSQSHLURuntimeSemantics|TestARMIntegerMemoryRuntimeCross|TestARMRawSaturationRuntimeCross)' \ + -ci-isolate-root TestCrossLinuxRuntimeMatrix -shards 16 -timeout 20m + + - name: Retain completion evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: cross-runtime-completion + path: _out/cross-runtime-aggregate + if-no-files-found: error test: - name: test (${{ matrix.os }}, ${{ matrix.go-version }}, ${{ matrix.nested_modules }}) + needs: ci_policy + name: test (${{ matrix.os }}, ${{ matrix.go-version }}, ${{ matrix.module-role }}) + timeout-minutes: 60 permissions: contents: read id-token: write @@ -254,16 +341,26 @@ jobs: # Keep both matrix variants visible even if one fails. fail-fast: false matrix: + os: [ubuntu-latest] + go-version: ['1.20.x', '1.21.x', '1.22.x', '1.23.x', '1.24.x', '1.25.x', '1.26.x', '1.27.x'] + module-role: [root-0, root-1, root-2, root-3, root-4, root-5, root-6, root-7, profile, other] include: - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.20.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.21.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.22.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.23.x', nested_modules: false} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.24.x', nested_modules: true} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.25.x', nested_modules: true} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.26.x', nested_modules: true} - - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], go-version: '1.27.x', nested_modules: true} - - {os: windows-latest, go-version: '1.27.x', nested_modules: true} + - {module-role: root-0, module-owner: 0, root-partition: 0} + - {module-role: root-1, module-owner: 0, root-partition: 1} + - {module-role: root-2, module-owner: 0, root-partition: 2} + - {module-role: root-3, module-owner: 0, root-partition: 3} + - {module-role: root-4, module-owner: 0, root-partition: 4} + - {module-role: root-5, module-owner: 0, root-partition: 5} + - {module-role: root-6, module-owner: 0, root-partition: 6} + - {module-role: root-7, module-owner: 0, root-partition: 7} + - {module-role: profile, module-owner: 1, root-partition: -1} + - {module-role: other, module-owner: 2, root-partition: -1} + - {os: ubuntu-latest, runner: [qiniu, ubuntu-24.04-large], nested_modules: false} + - {go-version: '1.24.x', nested_modules: true} + - {go-version: '1.25.x', nested_modules: true} + - {go-version: '1.26.x', nested_modules: true} + - {go-version: '1.27.x', nested_modules: true} + - {os: windows-latest, go-version: '1.27.x', module-role: other, module-owner: 2, root-partition: -1, nested_modules: true} steps: - name: Checkout @@ -283,17 +380,18 @@ jobs: - name: Install LLVM 22 (Ubuntu) if: runner.os == 'Linux' - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross - name: Install LLVM 22 (Windows) if: runner.os == 'Windows' uses: ./.github/actions/setup-windows-llvm22 + - name: Install required 386 execution runtime + if: runner.os == 'Linux' + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + - name: LLVM tools info shell: bash run: | @@ -302,52 +400,132 @@ jobs: command -v clang clang --version + - name: Setup required wasm execution runtime + uses: actions/setup-node@v6 + with: + node-version: '22' + - name: Go test (Ubuntu) if: runner.os != 'Windows' - run: go test ./... + run: bash scripts/test-module-ci.sh test execute ${{ matrix.module-owner }} ${{ matrix.root-partition }} + + - name: Required exact downloaded AMD64 numerical fixture + if: runner.os == 'Linux' && matrix.go-version == '1.27.x' && matrix.module-owner == 2 + shell: bash + run: | + set -euo pipefail + test "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" = linux/amd64 + scope='^TestAMD64DownloadedFixedLocalLLVM22Numerical$' + go test ./gotoolprofile -list="$scope" | grep -Fx 'TestAMD64DownloadedFixedLocalLLVM22Numerical' + go test ./gotoolprofile -run="$scope" -count=1 -timeout=10m + + - name: Retain complete module owner evidence (Ubuntu) + if: ${{ !cancelled() && runner.os != 'Windows' }} + uses: actions/upload-artifact@v6 + with: + name: module-test-${{ matrix.go-version }}-${{ matrix.module-role }} + path: _out/module-test/ + if-no-files-found: error - name: Go test (cmd/plan9asm, Ubuntu) - if: runner.os != 'Windows' && matrix.nested_modules - run: go test ./... + if: ${{ !cancelled() && runner.os != 'Windows' && matrix.nested_modules && matrix.module-owner == 2 }} + run: go test -timeout=30m ./... working-directory: cmd/plan9asm - name: Go test (cmd/plan9asmll, Ubuntu) - if: runner.os != 'Windows' && matrix.nested_modules - run: go test ./... + if: ${{ !cancelled() && runner.os != 'Windows' && matrix.nested_modules && matrix.module-owner == 2 }} + run: go test -timeout=30m ./... working-directory: cmd/plan9asmll + - name: Go test with coverage (Windows) if: runner.os == 'Windows' && matrix.go-version == '1.27.x' shell: bash run: | set -euo pipefail - go test ./... -coverprofile=coverage-windows-root.out - (cd cmd/plan9asm && go test ./... -coverprofile=../../coverage-windows-plan9asm.out) - (cd cmd/plan9asmll && go test ./... -coverprofile=../../coverage-windows-plan9asmll.out) + go test -count=1 -timeout=30m ./... -coverprofile=coverage-windows-root.out - - name: Go test (Windows) - if: runner.os == 'Windows' && matrix.go-version != '1.27.x' - shell: bash - run: | - set -euo pipefail - go test ./... - (cd cmd/plan9asm && go test ./...) - (cd cmd/plan9asmll && go test ./...) + - name: Setup Python for Windows coverage upload + if: runner.os == 'Windows' && matrix.go-version == '1.27.x' + uses: actions/setup-python@v7 + with: + python-version: '3.x' - name: Upload Windows coverage to Codecov if: runner.os == 'Windows' && matrix.go-version == '1.27.x' uses: codecov/codecov-action@v7 with: + # The binary path downloads a GPG key from cli.codecov.io, which + # failed on Windows after the Go coverage suites passed. + use_pypi: true use_oidc: true - files: ./coverage-windows-root.out,./coverage-windows-plan9asm.out,./coverage-windows-plan9asmll.out + files: ./coverage-windows-root.out disable_search: true flags: windows name: windows-go1.27 fail_ci_if_error: true verbose: true + # A root test failure/timeout must not prevent either nested module from + # running. Each CLI owns its full suite, job budget and coverage result. + windows-cli: + needs: ci_policy + name: test (windows-latest, 1.27.x, ${{ matrix.command }}) + runs-on: windows-latest + timeout-minutes: 60 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + command: [plan9asm, plan9asmll] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version: '1.27.x' + cache: true + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.x' + + - name: Install LLVM 22 + uses: ./.github/actions/setup-windows-llvm22 + + - name: Setup required wasm execution runtime + uses: actions/setup-node@v6 + with: + node-version: '22' + + - name: Run complete CLI suite with coverage + working-directory: cmd/${{ matrix.command }} + shell: bash + run: | + set -euo pipefail + go test -count=1 -timeout=30m ./... -coverprofile=../../coverage-windows-${{ matrix.command }}.out + test -s ../../coverage-windows-${{ matrix.command }}.out + + - name: Upload Windows CLI coverage to Codecov + uses: codecov/codecov-action@v7 + with: + use_pypi: true + use_oidc: true + files: ./coverage-windows-${{ matrix.command }}.out + disable_search: true + flags: windows + name: windows-go1.27-${{ matrix.command }} + fail_ci_if_error: true + verbose: true + arm-scan: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} strategy: fail-fast: false matrix: @@ -364,17 +542,8 @@ jobs: uses: actions/checkout@v7 - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - for attempt in 1 2 3; do - sudo apt-get update && sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev && break - if [ "$attempt" -eq 3 ]; then - exit 1 - fi - sleep 5 - done - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh - name: Setup Go uses: ./.github/actions/setup-go @@ -407,7 +576,33 @@ jobs: PY race: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + name: race (${{ matrix.module-role }}) + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + module-role: [root-0, root-1, root-2, root-3, root-4, root-5, root-6, root-7, root-8, root-9, root-10, root-11, root-12, root-13, root-14, root-15, profile, other] + include: + - {module-role: root-0, module-owner: 0, root-partition: 0} + - {module-role: root-1, module-owner: 0, root-partition: 1} + - {module-role: root-2, module-owner: 0, root-partition: 2} + - {module-role: root-3, module-owner: 0, root-partition: 3} + - {module-role: root-4, module-owner: 0, root-partition: 4} + - {module-role: root-5, module-owner: 0, root-partition: 5} + - {module-role: root-6, module-owner: 0, root-partition: 6} + - {module-role: root-7, module-owner: 0, root-partition: 7} + - {module-role: root-8, module-owner: 0, root-partition: 8} + - {module-role: root-9, module-owner: 0, root-partition: 9} + - {module-role: root-10, module-owner: 0, root-partition: 10} + - {module-role: root-11, module-owner: 0, root-partition: 11} + - {module-role: root-12, module-owner: 0, root-partition: 12} + - {module-role: root-13, module-owner: 0, root-partition: 13} + - {module-role: root-14, module-owner: 0, root-partition: 14} + - {module-role: root-15, module-owner: 0, root-partition: 15} + - {module-role: profile, module-owner: 1, root-partition: -1} + - {module-role: other, module-owner: 2, root-partition: -1} steps: - name: Checkout @@ -420,18 +615,49 @@ jobs: cache: true - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross + + - name: Install required 386 execution runtime + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + + - name: Setup required wasm execution runtime + uses: actions/setup-node@v6 + with: + node-version: '22' - name: Go test (race) - run: go test -race ./... + run: bash scripts/test-module-ci.sh race execute ${{ matrix.module-owner }} ${{ matrix.root-partition }} + + - name: Retain complete module race owner evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: module-race-1.27.x-${{ matrix.module-role }} + path: _out/module-race/ + if-no-files-found: error coverage: - runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-24.04' }} + needs: ci_policy + name: coverage (${{ matrix.module-role }}) + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + module-role: [root-0, root-1, root-2, root-3, root-4, root-5, root-6, root-7, profile, other] + include: + - {module-role: root-0, module-owner: 0, root-partition: 0} + - {module-role: root-1, module-owner: 0, root-partition: 1} + - {module-role: root-2, module-owner: 0, root-partition: 2} + - {module-role: root-3, module-owner: 0, root-partition: 3} + - {module-role: root-4, module-owner: 0, root-partition: 4} + - {module-role: root-5, module-owner: 0, root-partition: 5} + - {module-role: root-6, module-owner: 0, root-partition: 6} + - {module-role: root-7, module-owner: 0, root-partition: 7} + - {module-role: profile, module-owner: 1, root-partition: -1} + - {module-role: other, module-owner: 2, root-partition: -1} permissions: contents: read id-token: write @@ -447,31 +673,205 @@ jobs: cache: true - name: Install LLVM 22 - run: | - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-22 main" | sudo tee /etc/apt/sources.list.d/llvm.list - wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/llvm-snapshot.asc >/dev/null - sudo apt-get update - sudo apt-get install -y llvm-22-dev clang-22 libclang-22-dev lld-22 libunwind-22-dev libc++-22-dev - echo "PATH=/usr/lib/llvm-22/bin:$PATH" >> "$GITHUB_ENV" + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross + + - name: Install required 386 execution runtime + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + + - name: Setup required wasm execution runtime + uses: actions/setup-node@v6 + with: + node-version: '22' - name: Go test with coverage - run: | - go test ./... -coverprofile=coverage-root.out - go tool cover -func=coverage-root.out | tail -n 1 + run: bash scripts/test-module-ci.sh coverage execute ${{ matrix.module-owner }} ${{ matrix.root-partition }} + + - name: Retain complete module coverage owner evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: module-coverage-1.27.x-${{ matrix.module-role }} + path: _out/module-coverage/ + if-no-files-found: error - name: Go test with coverage (cmd/plan9asm) + if: ${{ !cancelled() && matrix.module-owner == 2 }} run: | - go test ./... -coverprofile=coverage-plan9asm.out + go test -timeout=30m ./... -coverprofile=coverage-plan9asm.out go tool cover -func=coverage-plan9asm.out | tail -n 1 working-directory: cmd/plan9asm - name: Go test with coverage (cmd/plan9asmll) + if: ${{ !cancelled() && matrix.module-owner == 2 }} run: | - go test ./... -coverprofile=coverage-plan9asmll.out + go test -timeout=30m ./... -coverprofile=coverage-plan9asmll.out go tool cover -func=coverage-plan9asmll.out | tail -n 1 working-directory: cmd/plan9asmll + module-completion: + needs: [test, race, coverage] + if: ${{ !cancelled() }} + runs-on: ${{ github.repository_owner == 'xgo-dev' && fromJSON('["qiniu", "ubuntu-24.04-large"]') || 'ubuntu-latest' }} + timeout-minutes: 30 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + include: + - {flavor: test, go-version: '1.20.x'} + - {flavor: test, go-version: '1.21.x'} + - {flavor: test, go-version: '1.22.x'} + - {flavor: test, go-version: '1.23.x'} + - {flavor: test, go-version: '1.24.x'} + - {flavor: test, go-version: '1.25.x'} + - {flavor: test, go-version: '1.26.x'} + - {flavor: test, go-version: '1.27.x'} + - {flavor: race, go-version: '1.27.x'} + - {flavor: coverage, go-version: '1.27.x'} + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Setup Go + uses: ./.github/actions/setup-go + with: + go-version: ${{ matrix.go-version }} + cache: true + - name: Install LLVM 22 + timeout-minutes: 14 + run: bash scripts/install-ci-llvm22-ubuntu.sh gcc-i686-linux-gnu libc6-dev-i386-cross + - name: Install required 386 execution runtime + timeout-minutes: 5 + run: bash scripts/install-ci-qemu.sh + - name: Setup required wasm execution runtime + uses: actions/setup-node@v6 + with: + node-version: '22' + - name: Download root part 0 + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-0 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0000 + - name: Download root part 1 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-1 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0001 + - name: Download root part 2 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-2 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0002 + - name: Download root part 3 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-3 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0003 + - name: Download root part 4 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-4 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0004 + - name: Download root part 5 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-5 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0005 + - name: Download root part 6 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-6 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0006 + - name: Download root part 7 + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-7 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0007 + - name: Download root part 8 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-8 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0008 + - name: Download root part 9 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-9 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0009 + - name: Download root part 10 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-10 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0010 + - name: Download root part 11 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-11 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0011 + - name: Download root part 12 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-12 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0012 + - name: Download root part 13 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-13 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0013 + - name: Download root part 14 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-14 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0014 + - name: Download root part 15 + if: ${{ !cancelled() && matrix.flavor == 'race' }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-root-15 + path: _out/module-${{ matrix.flavor }}-owners/owner-0000/part-0015 + - name: Download profile owner + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-profile + path: _out/module-${{ matrix.flavor }}-owners/owner-0001 + - name: Download remaining-package owner + if: ${{ !cancelled() }} + uses: actions/download-artifact@v8 + with: + name: module-${{ matrix.flavor }}-${{ matrix.go-version }}-other + path: _out/module-${{ matrix.flavor }}-owners/owner-0002 + - name: Check complete actual module inventory and coverage + if: ${{ !cancelled() }} + run: bash scripts/test-module-ci.sh ${{ matrix.flavor }} complete + - name: Retain independent aggregate evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v6 + with: + name: module-verified-${{ matrix.flavor }}-${{ matrix.go-version }} + path: _out/module-${{ matrix.flavor }}-verified/ + if-no-files-found: error + - name: Show complete module coverage + if: matrix.flavor == 'coverage' + run: go tool cover -func=coverage-root.out | tail -n 1 - name: Upload coverage to Codecov + if: matrix.flavor == 'coverage' uses: codecov/codecov-action@v7 with: use_oidc: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..bbe7e64e2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/_out/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..81dfd7fe1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,56 @@ +# plan9asm development + +plan9asm translates Go Plan 9 assembly to LLVM IR. This repository owns the +translator, shared Go source/tool observations, official instruction coverage +and independent regression tests. External scanning, ledgers and compatibility +batches live in [llgo-compat/plan9asm](https://github.com/llgo-compat/plan9asm). + +## Start here + +1. Inspect `git status --short`, `git worktree list` and `git remote -v`. + Preserve existing work and reuse a persistent idle worktree. +2. Read [current work](docs/development/current-work.md), + [instruction families](docs/development/instructions.md) and + [validation](docs/development/validation.md) for the task. + +## Rules + +- Push only to an allowed `cpunion` or `dastai` fork. Create/update the + contribution PR; do not directly push, merge, approve or close upstream PRs. +- Use current Go 1.27 and LLVM 22. Never fall back to another LLVM version; + missing required tools/backends fail, never skip. +- Support all five architectures: 386, amd64, arm, arm64 and wasm. + Host-inapplicable runtime tests require an explicit cross-runtime counterpart. + Never generate LLVM `blockaddress` for wasm. +- Use TDD. Consult complete official Go tables and implement typed instruction + families and operand formats. Each ecosystem-discovered defect must have an + independent regression and a rule-class fix, not a library-specific patch. +- Run tests with Go while llgo support is incomplete; do not add `!llgo` tags. +- Keep rejection, source context, translation, LLVM compilation and executed + semantics distinct. Infrastructure failures never establish source rejection. +- Use `apply_patch`, readable multi-line code and moderate blank lines, then + `gofmt`. Release LLVM objects and owned subprocess resources promptly. +- Do not commit private paths/hosts, package caches, binaries or reports. + Keep evidence under ignored `_out/`; preserve failed reports as failed. +- Freeze source and tools during validation, even documentation. Never edit a + running snapshot or reuse its report to certify a changed revision. +- Commit verified batches promptly. Stage repairs in the allowed fork; retain + draft status until current-head tests, CI, review and coverage pass. + +## Commands + +After selecting the tools documented in validation: + +```sh +go test . -run '' -count=1 +go test ./... -count=1 -timeout=45m +(cd cmd/plan9asm && go test ./... -count=1) +(cd cmd/plan9asmll && go test ./... -count=1) +scripts/check-go-asm-coverage.sh +scripts/check-arm64-plan9-corpus.sh +scripts/check-stdlib-corpus.sh +scripts/benchmark-compile.sh +``` + +Do not reintroduce external discovery jobs or ledger gates into translator CI. +Keep this entry point short; durable procedures belong in linked documents. diff --git a/README.md b/README.md index 4eb8494f1..50be145e3 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,18 @@ go run -C cmd/plan9asmll . \ -report /tmp/plan9asmll-x86.json ``` +For an ordinary external module, `-feature-profile` explicitly consumes a +private `actual_go_cpu_profile_consumer_v1` invocation artifact. It re-observes +the actual Go marker/environment/tool identity, loads that same environment, +checks selected source hashes and package-role macros, and records CPP/LLVM +output hashes. Source directories and module replacements must match the +artifact's exact source root. Special and test-only package roles remain +outside this mode. Without the flag, existing CLI behavior is unchanged. +The external runner in [llgo-compat/plan9asm](https://github.com/llgo-compat/plan9asm) +owns module discovery, ledger records and third-party compatibility CI. It pins +this translator and the shared `gotoolprofile` package; translation/object +checks remain distinct from operand-form and runtime coverage. + ## Output behavior - Every asm file is printed with explicit status (`OK` or `FAIL`). diff --git a/amd64_abi.go b/amd64_abi.go new file mode 100644 index 000000000..c227c591d --- /dev/null +++ b/amd64_abi.go @@ -0,0 +1,90 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) abi0CallStackPtr(off int64) (string, error) { + sp, err := c.loadReg(SP) + if err != nil { + return "", fmt.Errorf("ABI0 call stack: %w", err) + } + addr := sp + if off != 0 { + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", next, sp, off) + addr = "%" + next + } + ptr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", ptr, addr) + return "%" + ptr, nil +} + +func (c *amd64Ctx) abi0CallArgs(callee string, sig FuncSig) ([]string, error) { + slotsByArg := make([][]FrameSlot, len(sig.Args)) + for _, slot := range sig.Frame.Params { + if slot.Index < 0 || slot.Index >= len(sig.Args) { + return nil, fmt.Errorf("amd64 call %q: ABI0 parameter at +%d has invalid argument index %d", callee, slot.Offset, slot.Index) + } + slotsByArg[slot.Index] = append(slotsByArg[slot.Index], slot) + } + args := make([]string, 0, len(sig.Args)) + for argIndex, argType := range sig.Args { + slots := slotsByArg[argIndex] + if len(slots) == 0 { + return nil, fmt.Errorf("amd64 call %q: ABI0 argument %d has no frame slot", callee, argIndex) + } + if len(slots) == 1 && len(frameSlotFields(slots[0])) == 0 { + if slots[0].Type != argType { + return nil, fmt.Errorf("amd64 call %q: ABI0 argument %d frame type %s does not match %s", callee, argIndex, slots[0].Type, argType) + } + ptr, err := c.abi0CallStackPtr(slots[0].Offset) + if err != nil { + return nil, err + } + value := c.loadFPStorage(argType, ptr, ", align 1") + args = append(args, fmt.Sprintf("%s %s", argType, value)) + continue + } + aggregate := "undef" + for _, slot := range slots { + if len(frameSlotFields(slot)) == 0 { + return nil, fmt.Errorf("amd64 call %q: ABI0 aggregate argument %d has a scalar frame slot", callee, argIndex) + } + ptr, err := c.abi0CallStackPtr(slot.Offset) + if err != nil { + return nil, err + } + value := c.loadFPStorage(slot.Type, ptr, ", align 1") + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertvalue %s %s, %s %s%s\n", inserted, argType, aggregate, slot.Type, value, frameSlotExtractSuffix(slot)) + aggregate = "%" + inserted + } + args = append(args, fmt.Sprintf("%s %s", argType, aggregate)) + } + return args, nil +} + +func (c *amd64Ctx) storeABI0CallResult(callee string, sig FuncSig, result string) error { + if len(sig.Frame.Results) == 0 { + return fmt.Errorf("amd64 call %q: ABI0 result %s has no frame slot", callee, sig.Ret) + } + fields, aggregate := parseLiteralStructFields(sig.Ret) + for _, slot := range sig.Frame.Results { + value := result + if aggregate { + if slot.Index < 0 || slot.Index >= len(fields) || fields[slot.Index] != slot.Type { + return fmt.Errorf("amd64 call %q: ABI0 result slot %d does not match %s", callee, slot.Index, sig.Ret) + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", extracted, sig.Ret, result, slot.Index) + value = "%" + extracted + } else if slot.Index != 0 || slot.Type != sig.Ret { + return fmt.Errorf("amd64 call %q: ABI0 scalar result frame does not match %s", callee, sig.Ret) + } + ptr, err := c.abi0CallStackPtr(slot.Offset) + if err != nil { + return err + } + c.storeFPStorage(slot.Type, value, ptr, ", align 1", "") + } + return nil +} diff --git a/amd64_abi0_anchor_test.go b/amd64_abi0_anchor_test.go new file mode 100644 index 000000000..87668c8cd --- /dev/null +++ b/amd64_abi0_anchor_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func TestAMD64ABI0AnchorPreservesTailJumpRelocation(t *testing.T) { + args := make([]LLVMType, 15) + for i := range args { + args[i] = I32 + } + for _, anchorName := range []string{"gcasmABI0Keep", "relocationKeep"} { + source := fmt.Sprintf("TEXT ·%s(SB),NOSPLIT,$0-0\n", anchorName) + + "JMP ·Fn(SB)\nJMP ·FnTwo(SB)\nJMP ·FnThree(SB)\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{ + "example." + anchorName: {Name: "example." + anchorName, Ret: Void}, + "example.Fn": {Name: "example.Fn", Args: args, Ret: I32}, + "example.FnTwo": {Name: "example.FnTwo", Args: args, Ret: I32}, + "example.FnThree": {Name: "example.FnThree", Args: args, Ret: I32}, + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i386-apple-darwin"}, + {"386", "i686-pc-windows-msvc"}, + } { + triple := target.triple + t.Run(anchorName+"/"+triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: triple, + ResolveSym: testResolveSym("example"), Sigs: sigs, + }) + if err != nil { + t.Fatal(err) + } + for _, symbol := range []string{"example.Fn", "example.FnTwo", "example.FnThree"} { + if !strings.Contains(ir, "ptr @"+symbol) { + t.Fatalf("ABI0 anchor omitted direct symbol relocation for %s", symbol) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + objdump := findLLVM22Tool("llvm-objdump") + if objdump == "" { + t.Fatal("LLVM 22 llvm-objdump not found") + } + dir := t.TempDir() + llPath := filepath.Join(dir, "anchor.ll") + object := filepath.Join(dir, "anchor.o") + if err := os.WriteFile(llPath, []byte(ir), 0644); err != nil { + t.Fatal(err) + } + if output, err := exec.Command(llc, "-mtriple="+triple, "-filetype=obj", llPath, "-o", object).CombinedOutput(); err != nil { + t.Fatalf("LLVM 22 object compile: %v\n%s", err, output) + } + output, err := exec.Command(objdump, "-dr", object).CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22 object dump: %v\n%s", err, output) + } + if !regexp.MustCompile(`(?m)^\s*0: e9 `).Match(output) { + t.Fatalf("anchor is not an entry-point tail jump with target relocation:\n%s", output) + } + for _, symbol := range []string{"example.Fn", "example.FnTwo", "example.FnThree"} { + if !strings.Contains(string(output), symbol) { + t.Fatalf("object omitted relocation for %s:\n%s", symbol, output) + } + } + }) + } + } +} + +func TestAMD64NonAnchorStillRejectsUnmappedTailArguments(t *testing.T) { + const source = "TEXT ·forward(SB),NOSPLIT,$0-0\nJMP ·Fn(SB)\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + args := make([]LLVMType, 7) + for i := range args { + args[i] = I32 + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: testResolveSym("example"), + Sigs: map[string]FuncSig{ + "example.forward": {Name: "example.forward", Ret: Void}, + "example.Fn": {Name: "example.Fn", Args: args, Ret: I32}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "missing arg reg") { + t.Fatalf("unmapped non-anchor tail arguments unexpectedly accepted: %v", err) + } +} diff --git a/amd64_abi0_forward.go b/amd64_abi0_forward.go new file mode 100644 index 000000000..cc8c2e746 --- /dev/null +++ b/amd64_abi0_forward.go @@ -0,0 +1,108 @@ +package plan9asm + +// inferX86ABI0Forwarder proves the complete straight-line adapter shape before +// assigning a typed ABI0 signature to an indirect CALL. The final incoming +// word is the code pointer, not an argument of the callback. This is structural +// inference: no library, function or parameter names participate. +func inferX86ABI0Forwarder(fn Func, sig FuncSig, goarch string) *FuncSig { + if goarch != "amd64" || len(sig.ArgRegs) != 0 || len(sig.Args) == 0 { + return nil + } + codeIndex := len(sig.Args) - 1 + if sig.Args[codeIndex] != I64 && sig.Args[codeIndex] != Ptr { + return nil + } + var code FrameSlot + var params []FrameSlot + codeSlots := 0 + for _, slot := range sig.Frame.Params { + if slot.Index == codeIndex { + code = slot + codeSlots++ + } else { + params = append(params, slot) + } + } + if codeSlots != 1 || len(frameSlotFields(code)) != 0 { + return nil + } + + var instructions []Instr + for _, ins := range fn.Instrs { + switch ins.Op { + case "TEXT", "PCDATA", "FUNCDATA", "GO_ARGS", "NO_LOCAL_POINTERS": + continue + } + instructions = append(instructions, ins) + } + // Two instructions per copied slot, code-pointer load, CALL, then RET. + if len(instructions) != 2*len(params)+2*len(sig.Frame.Results)+3 { + return nil + } + for i, slot := range params { + load, store := instructions[2*i], instructions[2*i+1] + if !x86ABI0ForwardCopy(load, store, slot, true) || + store.Args[1].Mem.Off != slot.Offset || slot.Offset >= code.Offset { + return nil + } + } + callIndex := 2*len(params) + 1 + load, call := instructions[callIndex-1], instructions[callIndex] + if load.Op != "MOVQ" || len(load.Args) != 2 || load.Args[0].Kind != OpFP || + load.Args[0].FPOffset != code.Offset || load.Args[1].Kind != OpReg || + call.Op != "CALL" || len(call.Args) != 1 || call.Args[0].Kind != OpReg || + call.Args[0].Reg != load.Args[1].Reg { + return nil + } + results := append([]FrameSlot(nil), sig.Frame.Results...) + for i, slot := range results { + load, store := instructions[callIndex+1+2*i], instructions[callIndex+2+2*i] + if !x86ABI0ForwardCopy(load, store, slot, false) { + return nil + } + // Removing the trailing pointer shifts the callback's results one + // pointer word earlier. Do not infer a signature from arbitrary loads. + if load.Args[0].Mem.Off != slot.Offset-8 || slot.Offset-8 < code.Offset { + return nil + } + results[i].Offset -= 8 + } + last := instructions[len(instructions)-1] + if last.Op != "RET" || len(last.Args) != 0 { + return nil + } + return &FuncSig{ + Args: append([]LLVMType(nil), sig.Args[:codeIndex]...), Ret: sig.Ret, + Frame: FrameLayout{Params: params, Results: results}, + } +} + +func x86ABI0ForwardCopy(load, store Instr, slot FrameSlot, incoming bool) bool { + if len(load.Args) != 2 || len(store.Args) != 2 || load.Op != store.Op || + load.Args[1].Kind != OpReg || store.Args[0].Kind != OpReg || + load.Args[1].Reg != store.Args[0].Reg { + return false + } + var width int64 + switch load.Op { + case "MOVB": + width = 1 + case "MOVW": + width = 2 + case "MOVL", "MOVSS": + width = 4 + case "MOVQ", "MOVSD": + width = 8 + default: + return false + } + if width != frameTypeSize(slot.Type, 8) { + return false + } + fp, sp := load.Args[0], store.Args[1] + if !incoming { + fp, sp = store.Args[1], load.Args[0] + } + return fp.Kind == OpFP && fp.FPOffset == slot.Offset && + sp.Kind == OpMem && sp.Mem.Base == SP && sp.Mem.Index == "" && sp.Mem.Sym == "" +} diff --git a/amd64_abi0_forward_test.go b/amd64_abi0_forward_test.go new file mode 100644 index 000000000..a1c4b1781 --- /dev/null +++ b/amd64_abi0_forward_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" +) + +func TestX86ABI0ForwardCopyScalarWidths(t *testing.T) { + for _, test := range []struct { + op, register string + typ LLVMType + }{ + {"MOVB", "AX", I8}, + {"MOVW", "AX", I16}, + {"MOVL", "AX", I32}, + {"MOVQ", "AX", I64}, + {"MOVSS", "X0", "float"}, + {"MOVSD", "X0", "double"}, + } { + t.Run(test.op, func(t *testing.T) { + source := fmt.Sprintf(`TEXT copy(SB),$16-16 + %s arg+8(FP), %s + %s %s, 8(SP) + %s 8(SP), %s + %s %s, ret+8(FP) + RET +`, test.op, test.register, test.op, test.register, + test.op, test.register, test.op, test.register) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + var instructions []Instr + for _, ins := range file.Funcs[0].Instrs { + if ins.Op != "TEXT" { + instructions = append(instructions, ins) + } + } + slot := FrameSlot{Offset: 8, Type: test.typ} + if !x86ABI0ForwardCopy(instructions[0], instructions[1], slot, true) || + !x86ABI0ForwardCopy(instructions[2], instructions[3], slot, false) { + t.Fatal("rejected equal-width scalar frame copies") + } + slot.Type = I64 + if test.typ == I64 || test.typ == "double" { + slot.Type = I32 + } + if x86ABI0ForwardCopy(instructions[0], instructions[1], slot, true) { + t.Fatal("accepted a copy with the wrong frame-slot width") + } + }) + } +} + +func TestX86ABI0ForwarderRequiresCompleteTypedShape(t *testing.T) { + tc := issue44Cases[3] + pkg := mustGoPackage(t, "test/forward", "package forward\n"+tc.declarations) + fn := pkg.Types.Scope().Lookup("Y").(*types.Func) + sig, err := goFuncSigForDeclaredFunc("Y", fn, "amd64", nil, nil, true) + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name, old, replacement string + valid bool + }{ + {name: "exact", valid: true}, + {name: "renamed_code_parameter", old: "code+", replacement: "function_pointer+", valid: true}, + {name: "wrong_argument_offset", old: "AX, 8(SP)", replacement: "AX, 0(SP)"}, + {name: "wrong_argument_width", old: "MOVQ b+8(FP)", replacement: "MOVL b+8(FP)"}, + {name: "missing_argument", old: "MOVQ AX, 8(SP)\n", replacement: ""}, + {name: "wrong_code_slot", old: "code+16(FP)", replacement: "code+8(FP)"}, + {name: "wrong_code_register", old: "CALL *AX", replacement: "CALL *BX"}, + {name: "wrong_result_offset", old: "16(SP), AX", replacement: "8(SP), AX"}, + {name: "wrong_result_destination", old: "ret+24(FP)", replacement: "ret+16(FP)"}, + {name: "mutation", old: "CALL *AX", replacement: "INCQ 0(SP)\nCALL *AX"}, + {name: "second_call", old: "CALL *AX", replacement: "CALL *AX\nCALL *AX"}, + {name: "branch", old: "CALL *AX", replacement: "JMP end\nCALL *AX\nend:"}, + {name: "tail_jump", old: "RET", replacement: "JMP *AX"}, + } { + t.Run(test.name, func(t *testing.T) { + source := tc.assembly + if test.old != "" { + source = strings.Replace(source, test.old, test.replacement, 1) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + got := inferX86ABI0Forwarder(file.Funcs[0], sig, "amd64") + if (got != nil) != test.valid { + t.Fatalf("inferred %+v, want valid=%v", got, test.valid) + } + if got != nil && (len(got.Args) != 2 || got.Ret != I64 || got.Frame.Results[0].Offset != 16) { + t.Fatalf("wrong callback signature: %+v", got) + } + }) + } +} + +func TestGoABI0NestedAggregateFrameLayout(t *testing.T) { + pkg := mustGoPackage(t, "test/layout", `package layout +type Mixed struct { Byte uint8; Pair [2]uint64; Z complex128 } +func Y(a Mixed, b complex64) (r Mixed) +`) + fn := pkg.Types.Scope().Lookup("Y").(*types.Func) + for _, arch := range []string{"386", "amd64", "arm", "arm64", "wasm"} { + t.Run(arch, func(t *testing.T) { + sig, err := goFuncSigForDeclaredFunc("Y", fn, arch, nil, nil, true) + if err != nil { + t.Fatal(err) + } + if sig.Args[0] != "{ i8, [2 x i64], { double, double } }" || len(sig.Frame.Params) != 7 || len(sig.Frame.Results) != 5 { + t.Fatalf("incorrect aggregate layout: %+v", sig) + } + word := int64(goWordSize(arch)) + wantOffsets := []int64{0, word, word + 8, word + 16, word + 24, word + 32, word + 36} + wantPaths := []string{", 0", ", 1, 0", ", 1, 1", ", 2, 0", ", 2, 1", ", 0", ", 1"} + for i, slot := range sig.Frame.Params { + if slot.Offset != wantOffsets[i] || frameSlotExtractSuffix(slot) != wantPaths[i] { + t.Fatalf("slot %d = %+v, want offset=%d path=%s", i, slot, wantOffsets[i], wantPaths[i]) + } + } + }) + } +} diff --git a/amd64_address_memory_class_test.go b/amd64_address_memory_class_test.go new file mode 100644 index 000000000..87647e361 --- /dev/null +++ b/amd64_address_memory_class_test.go @@ -0,0 +1,287 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86MemoryClassKeepsAddressImmediatesDistinct(t *testing.T) { + for _, source := range []string{ + "$·cell(SB)", "$·cell+8(SB)", "$16(AX)", "$16(AX)(CX*4)", + } { + t.Run(source, func(t *testing.T) { + operand, err := parseOperandForArch(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if operand.Kind != OpSym || !strings.HasPrefix(operand.Sym, "$") { + t.Fatalf("source address identity lost: %#v", operand) + } + if isAMD64MemoryOperand(operand) { + t.Fatalf("Go TYPE_ADDR became a memory operand: %#v", operand) + } + // A caller-built operand must obey the same lexical boundary. + operand.Sym = " \t" + operand.Sym + if isAMD64MemoryOperand(operand) { + t.Fatal("whitespace granted address-immediate memory permission") + } + }) + } + + for _, source := range []string{ + "·cell(SB)", "·cell+8(SB)", "(AX)", "16(AX)(CX*4)", + "arg+0(FP)", "local-8(SP)", "(FS)", "(GS)", + } { + t.Run("memory/"+source, func(t *testing.T) { + operand, err := parseOperandForArch(ArchAMD64, source) + if err != nil || !isAMD64MemoryOperand(operand) { + t.Fatalf("ordinary memory class changed: %#v: %v", operand, err) + } + }) + } +} + +func TestX86AddressImmediateMemoryOnlyFormsRejectAcrossAPIs(t *testing.T) { + // Independent witnesses from Go's Ym/Ymb/Yml, yxchg, SSE memory, + // scalar-memory and non-temporal rows. The source contains a TYPE_ADDR + // where a real memory operand is required; relocation is not a load/store. + rows := []string{ + "XADDB AL,%s", "XADDW CX,%s", "XADDL CX,%s", + "XCHGB AL,%s", "XCHGW CX,%s", "XCHGL CX,%s", + "INCB %s", "INCW %s", "INCL %s", + "DECB %s", "DECW %s", "DECL %s", + "NEGB %s", "NEGW %s", "NEGL %s", + "NOTB %s", "NOTW %s", "NOTL %s", + "BTCL $1,%s", "BTRL $1,%s", "BTSL $1,%s", + "STMXCSR %s", "LDMXCSR %s", "CLFLUSH %s", + "MOVNTIL CX,%s", "MOVNTO X0,%s", "MOVNTPS X0,%s", + "ADDSUBPS %s,X0", "ADDSUBPD %s,X0", + "AESENC %s,X0", "PSHUFB %s,X0", "LDDQU %s,X0", + } + for _, arch := range []string{"386", "amd64"} { + for _, row := range rows { + for _, address := range []string{"$·cell(SB)", "$16(AX)"} { + instruction := fmt.Sprintf(row, address) + t.Run(arch+"/"+instruction, func(t *testing.T) { + memorySource := "TEXT ·Value(SB),4,$0-0\n" + + fmt.Sprintf(row, strings.TrimPrefix(address, "$")) + "\nRET\n" + goAssemblerOracleOutput(t, arch, memorySource, "", true) + source := "TEXT ·Value(SB),4,$0-0\n" + instruction + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + for _, target := range x86AddressMemoryTargets(arch) { + for _, api := range []string{"text", "module", "go"} { + t.Run(target+"/"+api, func(t *testing.T) { + _, err := x86AddressMemoryAPI(t, api, source, arch, target) + if err == nil { + t.Fatal("Go-rejected address immediate acquired memory semantics") + } + }) + } + } + }) + } + } + } +} + +func TestX86AddressClassActualGoRulesExcludeMemory(t *testing.T) { + // Read the actual selected assembler's complete classification/coverage + // relation. This oracle does not use the translator's operand predicate. + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(runtime.GOROOT(), + "src/cmd/internal/obj/x86/asm6.go"), nil, 0) + if err != nil { + t.Fatal(err) + } + classes := make(map[string]bool) + foundAddress, foundConstant := false, false + edges := make(map[string][]string) + ast.Inspect(file, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.FuncDecl: + if node.Name.Name != "oclass" { + return true + } + ast.Inspect(node.Body, func(child ast.Node) bool { + clause, ok := child.(*ast.CaseClause) + if !ok { + return true + } + for _, value := range clause.List { + name, ok := value.(*ast.SelectorExpr) + if !ok || name.Sel.Name != "TYPE_ADDR" && name.Sel.Name != "TYPE_CONST" { + continue + } + foundAddress = foundAddress || name.Sel.Name == "TYPE_ADDR" + foundConstant = foundConstant || name.Sel.Name == "TYPE_CONST" + for _, statement := range clause.Body { + ast.Inspect(statement, func(inside ast.Node) bool { + if result, ok := inside.(*ast.ReturnStmt); ok { + if len(result.Results) != 1 { + t.Fatal("unexpected Go operand classification return") + } + ident, ok := result.Results[0].(*ast.Ident) + if !ok || !strings.HasPrefix(ident.Name, "Y") { + t.Fatal("unregistered Go address/constant classifier") + } + classes[ident.Name] = true + } + return true + }) + } + } + return true + }) + return false + case *ast.AssignStmt: + for _, left := range node.Lhs { + index, ok := left.(*ast.IndexExpr) + if !ok { + continue + } + name, ok := index.X.(*ast.Ident) + if !ok || name.Name != "ycover" { + continue + } + sum, ok := index.Index.(*ast.BinaryExpr) + if !ok || sum.Op != token.ADD { + t.Fatal("unregistered Go class coverage index") + } + product, ok := sum.X.(*ast.BinaryExpr) + if !ok || product.Op != token.MUL { + t.Fatal("unregistered Go class coverage product") + } + from, fromOK := product.X.(*ast.Ident) + size, sizeOK := product.Y.(*ast.Ident) + to, toOK := sum.Y.(*ast.Ident) + if !fromOK || !sizeOK || !toOK || size.Name != "Ymax" { + t.Fatal("unregistered Go class coverage operands") + } + if from.Name == "i" && to.Name == "i" { + continue // The actual diagonal only relates a class to itself. + } + if !strings.HasPrefix(from.Name, "Y") || !strings.HasPrefix(to.Name, "Y") { + t.Fatal("unregistered computed Go class relation") + } + edges[from.Name] = append(edges[from.Name], to.Name) + } + } + return true + }) + if !foundAddress || !foundConstant || !classes["Yiauto"] || !classes["Yi32"] || len(edges) == 0 { + t.Fatal("incomplete actual Go TYPE_ADDR/constant/coverage inventory") + } + for changed := true; changed; { + changed = false + for from := range classes { + for _, to := range edges[from] { + if !classes[to] { + classes[to], changed = true, true + } + } + } + } + for _, memory := range []string{"Ym", "Ymb", "Yml", "Ymm", "Yxm", "Yxvm", "Yyvm", "Yzvm"} { + if classes[memory] { + t.Fatalf("actual Go address immediate acquired memory-class reachability: %s", memory) + } + } +} + +func TestX86AddressClassPreservesActualMemoryAndRelocationObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + move := "MOVQ" + if arch == "386" { + move = "MOVL" + } + for _, body := range []string{ + move + " $·cell(SB),AX", + move + " $16(AX),BX", + "XADDL CX,·cell(SB)", + "XCHGL CX,16(AX)", + "INCL ·cell(SB)", + "NEGL 16(AX)", + "STMXCSR ·cell(SB)", + "LDMXCSR 16(AX)", + "PSHUFB ·cell(SB),X0", + "MOVNTPS X0,16(AX)", + } { + source := "TEXT ·Value(SB),4,$0-0\n" + body + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + for _, target := range x86AddressMemoryTargets(arch) { + for _, api := range []string{"text", "module", "go"} { + t.Run(arch+"/"+body+"/"+target+"/"+api, func(t *testing.T) { + ir, err := x86AddressMemoryAPI(t, api, source, arch, target) + if err != nil { + t.Fatal("legitimate memory/address operand lost:", err) + } + x86ResultInitializationObject(t, llc, target, ir, "address-class") + }) + } + } + } + } +} + +func x86AddressMemoryTargets(arch string) []string { + if arch == "386" { + return []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + return []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } +} + +func x86AddressMemoryAPI(t *testing.T, api, source, arch, target string) (string, error) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + options := Options{ + Goarch: arch, TargetTriple: target, ResolveSym: resolve, + Sigs: map[string]FuncSig{"Value": {Name: "Value", Ret: Void}}, + } + if api == "text" { + return Translate(file, options) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "module" { + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + return "", err + } + return module.String(), nil + } + if api != "go" { + t.Fatalf("unregistered API %q", api) + } + pkg := mustGoPackage(t, "example.invalid/addressclass", "package addressclass\nfunc Value()\n") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: target, ResolveSym: resolve, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + return "", err + } + return translated.Module.String(), nil +} diff --git a/amd64_address_memory_sinks_test.go b/amd64_address_memory_sinks_test.go new file mode 100644 index 000000000..7a8f60e92 --- /dev/null +++ b/amd64_address_memory_sinks_test.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86SymbolMemoryPointerRejectsAddressIdentityBeforeEmission(t *testing.T) { + for _, source := range []string{ + "$·cell(SB)", "$·cell+8(SB)", " \t$·cell-8(SB)", + } { + t.Run(source, func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before, temporary := output.String(), ctx.tmp + pointer, err := ctx.ptrFromSB(source) + if err == nil || pointer != "" || !strings.Contains(err.Error(), "address immediate") { + t.Fatalf("address acquired a memory pointer: %q: %v", pointer, err) + } + if output.String() != before || ctx.tmp != temporary { + t.Fatal("rejected address emitted IR or consumed a temporary") + } + }) + } + + for _, source := range []string{"·cell(SB)", "·cell+8(SB)", "·cell-8(SB)"} { + t.Run("memory/"+source, func(t *testing.T) { + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + if pointer, err := ctx.ptrFromSB(source); err != nil || pointer == "" { + t.Fatalf("ordinary symbolic memory lost: %q: %v", pointer, err) + } + }) + } +} + +func x86AddressMemorySinkRows(arch string) []string { + // These ordinary stores and x87 loads use direct OpSym dispatch, not the + // shared memory-class predicate. Their Go encoder rows still require actual + // memory, including the complete scalar and x87 storage widths below. + rows := []string{ + "MOVB AL,%s", "MOVW CX,%s", "MOVL CX,%s", + "MOVSS X0,%s", "MOVSD X0,%s", "MOVOU X0,%s", + "FMOVF F0,%s", "FMOVFP F0,%s", "FMOVD F0,%s", "FMOVDP F0,%s", + "FMOVW F0,%s", "FMOVWP F0,%s", "FMOVL F0,%s", "FMOVLP F0,%s", + "FMOVVP F0,%s", "FMOVBP F0,%s", "FMOVXP F0,%s", + "FMOVF %s,F0", "FMOVD %s,F0", "FMOVW %s,F0", "FMOVL %s,F0", + "FMOVV %s,F0", "FMOVB %s,F0", "FMOVX %s,F0", + } + if arch == "amd64" { + rows = append(rows, "MOVQ CX,%s") + } + return rows +} + +func x86AddressMemorySinkSource(row, operand string) string { + // These are compile-only syntax/operand tests. Initialize explicit register + // inputs, but do not claim execution of the external cell or x87 state. + return "TEXT ·Value(SB),4,$0-0\nMOVL $7,AX\nMOVL $9,CX\n" + + "PXOR X0,X0\nFLDZ\n" + fmt.Sprintf(row, operand) + "\nRET\n" +} + +func TestX86AddressImmediateDirectMemorySinksRejectAcrossAPIs(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86AddressMemorySinkRows(arch) { + for _, operand := range []string{"$·cell(SB)", "$·cell+8(SB)"} { + t.Run(arch+"/"+fmt.Sprintf(row, operand), func(t *testing.T) { + ordinary := x86AddressMemorySinkSource(row, strings.TrimPrefix(operand, "$")) + goAssemblerOracleOutput(t, arch, ordinary, "", true) + source := x86AddressMemorySinkSource(row, operand) + goAssemblerOracleOutput(t, arch, source, "", false) + for _, target := range x86AddressMemoryTargets(arch) { + for _, api := range []string{"text", "module", "go"} { + t.Run(target+"/"+api, func(t *testing.T) { + if _, err := x86AddressMemoryAPI(t, api, source, arch, target); err == nil { + t.Fatal("Go-rejected address bypassed memory-role validation") + } + }) + } + } + }) + } + } + } +} + +func TestX86DirectSymbolMemorySinkObjectsRemainSupported(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86AddressMemorySinkRows(arch) { + source := x86AddressMemorySinkSource(row, "·cell+8(SB)") + goAssemblerOracleOutput(t, arch, source, "", true) + for _, target := range x86AddressMemoryTargets(arch) { + for _, api := range []string{"text", "module", "go"} { + t.Run(arch+"/"+row+"/"+target+"/"+api, func(t *testing.T) { + ir, err := x86AddressMemoryAPI(t, api, source, arch, target) + if err != nil { + t.Fatal("ordinary memory sink lost:", err) + } + x86ResultInitializationObject(t, llc, target, ir, "memory-sink") + }) + } + } + } + } +} diff --git a/amd64_aes_family_effects_test.go b/amd64_aes_family_effects_test.go new file mode 100644 index 000000000..6cd884335 --- /dev/null +++ b/amd64_aes_family_effects_test.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +// Independent source inventory: six SSE names and their six AVX names. AES +// round rows are destructive only in their legacy spelling; no Go AES row +// has a K operand, broadcast or zeroing suffix. +var x86AESOracleNames = []string{"AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC", "AESKEYGENASSIST", "VAESENC", "VAESENCLAST", "VAESDEC", "VAESDECLAST", "VAESIMC", "VAESKEYGENASSIST"} + +func x86AESOracleRow(name, source, bank string) (string, int) { + if strings.Contains(name, "KEYGEN") { + return name + " $27," + source + "," + bank + "2", 2 + } + if strings.HasSuffix(name, "IMC") { + return name + " " + source + "," + bank + "2", 2 + } + if strings.HasPrefix(name, "V") { + return name + " " + source + "," + bank + "1," + bank + "2", 2 + } + return name + " " + source + "," + bank + "1", 1 +} + +func TestX86AESFamilyInitialTypedAndPrivateResults(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "X0", "X") + t.Run(arch+"/"+name, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86YMMReadSource(t, row) + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || effect.flagsRead != 0 || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatalf("complete AES typed byte/flags effect absent: known=%t err=%v", known, err) + } + for _, transfer := range []struct { + name string + fn func(Instr, *x86ResultState, []bool, int64, string) bool + }{{"public", x86ResultTransfer}, {"private", x86ScalarFPResultTransfer}} { + state := x86ResultState{flags: x86ByteFlagsAll} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + if !transfer.fn(ins, &state, nil, 0, arch) || state.xmm[destination] != 0xffff || state.flags != x86ByteFlagsAll { + t.Fatal(transfer.name, "lost complete AES input/result or preserved flags", state) + } + } + }) + } + } +} + +func TestX86AESFamilyInitialCallerSourceEffects(t *testing.T) { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "X0", "X") + t.Run(name, func(t *testing.T) { + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n" + row + fmt.Sprintf("\nVMOVSS X%d,(DX)", destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("actual Go accepted whole-family caller has no registered AES source effect", err) + } + }) + } +} + +func TestX86AESFamilyResultEveryNativeMemoryByte(t *testing.T) { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "a+0(FP)", "X") + ins := x86YMMReadSource(t, row) + for missing := -1; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/missing%d", name, missing), func(t *testing.T) { + state := x86ResultState{fp: make([]bool, 16), flags: x86ByteFlagsAll} + state.xmm[1] = 0xffff + owned := make([]bool, 16) + for i := range owned { + owned[i], state.fp[i] = true, i != missing + } + before := state.clone() + got := x86ScalarFPResultTransfer(ins, &state, owned, 16, "amd64") + if got != (missing == -1) || got && state.xmm[destination] != 0xffff || !got && !reflect.DeepEqual(state, before) { + t.Fatal("native m128 envelope/unknown byte must not be shrunk to numerical subwords", got, state) + } + }) + } + } +} + +func TestX86AESFamilyWholeSourceResultsCannotLaunderUndefinedInputs(t *testing.T) { + for _, name := range x86AESOracleNames { + row, destination := x86AESOracleRow(name, "X0", "X") + for _, target := range x86RawReturnTargets { + for _, api := range []string{"text", "module", "Go"} { + t.Run(name+"/"+target.triple+"/"+api, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·F(SB),4,$0-4\n%s\nMOVL X%d,AX\nMOVL AX,ret+0(FP)\nRET\n", row, destination) + _, err := x86ScalarFPBitViewAPI(t, source, "func F() uint32\n", target.arch, target.triple, api) + if err == nil { + t.Fatal("undefined AES byte escaped through GP result-store fallback") + } + }) + } + } + } +} diff --git a/amd64_aes_family_encoder_test.go b/amd64_aes_family_encoder_test.go new file mode 100644 index 000000000..8e872354d --- /dev/null +++ b/amd64_aes_family_encoder_test.go @@ -0,0 +1,487 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86AESFamilyNamedAndRawThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range x86RawReturnTargets { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw=%t", target.triple, raw), func(t *testing.T) { + var source, declarations strings.Builder + declarations.WriteString("package aesobjects\n") + count := 0 + for _, name := range x86AESOracleNames { + widths := []int{16} + if strings.HasPrefix(name, "V") && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, operand := range []string{bank + "0", fmt.Sprintf("%d(AX)(CX*2)", width)} { + row, _ := x86AESOracleRow(name, operand, bank) + code := x86OrderingHintGoBytes(t, target.arch, "TEXT actual(SB),4,$0-0\n"+row+"\nRET\n") + if raw { + row = x86DwordProductDirectives(code[:len(code)-1]) + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", count, row) + fmt.Fprintf(&declarations, "func F%d()\n", count) + count++ + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "aes-object-"+api+".ll", "aes-object-"+api+".o", ir) + }) + } + }) + } + } +} + +func TestX86AESFamilyCompressedDisp8EntireTupleAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames[6:10] { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, disp := range []int{-128, -1, 1, 127} { + for _, address := range []string{fmt.Sprintf("%d(AX)", disp*width), fmt.Sprintf("%d(AX)(CX*2)", disp*width), fmt.Sprintf("%d(BP)", disp*width)} { + // X16 forces Go's EVEX X/Y rows. Their original bytes are + // never patched into a different register contract. Go accepts + // that spelling in mode32, but physical high banks remain Context. + row := name + " " + address + "," + bank + "1," + bank + "16" + if width == 64 { + row = name + " " + address + ",Z1,Z2" + } + t.Run(arch+"/"+row, func(t *testing.T) { + code := x86OrderingHintGoBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + mode := 64 + if arch == "386" { + if width != 64 { + if _, _, known, err := decodedX86AESInstruction(code, 32); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("actual mode32 high destination acquired a low-bank identity", row, fmt.Sprintf("%x", code), err) + } + // Inspect exactly the same original bytes in physical + // mode64 for their tuple address, not mode32 permission. + } else { + mode = 32 + } + } + ins, length, known, err := decodedX86AESInstruction(code, mode) + if !known || err != nil || length != len(code)-1 || ins.Args[0].Mem.Off != int64(disp*width) { + t.Fatal("actual Go compressed full tuple did not retain exact native address", row, fmt.Sprintf("%x", code), ins, err) + } + }) + } + } + } + } + } +} + +func TestX86AESFamilyOriginalGoPhysicalRegisterExtensionAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames[6:10] { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for role := 0; role < 3; role++ { + for _, index := range []int{0, 7, 8, 15, 16, 31} { + args := []string{bank + "0", bank + "1", bank + "2"} + args[role] = fmt.Sprintf("%s%d", bank, index) + row := name + " " + strings.Join(args, ",") + t.Run(arch+"/"+row, func(t *testing.T) { + if arch == "386" && width == 64 && index >= 8 { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + if _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), arch); !known || err == nil { + t.Fatal("actual Go rejected mode32 Z bank acquired a valid grammar row", row, err) + } + return + } + code := x86OrderingHintGoBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + mode := 64 + if arch == "386" { + mode = 32 + if index >= 8 { + if _, _, known, err := decodedX86AESInstruction(code, mode); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("actual Go high spelling acquired a different physical mode32 register", row, fmt.Sprintf("%x", code), err) + } + return + } + } + ins, length, known, err := decodedX86AESInstruction(code, mode) + if !known || err != nil || length != len(code)-1 || ins.Args[role].Reg != Reg(args[role]) { + t.Fatal("all original VEX/EVEX source/state/destination extension bits", row, fmt.Sprintf("%x", code), ins, err) + } + }) + } + } + } + } + } +} + +func TestX86AESFamilyMode32NamedHighAndRawReservedBoundaries(t *testing.T) { + for _, name := range x86AESOracleNames[6:10] { + for _, bank := range []string{"X", "Y"} { + row := name + " " + bank + "16," + bank + "1," + bank + "2" + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + if _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go named high mode32 vector borrowed physical identity", row, err) + } + } + } + valid := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\nVAESENC 64(AX),Z1,Z2\nRET\n") + for _, mutation := range []func([]byte){func(b []byte) { b[2] &^= 4 }, func(b []byte) { b[3] |= 1 }, func(b []byte) { b[3] |= 0x80 }, func(b []byte) { b[3] |= 0x10 }, func(b []byte) { b[3] |= 0x60 }} { + changed := append([]byte(nil), valid...) + mutation(changed) + if _, _, known, err := decodedX86AESInstruction(changed, 64); !known || err == nil { + t.Fatal("reserved AES bytes acquired an unrelated raw effect", fmt.Sprintf("%x", changed), err) + } + } +} + +func TestX86AESFamilyPhysicalWIGIndependentLLVMDecoder(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 required") + } + for _, name := range x86AESOracleNames[6:] { + forms := []struct { + width int + destination int + }{{16, 2}} + if !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + forms = append(forms, struct { + width int + destination int + }{32, 2}, struct { + width int + destination int + }{16, 16}, struct { + width int + destination int + }{32, 16}, struct { + width int + destination int + }{64, 2}) + } + for _, form := range forms { + width := form.width + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row, _ := x86AESOracleRow(name, fmt.Sprintf("%d(AX)", width), bank) + row = strings.TrimSuffix(row, bank+"2") + fmt.Sprintf("%s%d", bank, form.destination) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + // Intel's VEX and EVEX AES attributes are WIG. This is an + // explicit physical-attribute fixture, not a claimed Go named row. + for _, wide := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/W=%t", row, wide), func(t *testing.T) { + physical := append([]byte(nil), code[:len(code)-1]...) + if wide { + physical[2] |= 0x80 + } + var input strings.Builder + for _, b := range physical { + fmt.Fprintf(&input, "0x%02x ", b) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=x86_64-unknown-linux-gnu") + command.Stdin = strings.NewReader(input.String() + "\n") + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil || !strings.Contains(strings.ToLower(string(output)), strings.ToLower(name)) || strings.Contains(string(output), "warning:") { + t.Fatal("independent LLVM22 physical WIG decoder", err, string(output)) + } + ins, length, known, err := decodedX86AESInstruction(physical, 64) + if !known || err != nil || length != len(physical) || ins.Op != Op(name) { + t.Fatal("complete physical AES WIG attribute lost", fmt.Sprintf("%x", physical), ins, err) + } + }) + } + } + } +} + +func TestX86AESFamilyLogicalFPNumericalSpanContracts(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, name := range x86AESOracleNames { + row, _ := x86AESOracleRow(name, "a+0(FP)", "X") + for _, contract := range []struct { + parameters string + accepted bool + }{{"a,b uint64", true}, {"a,b,c,d float32", true}, {"a,b float64", true}, {"a uint8,b uint64", false}, {"a,b *byte", false}} { + t.Run(target.triple+"/"+name+"/"+contract.parameters, func(t *testing.T) { + source := "TEXT ·F(SB),4,$0-16\n" + row + "\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, "func F("+contract.parameters+")\n", target.arch, target.triple, api) + accepted := contract.accepted && strings.HasPrefix(name, "V") + if (err == nil) != accepted || !accepted && ir != "" { + t.Fatal("numeric FP bytes cannot supply native legacy AES alignment, holes or pointer backing", accepted, err) + } + }) + } + }) + } + } + } +} + +func TestX86AESFamilyAbsoluteDisp32AndImmediateClasses(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames { + for _, address := range []int64{-2147483649, -2147483648, 2147483647, 4294967295, 4294967296} { + accepted := address == int64(int32(address)) || arch == "386" && address == int64(uint32(address)) + row, _ := x86AESOracleRow(name, fmt.Sprint(address), "X") + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", accepted) + _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), arch) + if !known || (err == nil) != accepted { + t.Fatal("physical Go disp32 extent differs from shared AES grammar", accepted, err) + } + }) + } + } + for _, name := range []string{"AESKEYGENASSIST", "VAESKEYGENASSIST"} { + var all strings.Builder + for immediate := -129; immediate <= 256; immediate++ { + accepted := immediate >= 0 && immediate <= 255 || name == "VAESKEYGENASSIST" && immediate >= -128 && immediate < 0 + row := fmt.Sprintf("%s $%d,X0,X2", name, immediate) + if accepted { + all.WriteString(row + "\n") + } else { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + } + if _, known, err := x86AESProperties(Op(name), x86YMMReadSource(t, row), arch); !known || (err == nil) != accepted { + t.Fatal("complete Yu8/Yi8 immediate classes", arch, row, accepted, err) + } + } + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+all.String()+"RET\n", "", true) + } + } +} + +func TestX86AESFamilyCompleteIndependentEncoderRows(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + data, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, table := range []struct{ name, body string }{{"yaes", "{Zlitm_r, 2, argList{Yxm, Yxr}},"}, {"yxshuf", "{Zibm_r, 2, argList{Yu8, Yxm, Yxr}},"}} { + start := strings.Index(string(data), "var "+table.name+" = []ytab{") + if start < 0 { + t.Fatal(table.name) + } + actual := strings.SplitN(strings.SplitN(string(data)[start:], "\n}", 2)[0], "[]ytab{", 2)[1] + a, err := x86SignedPackedFloatGoOpcodeTokens(actual) + if err != nil { + t.Fatal(err) + } + w, err := x86SignedPackedFloatGoOpcodeTokens(table.body) + if err != nil || !reflect.DeepEqual(a, w) { + t.Fatal("complete legacy zcase/zoffset/classes changed", table.name, actual) + } + } + legacyRows := regexp.MustCompile(`\{A(AES[A-Z]+), ([^,]+), ([^,]+), opBytes\{([^}]+)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(legacyRows) != 6 || len(x86AESSpecs) != 12 { + t.Fatal("whole family inventory changed", len(legacyRows), len(x86AESSpecs)) + } + for i, name := range x86AESOracleNames[:6] { + table, escape, opcode := "yaes", "0x38", []string{"0xdc", "0xdd", "0xde", "0xdf", "0xdb", "0xdf"}[i] + if i == 5 { + table, escape = "yxshuf", "0x3a" + } + want := fmt.Sprintf("{A%s, %s, Pq, opBytes{%s, %s, 0}}", name, table, escape, opcode) + if !strings.Contains(string(data), want) { + t.Fatal("legacy complete prefix/opcode attributes changed", want) + } + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, rule := range rules { + if !strings.HasPrefix(rule.Opcode, "VAES") { + continue + } + index := counts[rule.Opcode] + counts[rule.Opcode]++ + isIMC, isKey := rule.Opcode == "VAESIMC", rule.Opcode == "VAESKEYGENASSIST" + opcode := map[string]string{"VAESENC": "0xDC", "VAESENCLAST": "0xDD", "VAESDEC": "0xDE", "VAESDECLAST": "0xDF", "VAESIMC": "0xDB", "VAESKEYGENASSIST": "0xDF"}[rule.Opcode] + if opcode == "" { + t.Fatal("unaccounted actual Go AES member", rule.Opcode) + } + table, classes, zcase := "_yvaesdec", []string{"Yxm,Yxr,Yxr", "Yym,Yyr,Yyr", "YxmEvex,YxrEvex,YxrEvex", "YymEvex,YyrEvex,YyrEvex", "Yzm,Yzr,Yzr"}, "Zvex_rm_v_r" + zoffset, offset := []int{2, 2, 3, 3, 3}, []int{0, 2, 4, 7, 10} + var opbytes []string + if isKey { + table, classes, zcase = "_yvaeskeygenassist", []string{"Yu8,Yxm,Yxr", "Yi8,Yxm,Yxr"}, "Zvex_i_rm_r" + zoffset, offset = []int{0, 2}, []int{0, 0} + opbytes = []string{"avxEscape | vex128 | vex66 | vex0F3A | vexW0", opcode} + } else if isIMC { + table, classes = "_yvaesimc", []string{"Yxm,Yxr"} + zoffset, offset = []int{2}, []int{0} + opbytes = []string{"avxEscape | vex128 | vex66 | vex0F38 | vexW0", opcode} + } else { + for _, width := range []int{128, 256} { + opbytes = append(opbytes, fmt.Sprintf("avxEscape | vex%d | vex66 | vex0F38 | vexW0", width), opcode) + } + for _, width := range []int{128, 256, 512} { + opbytes = append(opbytes, fmt.Sprintf("avxEscape | evex%d | evex66 | evex0F38 | evexW0", width), fmt.Sprintf("evexN%d", width/8), opcode) + } + if index >= 2 { + zcase = "Zevex_rm_v_r" + } + } + if index >= len(classes) || rule.RowIndex != index || rule.Ytab != table || rule.Prefix != "Pavx" || rule.ZCase != zcase || rule.ZOffset != zoffset[index] || rule.EncodingOffset != offset[index] || strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatal("independent entire encoder row/attributes changed", rule) + } + } + for i, name := range x86AESOracleNames[6:] { + want := 5 + if i == 4 { + want = 1 + } + if i == 5 { + want = 2 + } + if counts[name] != want { + t.Fatal(name, counts, want) + } + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || regexp.MustCompile(`instructions\["V?AES`).Match(aliases) { + t.Fatal("actual Go AES alias inventory changed", err) + } +} + +func TestX86AESFamilyActualGoRawEncoderForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86AESOracleNames { + widths := []int{16} + if strings.HasPrefix(name, "V") && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, source := range []string{bank + "0", fmt.Sprintf("%d(AX)", width), fmt.Sprintf("-%d(AX)(CX*2)", width), "(FS)", "(GS)"} { + row, _ := x86AESOracleRow(name, source, bank) + t.Run(arch+"/"+row, func(t *testing.T) { + code := x86OrderingHintGoBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+x86DwordProductDirectives(code[:len(code)-1])+"\nRET\n") + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal("actual Go AES physical row missing", row, fmt.Sprintf("%x", code), err) + } + ins := decoded.Instrs[1] + form, known, err := x86AESProperties(ins.Op, ins, arch) + if !known || err != nil || form.byteWidth != width { + t.Fatal("actual Go raw AES identity", ins, err) + } + named := x86YMMReadSource(t, row) + for i := range named.Args { + if named.Args[i].Kind == OpMem && (named.Args[i].Mem.Base == FS || named.Args[i].Mem.Base == GS) { + named.Args[i].Mem.Segment, named.Args[i].Mem.Base = named.Args[i].Mem.Base, "" + } + if named.Args[i].Kind == OpMem && named.Args[i].Mem.Index == "" { + named.Args[i].Mem.Scale = ins.Args[i].Mem.Scale // no-index scale is not an encoded input + } + if named.Args[i].Kind == OpMem && strings.Contains(name, "KEYGEN") { + named.Args[i].Mem.Segment = "" // actual From3 prefix omission, independently bound above + } + } + if ins.Op != named.Op || !reflect.DeepEqual(ins.Args, named.Args) { + t.Fatalf("physical original operand/tuple width changed %s code=%x actual=%#v named=%#v", row, code, ins.Args, named.Args) + } + }) + } + } + } + } + // Independent fixed byte witness: prefix, map, opcode and immediate. The + // source encoder is checked independently of our decoder/typed grammar. + for name, want := range map[string][]byte{"AESENC X0,X1": {0x66, 0x0f, 0x38, 0xdc, 0xc8, 0xc3}, "AESKEYGENASSIST $27,X0,X2": {0x66, 0x0f, 0x3a, 0xdf, 0xd0, 27, 0xc3}, "VAESENC X0,X1,X2": {0xc4, 0xe2, 0x71, 0xdc, 0xd0, 0xc3}, "VAESKEYGENASSIST $-128,X0,X2": {0xc4, 0xe3, 0x79, 0xdf, 0xd0, 128, 0xc3}} { + got := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+name+"\nRET\n") + if !bytes.Equal(got, want) { + t.Fatal("independent fixed AES prefix/opcode bytes", name, got, want) + } + } +} + +func TestX86AESFamilyThreeAPIsFiveTargetNumericResults(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, name := range x86AESOracleNames { + for _, memory := range []bool{false, true} { + first, params, offset := "X0", "", 0 + if memory { + first, params, offset = "a+0(FP)", "a,b uint64", 16 + } + row, destination := x86AESOracleRow(name, first, "X") + if memory && !strings.HasPrefix(name, "V") { + row, destination = x86AESOracleRow(name, "X3", "X") + row = "MOVOU a+0(FP),X3\n" + row + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-%d\nMOVL $1,AX\nMOVL AX,X0\nMOVL AX,X1\n%s\nMOVQ X%d,ret+%d(FP)\nRET\n", count, offset+8, row, destination, offset) + fmt.Fprintf(&declarations, "func F%d(%s) uint64\n", count, params) + count++ + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "aes-results-"+api+".ll", "aes-results-"+api+".o", ir) + }) + } + }) + } +} + +func TestX86AESFamilyPermissionAndEncodingBoundaries(t *testing.T) { + for _, name := range x86AESOracleNames { + for _, source := range []string{"(SP)", "(BP)", "tmp(SP)"} { + row, _ := x86AESOracleRow(name, source, "X") + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-16\n"+row+"\nRET\n", "", true) + ir, err := x86ScalarFPBitViewAPI(t, "TEXT ·F(SB),4,$0-16\n"+row+"\nRET\n", "func F(a,b uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("AES grammar granted unbound native stack storage", row, err) + } + } + } +} diff --git a/amd64_aes_family_fault_runtime_unix_test.go b/amd64_aes_family_fault_runtime_unix_test.go new file mode 100644 index 000000000..ffb2b7771 --- /dev/null +++ b/amd64_aes_family_fault_runtime_unix_test.go @@ -0,0 +1,138 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86AESFamilyLiveDeadFaultEnvelopes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, rows strings.Builder + declarations.WriteString("package aesfault\n") + count, outcomes := 0, 0 + for _, name := range x86AESOracleNames { + legacy := !strings.HasPrefix(name, "V") + widths := []int{16} + if !legacy && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + for _, dead := range []bool{false, true} { + fn := fmt.Sprintf("F%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row, destination := x86AESOracleRow(name, "(DI)", bank) + store := "" + if !dead { + store = fmt.Sprintf("MOVQ out+8(FP),SI\nVMOVUPS %s%d,(SI)\n", bank, destination) + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DI\nVPXORD Z1,Z1,Z1\n%s\n%sRET\n", fn, row, store) + fmt.Fprintf(&declarations, "func %s(p *[64]byte,out *[64]byte)\n", fn) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint8_t*);\n", fn) + original := "NULL" + legacyInt := 0 + if legacy { + legacyInt = 1 + original = "Original" + fn + nativeStore := "" + if !dead { + nativeStore = fmt.Sprintf("MOVOU X%d,(SI)\n", destination) + } + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\nPXOR X1,X1\n"+row+"\n"+nativeStore+"RET\n") + var encoded []string + for _, value := range actual { + encoded = append(encoded, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(encoded, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint8_t*);\n", original) + outcomes += 9 + } + outcomes += 9 + fmt.Fprintf(&rows, "{%s,%s,%d,%d},\n", fn, original, width, legacyInt) + } + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + ir := x86NarrowOracleIR(t, source.String(), declarations.String(), execution.triple) + // Rosetta is a differential diagnostic, not the native alignment oracle. + // The Linux AMD64 owner must additionally enforce the architectural fault. + nativeAlignment := runtime.GOOS == "linux" && runtime.GOARCH == "amd64" + reference := fmt.Sprintf("#define NATIVE_ALIGNMENT %d\n", x86FloatAddSubBoolInt(nativeAlignment)) + + fmt.Sprintf(x86AESFaultC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{{"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}} { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "aes-fault", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("aes live/dead kernels %d exact fault outcomes %d tripwires 4 failures 0\n", count, outcomes) + if string(output) != want { + t.Fatal("AES original/native-width/dead-read faults", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86AESFaultC = ` +#include +#include +#include +#include +#include +#include +#include +#include +%s +%s +typedef void(*Program)(const uint8_t*,uint8_t*); +typedef struct {Program translated,original;unsigned width,legacy;} Kernel; +static Kernel kernels[]={%s}; +static int observe(Program p,unsigned width,unsigned address){ + pid_t child=fork();if(child<0)return -1; + if(child==0){ + struct rlimit no_core={0,0};if(setrlimit(RLIMIT_CORE,&no_core))_exit(120); + _Alignas(64) uint8_t data[128],out[64];memset(data,0xa7,sizeof(data));const uint8_t *input=data; + if(address==1)input++; + if(address==2)input=NULL; + if(address>=3){long page=sysconf(_SC_PAGESIZE);if(page<=0)_exit(121); + uint8_t *memory=mmap(NULL,2*(size_t)page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0);if(memory==MAP_FAILED)_exit(122); + memset(memory,0xa7,2*(size_t)page); + if(address<7){if(mprotect(memory+page,(size_t)page,PROT_NONE))_exit(123); + unsigned available[]={0,width-1,width,width+1};input=memory+page-available[address-3]; + }else{if(mprotect(memory,(size_t)page,PROT_NONE))_exit(123);input=memory+page-(address==7?4:12);} + } + p(input,out);_exit(0); + } + int status=0;if(waitpid(child,&status,0)!=child)return -2; + if(WIFEXITED(status))return WEXITSTATUS(status)?-3:0; + if(WIFSIGNALED(status)&&(WTERMSIG(status)==SIGSEGV||WTERMSIG(status)==SIGBUS))return 1; + return -4; +} +static void missing(const uint8_t*p,uint8_t*out){(void)p;(void)out;} +static void short_read(const uint8_t*p,uint8_t*out){out[0]=*(volatile const uint8_t*)p;} +static void over_read(const uint8_t*p,uint8_t*out){out[0]=*((volatile const uint8_t*)p+63);} +static void missing_prefix(const uint8_t*p,uint8_t*out){out[0]=*((volatile const uint8_t*)p+4)^*((volatile const uint8_t*)p+12);} +int main(void){unsigned count=0; + if(observe(missing,16,2)!=0||observe(short_read,64,4)!=0||observe(over_read,16,5)!=1||observe(missing_prefix,16,7)!=0)return 4; + for(unsigned i=0;i=7; + // Intel SDM Table 2-21: legacy SSE memory operands retain #GP for a + // non-16-byte-aligned address; VEX/EVEX do not inherit that exception. + int unaligned=k.legacy&&(a==1||a==6); + if(unaligned&&NATIVE_ALIGNMENT)want=1; + for(int original=1;original>=0;original--){if(original&&!k.original)continue; + int got=observe(original?k.original:k.translated,k.width,a);count++; + // Diagnostic hosts still require exact original/translated agreement. + if(original&&unaligned&&!NATIVE_ALIGNMENT&&(got==0||got==1))want=got; + if(got!=want){fprintf(stderr,"AES kernel%%u width%%u address%%u original%%d got%%d want%%d\n",i,k.width,a,original,got,want);return 1;} + } + } + printf("aes live/dead kernels %%zu exact fault outcomes %%u tripwires 4 failures 0\n",sizeof(kernels)/sizeof(kernels[0]),count);return 0; +} +` diff --git a/amd64_aes_family_fixed_local_test.go b/amd64_aes_family_fixed_local_test.go new file mode 100644 index 000000000..c6b68b55e --- /dev/null +++ b/amd64_aes_family_fixed_local_test.go @@ -0,0 +1,51 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86AESFamilyActualBoundFixedLocalConsumers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + var assembly, declarations strings.Builder + count := 0 + for _, name := range x86AESOracleNames { + for _, source := range []string{"16(SP)", "tmp-144(SP)"} { + row, _ := x86AESOracleRow(name, source, "X") + if strings.HasPrefix(name, "V") && !strings.Contains(name, "IMC") && !strings.Contains(name, "KEYGEN") { + row = name + " " + source + ",X0,X0" + } else if strings.Contains(name, "KEYGEN") { + row = name + " $27," + source + ",X0" + } else { + row = name + " " + source + ",X0" + } + body, _, _ := x86PrivateAlignedFrameAuditFixture(t, row) + fn := fmt.Sprintf("F%d", count) + count++ + assembly.WriteString(strings.Replace(body, "·Value", "·"+fn, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", fn) + } + } + for _, target := range []struct{ name, triple string }{{"linux/amd64", "x86_64-unknown-linux-gnu"}, {"darwin/amd64", "x86_64-apple-darwin"}} { + t.Run(target.name, func(t *testing.T) { + pkg, request, file, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + observer := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observer.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("AES consumer of independently sealed fixed local native bytes", err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "aes-native-local.ll", "aes-native-local.o", ir) + }) + }) + } +} diff --git a/amd64_aes_family_forms.go b/amd64_aes_family_forms.go new file mode 100644 index 000000000..b30a33a32 --- /dev/null +++ b/amd64_aes_family_forms.go @@ -0,0 +1,225 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86AESOperation uint8 + +const ( + x86AESEncrypt x86AESOperation = iota + x86AESEncryptLast + x86AESDecrypt + x86AESDecryptLast + x86AESInverseMix + x86AESKeyAssist +) + +type x86AESSpec struct { + operation x86AESOperation + vector bool +} + +// All AES optab attributes are explicit. In particular _yvaesdec has EVEX +// full tuples but no mask, broadcast, zeroing, rounding or SAE capability. +var x86AESSpecs = map[Op]x86AESSpec{ + "AESENC": {x86AESEncrypt, false}, "AESENCLAST": {x86AESEncryptLast, false}, + "AESDEC": {x86AESDecrypt, false}, "AESDECLAST": {x86AESDecryptLast, false}, + "AESIMC": {x86AESInverseMix, false}, "AESKEYGENASSIST": {x86AESKeyAssist, false}, + "VAESENC": {x86AESEncrypt, true}, "VAESENCLAST": {x86AESEncryptLast, true}, + "VAESDEC": {x86AESDecrypt, true}, "VAESDECLAST": {x86AESDecryptLast, true}, + "VAESIMC": {x86AESInverseMix, true}, "VAESKEYGENASSIST": {x86AESKeyAssist, true}, +} + +type x86AESForm struct { + spec x86AESSpec + byteWidth, source, state, destination, memory int + sourceOperand Operand // effective Go source role; original instruction is immutable +} + +// This shared grammar supplies operand roles only. No bank initialization, +// native memory, source authentication, frame or pointer authority is created. +func x86AESProperties(op Op, ins Instr, arch string) (x86AESForm, bool, error) { + name, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86AESSpecs[Op(name)] + if !known { + return x86AESForm{}, false, nil + } + f := x86AESForm{spec: spec, byteWidth: 16, source: 0, state: -1, destination: len(ins.Args) - 1, memory: -1} + fail := func() (x86AESForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete Go AES grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || suffix != "" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + count := 2 + if spec.operation == x86AESKeyAssist { + count, f.source = 3, 1 + if len(ins.Args) != count || ins.Args[0].ImmRaw != "" || ins.Args[0].ImmIsFloat || !amd64UnsignedImmediate(ins.Args[0], 8) && !(spec.vector && amd64VectorByteShiftImmediate(ins.Args[0])) { + return fail() + } + } else if spec.operation < x86AESInverseMix { + f.state = 1 + if spec.vector { + count = 3 + } + } + if len(ins.Args) != count { + return fail() + } + if spec.vector && spec.operation < x86AESInverseMix { + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + } + register := func(a Operand) bool { + if a.Kind != OpReg { + return false + } + if spec.vector && spec.operation < x86AESInverseMix { + return amd64GoPackedVectorMoveRegister(a, f.byteWidth, arch) + } + index, valid := amd64ParseXReg(a.Reg) + return valid && index < 16 && (spec.vector || arch != "386" || index < 8) + } + if !register(ins.Args[f.destination]) || f.state >= 0 && !register(ins.Args[f.state]) { + return fail() + } + source := ins.Args[f.source] + if isAMD64MemoryOperand(source) { + validation := source + if source.Kind == OpMem && arch == "386" && strings.HasSuffix(source.Mem.Sym, "(SB)") { + validation.Mem.Base = "" // actual Go ignores Reg, not Index, for named SB memory + } + if !x86PackedShiftMemoryValid(validation, arch) { + return fail() + } + if source.Kind == OpSym { + if address, err := parseInt(strings.TrimSpace(source.Sym)); err == nil && + address != int64(int32(address)) && !(arch == "386" && address == int64(uint32(address))) { + return fail() + } + } + if source.Kind == OpMem { + if !x86ConditionalMoveOffsetKnown(source.Mem) { + return fail() + } + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return f, true, err + } + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return f, true, fmt.Errorf("%w: AES source has no bound native address-size override", ErrProbeNeedsContext) + } + f.memory = f.source + } else if !register(source) { + return fail() + } + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return f, true, err + } + f.sourceOperand = source + if spec.operation == x86AESKeyAssist && source.Kind == OpMem && !ins.x86Encoded && ins.x86SourceRaw == nil { + // cmd/asm puts this memory in From3; obj/x86 emits segment prefixes + // only from From and To. Its real memory bytes therefore omit this + // annotation. Raw physical instructions retain their actual prefix. + f.sourceOperand.Mem.Segment = "" + if source.Mem.Base == FS || source.Mem.Base == GS { + f.sourceOperand.Mem.Base = "" + } + } + return f, true, nil +} + +func x86TypedAESByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + f, known, err := x86AESProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + e := x86TypedByteEffect{operands: []x86TypedByteOperand{{index: f.source, operand: f.sourceOperand, width: int64(f.byteWidth), read: true}}} + if f.state >= 0 && f.state != f.destination { + e.operands = append(e.operands, x86TypedByteOperand{index: f.state, operand: ins.Args[f.state], width: int64(f.byteWidth), read: true}) + } + e.operands = append(e.operands, x86TypedByteOperand{index: f.destination, operand: ins.Args[f.destination], width: int64(f.byteWidth), read: f.state == f.destination, write: true}) + return e, true, nil +} + +// Numerical dependencies are independent of the complete physical memory +// envelope. A SubWord/RotWord observes only source dwords 1 and 3; an AES round +// never has a zero idiom when the two sources alias. Wider lanes are 128-bit. +func x86AESDefinedBytes(f x86AESForm, source, state uint64) uint64 { + known := uint64(0) + for lane := 0; lane < f.byteWidth; lane += 16 { + for out := 0; out < 16; out++ { + var sourceMask, stateMask uint64 + switch f.spec.operation { + case x86AESKeyAssist: + at := out/8*8 + 4 + out%4 + if out%8 >= 4 { + at = out/8*8 + 4 + (out+1)%4 + } + sourceMask = uint64(1) << uint(lane+at) + case x86AESInverseMix: + sourceMask = uint64(15) << uint(lane+out/4*4) + default: + sourceMask = uint64(1) << uint(lane+out) + column, row := out/4, out%4 + first, last := 0, 4 + if f.spec.operation == x86AESEncryptLast || f.spec.operation == x86AESDecryptLast { + first, last = row, row+1 + } + for r := first; r < last; r++ { + shift := r + if f.spec.operation == x86AESDecrypt || f.spec.operation == x86AESDecryptLast { + shift = 4 - r + } + stateMask |= uint64(1) << uint(lane+((column+shift)%4)*4+r) + } + } + if source&sourceMask == sourceMask && state&stateMask == stateMask { + known |= uint64(1) << uint(lane+out) + } + } + } + return known +} + +func x86ResultAESStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + f, known, err := x86AESProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil { + return false, true + } + read := func(index int) (uint64, bool) { + a := ins.Args[index] + if index == f.source { + a = f.sourceOperand + } + if a.Kind == OpReg { + i, valid := amd64VectorRegisterIndex(a.Reg, f.byteWidth) + if !valid || i >= len(state.xmm) { + return 0, false + } + return state.vectorBytes(i), true + } + return x86CallerVectorByteMask(int64(f.byteWidth)), x86ResultRead(a, int64(f.byteWidth), state, owned, arch) + } + source, valid := read(f.source) + if !valid { + return false, true + } + input := uint64(0) + if f.state >= 0 { + input, valid = read(f.state) + if !valid { + return false, true + } + } + destination, _ := amd64VectorRegisterIndex(ins.Args[f.destination].Reg, f.byteWidth) + if destination >= len(state.xmm) { + return false, true + } + state.writeVector(destination, f.byteWidth, x86AESDefinedBytes(f, source, input), !f.spec.vector) + return true, true +} diff --git a/amd64_aes_family_runtime_test.go b/amd64_aes_family_runtime_test.go new file mode 100644 index 000000000..b332c274f --- /dev/null +++ b/amd64_aes_family_runtime_test.go @@ -0,0 +1,241 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// FIPS-197 byte arithmetic, independent of LLVM intrinsics, the source grammar +// and the production defined-byte model. Compute the S-box from GF(2^8), not +// from a table shared with any emitter. +const x86AESIndependentC = ` +#include +#include +#include +#include +static uint8_t mul(uint8_t a,uint8_t b){uint8_t v=0;for(int i=0;i<8;i++){if(b&1)v^=a; a=(uint8_t)((a<<1)^((a&128)?0x1b:0));b>>=1;}return v;} +static uint8_t rol(uint8_t a,int n){return (uint8_t)((a<>(8-n)));} +static uint8_t box(uint8_t a){uint8_t p=1,b=a;int n=254;if(!a)p=0;else while(n){if(n&1)p=mul(p,b);b=mul(b,b);n>>=1;}return p^rol(p,1)^rol(p,2)^rol(p,3)^rol(p,4)^0x63;} +static uint8_t inverse(uint8_t a){for(int i=0;i<256;i++)if(box((uint8_t)i)==a)return (uint8_t)i;return 0;} +static void mix(uint8_t *out,const uint8_t *in,int inv){for(int c=0;c<4;c++)for(int r=0;r<4;r++){ + const uint8_t forward[4]={2,3,1,1},backward[4]={14,11,13,9};out[c*4+r]=0; + for(int k=0;k<4;k++)out[c*4+r]^=mul(in[c*4+k],inv?backward[(k-r+4)%4]:forward[(k-r+4)%4]);}} +static void aes(uint8_t *out,const uint8_t *state,const uint8_t *key,int op,int imm){ + uint8_t shifted[16],temp[16]; + if(op==4){mix(out,key,1);return;} + if(op==5){for(int h=0;h<2;h++)for(int b=0;b<4;b++){out[h*8+b]=box(key[h*8+4+b]);out[h*8+4+b]=box(key[h*8+4+(b+1)%4])^(b==0?imm:0);}return;} + for(int c=0;c<4;c++)for(int r=0;r<4;r++){int source=((c+(op>=2?4-r:r))%4)*4+r;shifted[c*4+r]=op>=2?inverse(state[source]):box(state[source]);} + if(op==0||op==2)mix(temp,shifted,op==2);else memcpy(temp,shifted,16); + for(int i=0;i<16;i++)out[i]=temp[i]^key[i]; +} +typedef void(*Program)(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t); +struct Aligned {_Alignas(64) uint8_t before[64],value[80],after[64];}; +static int check(Program p,const char *name,int width,int op,int legacy,int memory,int alias,int imm){ + for(int sample=0;sample<37;sample++)for(int profile=0;profile<2;profile++){ + struct Aligned a,b,old,out;memset(&a,0xa7,sizeof(a));memset(&b,0xb9,sizeof(b));memset(&old,0x6d,sizeof(old));memset(&out,0xc3,sizeof(out)); + for(int i=0;i<64;i++){a.value[i]=(uint8_t)(sample*19+i*37);b.value[i]=(uint8_t)(sample*71+i*11);old.value[i]=(uint8_t)(sample*97+i*3);} + struct Aligned oa=a,ob=b,oo=old;uint64_t seed=profile?0x8d5:0;uint32_t csr=0x1f80|((sample%4)<<13)|(sample%64),saved=_mm_getcsr();_mm_setcsr(csr); + p(out.value,a.value,b.value,old.value,seed,csr);uint32_t after=_mm_getcsr();_mm_setcsr(saved); + uint64_t flags;uint32_t reported;memcpy(&flags,out.value+64,8);memcpy(&reported,out.value+72,4); + if((flags&0x8d5)!=seed||reported!=csr||after!=csr||memcmp(&a,&oa,sizeof(a))||memcmp(&b,&ob,sizeof(b))||memcmp(&old,&oo,sizeof(old)))return 1; + uint8_t expected[64];const uint8_t *retained=alias==0?a.value:alias==1?b.value:old.value; + memcpy(expected,legacy?retained:(const uint8_t[64]){0},64); + for(int lane=0;lane= 4 { + row = strings.TrimSuffix(row, "X2") + fmt.Sprintf("X%d", alias) + destination = alias + } + fn := fmt.Sprintf("NativeAES%d", count) + count++ + // SysV DI/SI/DX/CX/R8/R9 provide out/a/b/old/flags/csr. + body := fmt.Sprintf("TEXT actual(SB),4,$0-0\nMOVOU (SI),X0\nMOVOU (DX),X1\nMOVOU (CX),X2\nMOVL R9,76(DI)\nLDMXCSR 76(DI)\nPUSHQ R8\nPOPFQ\n%s\nMOVOU X%d,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,64(DI)\nSTMXCSR 72(DI)\nRET\n", row, destination) + code := x86OrderingHintGoBytes(t, "amd64", body) + var encoded []string + for _, b := range code { + encoded = append(encoded, fmt.Sprintf("0x%02x", b)) + } + symbol := fn + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + original.WriteString(x86OriginalSysVDeclaration("void", fn, "uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t")) + fmt.Fprintf(&original, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(encoded, ",")+"\n") + fmt.Fprintf(&checks, "if(nativecheck(%s,%q,%d))return 1;\n", fn, row, op) + } + } + } + reference := x86AESIndependentC + original.String() + x86AESNativeCheckC + "int main(void){\n" + checks.String() + fmt.Sprintf("printf(\"original Go AES X128 kernels %d samples %d full-byte failures 0\\n\");return 0;}\n", count, count*37*2) + output := execution.compileAndRunLLVM(t, "aes-native-go-bytes", "target triple = \""+execution.triple+"\"\n", reference, "-O2") + want := fmt.Sprintf("original Go AES X128 kernels %d samples %d full-byte failures 0\n", count, count*37*2) + if string(output) != want { + t.Fatal("actual Go full-X128 source-byte oracle", string(output)) + } + t.Log(strings.TrimSpace(string(output))) +} + +var x86AESNativeCheckC = x86OriginalSysVPointer("void", "OriginalProgram", + "uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t") + ` +static int nativecheck(OriginalProgram p,const char *name,int op){ + for(int sample=0;sample<37;sample++)for(int profile=0;profile<2;profile++){ + struct Aligned a,b,old,out;memset(&a,0xa7,sizeof(a));memset(&b,0xb9,sizeof(b));memset(&old,0x6d,sizeof(old));memset(&out,0xc3,sizeof(out)); + for(int i=0;i<64;i++){a.value[i]=(uint8_t)(sample*19+i*37);b.value[i]=(uint8_t)(sample*71+i*11);old.value[i]=(uint8_t)(sample*97+i*3);} + struct Aligned oa=a,ob=b,oo=old;uint64_t seed=profile?0x8d5:0;uint32_t csr=0x1f80|((sample%4)<<13)|(sample%64),saved=_mm_getcsr();_mm_setcsr(csr); + p(out.value,a.value,b.value,old.value,seed,csr);uint32_t after=_mm_getcsr();_mm_setcsr(saved); + uint64_t flags;uint32_t reported,scratch;memcpy(&flags,out.value+64,8);memcpy(&reported,out.value+72,4);memcpy(&scratch,out.value+76,4); + uint8_t expected[16];aes(expected,b.value,a.value,op,27); + if(memcmp(out.value,expected,16)||(flags&0x8d5)!=seed||reported!=csr||scratch!=csr||after!=csr||memcmp(&a,&oa,sizeof(a))||memcmp(&b,&ob,sizeof(b))||memcmp(&old,&oo,sizeof(old))){fprintf(stderr,"original %s sample%d full bytes/flags/CSR/input mismatch\n",name,sample);return 1;} + for(int i=16;i<64;i++)if(out.value[i]!=0xc3)return 2; + for(int i=0;i<64;i++)if(out.before[i]!=0xc3||out.after[i]!=0xc3)return 3; + }return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86AESFamilyLogicalFloatBitPayloads(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, reference, checks strings.Builder + declarations.WriteString("package aesbits\n") + reference.WriteString(x86AESIndependentC) + count := 0 + for op, name := range x86AESOracleNames { + for _, bits := range []int{32, 64} { + fn := fmt.Sprintf("FloatAES%d", count) + count++ + row, destination := x86AESOracleRow(name, "a+0(FP)", "X") + if !strings.HasPrefix(name, "V") { + // This is a logical bit-payload oracle, not native FP backing. + // Use Go's unaligned move before the legacy register operation; + // the separate memory oracle checks its alignment exceptions. + row, destination = x86AESOracleRow(name, "X3", "X") + row = "MOVOU a+0(FP),X3\n" + row + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nVPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n%s\nMOVQ out+16(FP),AX\nVMOVUPS Z%d,(AX)\nRET\n", fn, row, destination) + if bits == 32 { + checks.WriteString("for(unsigned j=0;j<4;j++){uint32_t v=payloads32[(sample+j)%8];memcpy(key+j*4,&v,4);}\n") + fmt.Fprintf(&declarations, "func %s(a,b,c,d float32,out *[64]byte)\n", fn) + fmt.Fprintf(&reference, "extern void %s(float,float,float,float,uint8_t*);\n", fn) + fmt.Fprintf(&checks, "{float f[4];memcpy(f,key,16);%s(f[0],f[1],f[2],f[3],out.value);}\n", fn) + } else { + checks.WriteString("for(unsigned j=0;j<2;j++){uint64_t v=payloads64[(sample+j)%8];memcpy(key+j*8,&v,8);}\n") + fmt.Fprintf(&declarations, "func %s(a,b float64,out *[64]byte)\n", fn) + fmt.Fprintf(&reference, "extern void %s(double,double,uint8_t*);\n", fn) + fmt.Fprintf(&checks, "{double f[2];memcpy(f,key,16);%s(f[0],f[1],out.value);}\n", fn) + } + fmt.Fprintf(&checks, "aes(expected,(const uint8_t[16]){0},key,%d,27);\n", op%6) + fmt.Fprintf(&checks, `if (memcmp(out.value,expected,64) || _mm_getcsr()!=csr) { + fprintf(stderr,"%s sample%%u csr%%x want%%x\n",sample,_mm_getcsr(),csr); + for(int j=0;j<64;j++) if(out.value[j]!=expected[j]) + fprintf(stderr,"byte%%d got%%x want%%x\n",j,out.value[j],expected[j]); + return 1; +} +`, fn) + checks.WriteString(`for(int j=0;j<64;j++) if(out.before[j]!=0xc3 || out.after[j]!=0xc3) return 2; +for(int j=64;j<80;j++) if(out.value[j]!=0xc3) return 3; +memset(&out,0xc3,sizeof(out)); +`) + } + } + reference.WriteString(` +int main(void){ + const uint32_t payloads32[]={0,0x80000000,0x7fc00013,0x7f800001,0xff800001,0x7f800000,0xff800000,1}; + const uint64_t payloads64[]={0,UINT64_C(0x8000000000000000),UINT64_C(0x7ff8000000000013),UINT64_C(0x7ff0000000000001),UINT64_C(0xfff0000000000001),UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000),1}; + for(unsigned sample=0;sample<8;sample++){ + uint8_t key[16],expected[64]={0};struct Aligned out;memset(&out,0xc3,sizeof(out)); + unsigned csr=0x1f80|((sample%4)<<13)|(sample%64),saved=_mm_getcsr();_mm_setcsr(csr); +` + checks.String() + `_mm_setcsr(saved); + }return 0; +} +`) + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + ir := x86NarrowOracleIR(t, source.String(), declarations.String(), execution.triple) + for _, axis := range []struct{ name, ir, backend string }{{"backend_O0", ir, "-O0"}, {"backend_O2", ir, "-O2"}, {"IR_O2/backend_O2", x86DwordProductIROptimize(t, ir), "-O2"}} { + t.Run(axis.name, func(t *testing.T) { + execution.compileAndRunLLVM(t, "aes-float-bits", axis.ir, reference.String(), axis.backend) + }) + } + t.Logf("%d complete F32/F64 logical bit-view kernels ×8 NaN/zero/infinity/denormal payloads on three axes; full64 bytes/MXCSR/canaries, no numeric float conversion or native-FP certificate", count) +} diff --git a/amd64_aes_family_sdk_test.go b/amd64_aes_family_sdk_test.go new file mode 100644 index 000000000..e71928f37 --- /dev/null +++ b/amd64_aes_family_sdk_test.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "crypto/sha256" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// This is a tiny actual-SDK object/source fixture, not a local stdlib/corpus +// replay. Original AES instructions and selected function bodies are retained. +// Dynamic aeskeysched contents alone are not extent, pointer or initialization +// authority. A later global producer must authenticate its own source/layout. +func TestX86AESFamilyOriginalSDKMemhash386GlobalBoundary(t *testing.T) { + relative := "src/internal/runtime/maps/memhash_386.s" + names := []string{"·memHash32AES", "·memHash64AES"} + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), relative)) + if errors.Is(err, os.ErrNotExist) { + relative = "src/runtime/asm_386.s" + names = []string{"runtime·memhash32", "runtime·memhash64"} + data, err = os.ReadFile(filepath.Join(runtime.GOROOT(), relative)) + } + if err != nil { + t.Fatal(err) + } + headerPath := filepath.Join(runtime.GOROOT(), "pkg/include/textflag.h") + header, err := os.ReadFile(headerPath) + if err != nil { + t.Fatal(err) + } + for _, name := range names { + t.Run(name, func(t *testing.T) { + start := strings.Index(string(data), "TEXT "+name+"(SB)") + if start < 0 { + t.Fatal("actual SDK selected function disappeared", relative, name) + } + body := string(data[start:]) + if next := strings.Index(body[1:], "\nTEXT "); next >= 0 { + body = body[:next+1] + } + original := "#include \"textflag.h\"\n" + body + dir := t.TempDir() + input, object := filepath.Join(dir, "actual-sdk.s"), filepath.Join(dir, "actual-sdk.o") + if err := os.WriteFile(input, []byte(original), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-I", filepath.Dir(headerPath), "-p", "actual-sdk", "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=386") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, "386"); failure != nil { + t.Fatal("unchanged SDK function original object", failure, string(output)) + } + physical, err := os.ReadFile(object) + if err != nil { + t.Fatal(err) + } + processed, err := PreprocessAssemblySource(original, AssemblyPreprocessOptions{FileName: relative, ReadInclude: func(parent, name string) (string, []byte, error) { + if name != "textflag.h" { + return "", nil, fmt.Errorf("unselected header %s", name) + } + return "pkg/include/textflag.h", header, nil + }}) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, processed) // shared x86 parser; the typed consumer is explicitly mode386 + if err != nil || len(file.Funcs) != 1 { + t.Fatal("actual source/include parser", err) + } + aes := 0 + for _, ins := range file.Funcs[0].Instrs { + if ins.Op != "AESENC" { + continue + } + aes++ + form, known, err := x86AESProperties(ins.Op, ins, "386") + if !known || err != nil || form.byteWidth != 16 || form.memory != 0 { + t.Fatal("unchanged actual SDK AES source must have registered whole-family grammar", ins, err) + } + state := x86ResultState{} + state.xmm[0] = 0xffff + if ok, known := x86ResultAESStep(ins, &state, nil, "386"); !known || ok { + t.Fatal("dynamic SB numerical values borrowed unproved initialized/extent authority", ins) + } + } + if aes != 3 { + t.Fatal("actual SDK AES inventory changed", aes) + } + t.Logf("SDK=%s full-sourceSHA256=%x selected-originalSHA256=%x headerSHA256=%x original-objectSHA256=%x bytes=%d exactAES=%d; global producer absent, not a corpus/version pass", relative, sha256.Sum256(data), sha256.Sum256([]byte(original)), sha256.Sum256(header), sha256.Sum256(physical), len(physical), aes) + }) + } +} diff --git a/amd64_aes_scalar_integration_runtime_test.go b/amd64_aes_scalar_integration_runtime_test.go new file mode 100644 index 000000000..746118516 --- /dev/null +++ b/amd64_aes_scalar_integration_runtime_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Execute the exact original FP source, not a GP-source substitute. The tiny +// test-only native bridge owns an ABI0 argument/result area for those actual +// Go object bytes and retains the four bytes immediately after the result. +// Neither this bridge nor its initialized bytes grant production native-FP, +// fixed-local, caller or pointer permission. +func TestCrossLinuxRuntimeMatrixX86AESFamilyScalarInsertResult(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + const source = "TEXT ·view(SB),4,$0-12\nMOVL k+0(FP),X0\nPINSRD $1,k+0(FP),X0\nAESENC X0,X0\nMOVL X0,ret+8(FP)\nRET\n" + code := x86OrderingHintGoBytes(t, "amd64", source) + var encoded []string + for _, value := range code { + encoded = append(encoded, fmt.Sprintf("0x%02x", value)) + } + original, bridge := "OriginalGoAESScalar", "OriginalGoAESScalarBridge" + if strings.Contains(execution.triple, "apple") { + original, bridge = "_"+original, "_"+bridge + } + assembly := ".text\n.globl " + original + "\n" + original + ":\n.byte " + strings.Join(encoded, ",") + + "\n.globl " + bridge + "\n" + bridge + ":\nsubq $24,%rsp\nmovl %edi,0(%rsp)\nmovl %esi,4(%rsp)\nmovl $0xc3c3c3c3,8(%rsp)\nmovl $0x6d6d6d6d,12(%rsp)\ncall " + original + "\nmovq 8(%rsp),%rax\naddq $24,%rsp\nret\n" + // This reference uses independent FIPS-197 arithmetic. It does not reuse + // the production grammar, LLVM AES intrinsic or byte-definedness model. + reference := x86AESIndependentC + fmt.Sprintf("\n__asm__(%q);\n", assembly) + + x86OriginalSysVDeclaration("uint64_t", "OriginalGoAESScalarBridge", "uint32_t,uint32_t") + ` +extern uint32_t view(uint32_t,uint32_t); +int main(void){ + for(unsigned sample=0;sample<64;sample++){ + const uint32_t special[4]={0,0xffffffff,0x80000000,0x01234567}; + uint32_t k=sample<4?special[sample]:sample*0x9137a5c3u+0x87654321u,h=k^0x6b5917d3u; + uint8_t input[16]={0},expected[16];uint32_t want; + memcpy(input,&k,4);memcpy(input+4,&k,4);aes(expected,input,input,0,0);memcpy(&want,expected,4); + uint64_t original=OriginalGoAESScalarBridge(k,h);uint32_t translated=view(k,h); + if((uint32_t)original!=want||(original>>32)!=0x6d6d6d6du||translated!=want){ + fprintf(stderr,"mixed AES scalar sample%u original%llx translated%x expected%x\n",sample,(unsigned long long)original,translated,want);return 1; + } + } + puts("mixed AES scalar exact Go bytes/FIPS/translated samples 64 result-canary failures 0");return 0; +} +` + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(k,h uint32) uint32\n", "amd64", execution.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, axis := range []struct{ name, level string }{{"backend_O0", "-O0"}, {"backend_O2", "-O2"}, {"IR_O2/backend_O2", "-O2"}} { + t.Run(api+"/"+axis.name, func(t *testing.T) { + optimized := ir + if strings.HasPrefix(axis.name, "IR_O2/") { + optimized = x86DwordProductIROptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "aes-scalar-result", optimized, reference, axis.level) + if string(output) != "mixed AES scalar exact Go bytes/FIPS/translated samples 64 result-canary failures 0\n" { + t.Fatal("unbound mixed-source runtime result", string(output)) + } + }) + } + } + t.Logf("exact original Go FP source bytes %x; 3 translation APIs x3 explicit optimization axes x64 independent result/canary samples (host transport, not native Linux)", code) +} diff --git a/amd64_andn_feature_identity_test.go b/amd64_andn_feature_identity_test.go new file mode 100644 index 000000000..062e7d32c --- /dev/null +++ b/amd64_andn_feature_identity_test.go @@ -0,0 +1,63 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// Go's scalar BMI rows are ANDNL and ANDNQ. The 32 packed logical names +// are a separate family even when their source mnemonic starts with ANDN. +func TestX86ANDNFeatureIdentityDoesNotBorrowScalarBMI(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86ScalarFPLogicalOpcodes { + t.Run(arch+"/"+op, func(t *testing.T) { + row, _ := x86ScalarFPLogicalRow(op) + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, file.Funcs[0]) + for _, feature := range strings.Split(features, ",") { + if feature == "+bmi" || feature == "+bmi2" { + t.Errorf("packed opcode %s borrowed scalar ISA: %s", op, features) + } + } + }) + } + } +} + +func TestX86ANDNNativeMemoryKeepsItsOwnISAAndScalarControls(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, tc := range []struct{ row, want string }{ + {"ANDNPS (AX),X1", "+sse"}, + {"ANDNPD (AX),X1", "+sse2"}, + {"ANDNL AX,BX,CX", "+bmi"}, + } { + t.Run(arch+"/"+tc.row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + tc.row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, file.Funcs[0]); got != tc.want { + t.Errorf("native source ISA=%q want=%q", got, tc.want) + } + }) + } + } + // The Q row is independently retained, not accidentally dropped while + // removing packed names from the former broad prefix heuristic. + source := "TEXT f(SB),4,$0-0\nANDNQ AX,BX,CX\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", file.Funcs[0]); got != "+bmi" { + t.Fatalf("scalar Q BMI row lost its ISA: %q", got) + } +} diff --git a/amd64_atomic_memory_test.go b/amd64_atomic_memory_test.go new file mode 100644 index 000000000..fe0b06306 --- /dev/null +++ b/amd64_atomic_memory_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +// An object-only success is insufficient: LLVM can legalize an under-aligned +// atomicrmw into an unresolved libatomic call, absent from the Windows CRT. +func TestX86AtomicExchangeMemoryNeedsNoRuntimeHelpers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"386", "i686-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-w64-windows-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"amd64", "x86_64-w64-windows-gnu"}, + {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-apple-darwin"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT exchange(SB),$0-0\n") + for _, family := range []string{"XADD", "XCHG"} { + widths := "BWL" + if target.arch == "amd64" { + widths += "Q" + } + for _, width := range widths { + reg := "DX" + if width == 'B' { + reg = "DL" + } + fmt.Fprintf(&source, "%s%c %s, 1(BX)\n", family, width, reg) + fmt.Fprintf(&source, "%s%c %s, 1(FS)\n", family, width, reg) + fmt.Fprintf(&source, "%s%c %s, 1(GS)\n", family, width, reg) + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"exchange": {Name: "exchange", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + input := filepath.Join(dir, "exchange.ll") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + asm, err := exec.Command(llc, "-mtriple="+target.triple, "-filetype=asm", input, "-o", "-").CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22: %v\n%s", err, asm) + } + if strings.Contains(string(asm), "__atomic") || strings.Contains(string(asm), "__sync") { + t.Fatalf("native x86 exchange acquired an external atomic helper:\n%s", asm) + } + for _, segment := range []string{"%fs:", "%gs:"} { + want := 6 + if target.arch == "amd64" { + want = 8 + } + if got := strings.Count(string(asm), segment); got != want { + t.Fatalf("atomic memory operand retained %d/%d %s overrides:\n%s", got, want, segment, asm) + } + } + compileLLVMToObject(t, llc, target.triple, "exchange.ll", "exchange.o", ir) + }) + } +} + +func TestX86ExchangeGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A((?:XADD|XCHG)[BWLQ]),\s*(\w+),`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 8 || len(rows) != len(amd64ExchangeSpecs) { + t.Fatalf("Go encoder has %d exchange opcodes; grammar has %d", len(rows), len(amd64ExchangeSpecs)) + } + for _, row := range rows { + spec, ok := amd64ExchangeSpecs[Op(row[1])] + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[row[1][len(row[1])-1]] + kind, table := amd64ExchangeSwap, "yxchg" + if strings.HasPrefix(row[1], "XADD") { + kind, table = amd64ExchangeAdd, "yrl_ml" + if bits == 8 { + table = "yrb_mb" + } + } else if bits == 8 { + table = "yml_mb" + } + if !ok || spec.bits != bits || spec.kind != kind || row[2] != table { + t.Errorf("Go %s/%s disagrees with typed grammar: %+v", row[1], row[2], spec) + } + } +} + +func TestAMD64ExchangeUnalignedRuntimeSemantics(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64 or Rosetta; the all-host object matrix remains required") + } + var source, declarations, cases strings.Builder + sigs := make(map[string]FuncSig) + for _, family := range []string{"XADD", "XCHG", "REVERSE"} { + for i, width := range "BWLQ" { + bits := []int{8, 16, 32, 64}[i] + name := fmt.Sprintf("exchange_%s_%d", family, bits) + reg := "AX" + if bits == 8 { + reg = "AL" + } + instruction := fmt.Sprintf("%s%c %s, 1(BX)", family, width, reg) + if family == "REVERSE" { + instruction = fmt.Sprintf("XCHG%c 1(BX), %s", width, reg) + } + fmt.Fprintf(&source, "TEXT %s(SB),$0-24\nMOVQ p+0(FP), BX\nMOVQ v+8(FP), AX\n%s\nMOVQ AX, ret+16(FP)\nRET\n", name, instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, I64}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0}}, + }} + fmt.Fprintf(&declarations, "extern uint64_t %s(uint8_t *, uint64_t);\n", name) + add := 0 + if family == "XADD" { + add = 1 + } + fmt.Fprintf(&cases, "{%s, %d, %d},\n", name, bits, add) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple, runPrefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +` + declarations.String() + ` +int main(void) { + struct { uint64_t (*fn)(uint8_t *, uint64_t); int bits, add; } cases[] = { +` + cases.String() + ` + }; + const uint64_t values[] = {0, 1, UINT64_MAX, UINT64_C(0x8877665544332211), UINT64_C(0x8000000080008080)}; + for (unsigned c = 0; c < sizeof(cases)/sizeof(cases[0]); c++) { + int bits = cases[c].bits, bytes = bits/8; + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + for (unsigned offset = 0; offset < 8; offset++) { + for (unsigned a = 0; a < 5; a++) for (unsigned b = 0; b < 5; b++) { + _Alignas(64) uint8_t storage[64]; + memset(storage, 0x5a, sizeof(storage)); + uint64_t initial = values[a], value = values[b]; + memcpy(storage + offset + 1, &initial, bytes); + uint64_t got = cases[c].fn(storage + offset, value), memory = 0; + memcpy(&memory, storage + offset + 1, bytes); + uint64_t want = initial & mask; + if (bits < 32) want |= value & ~mask; + if (got != want || memory != ((cases[c].add ? initial + value : value) & mask)) return 1; + for (unsigned i = 0; i < sizeof(storage); i++) { + if ((i < offset + 1 || i >= offset + 1 + bytes) && storage[i] != 0x5a) return 2; + } + } + } + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "exchange_unaligned", triple, ir, mainC, runPrefix) +} diff --git a/amd64_auxiliary_carry_runtime_test.go b/amd64_auxiliary_carry_runtime_test.go new file mode 100644 index 000000000..3bd59f2fb --- /dev/null +++ b/amd64_auxiliary_carry_runtime_test.go @@ -0,0 +1,310 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// The reference computes carry/borrow using widened arithmetic, including the +// low nibble for AF; it does not repeat the lowerer's XOR-bit implementation. +const x86AuxiliaryGoReference = `package auxiliary + +import ( + "encoding/binary" + "math/bits" + "testing" +) + +type kernel struct { + fn func(*[64]byte, uint64, uint64, uint64) + op string + width, form int +} + +func expected(op string, width, form int, a, b, seed uint64) uint64 { + if width == 0 || (op == "CMPS" || op == "SCAS") && form != 0 { + return seed & 0x8d5 + } + mask := ^uint64(0) + if width < 64 { + mask = (uint64(1) << width) - 1 + } + a &= mask + b &= mask + if form == 3 { + b = 1 + } + carry := uint64(0) + if op == "ADC" || op == "SBB" { + carry = seed & 1 + } + add := op == "ADD" || op == "ADC" || op == "INC" || op == "XADD" + switch op { + case "INC", "DEC": + b = 1 + case "NEG": + b, a = a, 0 + case "CMPXCHG": + a, b = b, a + } + + result := a - b - carry + cf := a < b || a == b && carry != 0 + af := (a & 15) < (b & 15) + carry + if add { + var c uint64 + result, c = bits.Add64(a, b, carry) + cf = c != 0 || width < 64 && result > mask + af = (a & 15) + (b & 15) + carry > 15 + } + result &= mask + sign := uint64(1) << (width - 1) + overflow := ((a ^ b) & (a ^ result) & sign) != 0 + if add { + overflow = ((^(a ^ b)) & (a ^ result) & sign) != 0 + } + if op == "INC" || op == "DEC" { + cf = seed & 1 != 0 + } + + f := uint64(0) + if cf { f |= 1 } + if bits.OnesCount8(uint8(result)) % 2 == 0 { f |= 4 } + if af { f |= 16 } + if result == 0 { f |= 64 } + if result & sign != 0 { f |= 128 } + if overflow { f |= 2048 } + return f +} + +func check(t *testing.T, k kernel, a, b, seed uint64) { + t.Helper() + var out [64]byte + for i := range out { + out[i] = 0xa7 + } + k.fn(&out, a, b, seed) + want := expected(k.op, k.width, k.form, a, b, seed) + got := binary.LittleEndian.Uint64(out[24:32]) & 0x8d5 + if got != want || out[32] != byte(want) | 2 { + t.Fatalf("%s width=%d form=%d a=%x b=%x seed=%x flags=%x LAHF=%x want=%x", k.op, k.width, k.form, a, b, seed, got, out[32], want) + } + flags := []uint64{want & 1, (want >> 2) & 1, (want >> 6) & 1, (want >> 7) & 1, (want >> 11) & 1, ((want >> 7) ^ (want >> 11)) & 1} + for i, v := range flags { + if out[33+i] != byte(v) { + t.Fatalf("%s SETcc %d got=%d want=%d", k.op, i, out[33+i], v) + } + } + for i, v := range out { + if (i >= 16 && i < 24 || i >= 39) && v != 0xa7 { + t.Fatalf("%s byte canary %d", k.op, i) + } + } +} + +func TestOriginalArithmeticStatus(t *testing.T) { + edges := []uint64{0, 1, 15, 16, 17, 127, 128, 255, 256, 32767, 32768, 65535, 0x7fffffff, 0x80000000, 0xffffffff, 0x7fffffffffffffff, 0x8000000000000000, ^uint64(0)} + for _, k := range kernels { + for _, seed := range []uint64{0, 0x8d5} { + for a := uint64(0); a < 16; a++ { + for b := uint64(0); b < 16; b++ { + check(t, k, a, b, seed) + } + } + for i, a := range edges { + for _, b := range []uint64{edges[(i+1)%len(edges)], edges[(i+7)%len(edges)]} { + check(t, k, a, b, seed) + } + } + } + } +} +` + +func TestCrossLinuxRuntimeMatrixX86AuxiliaryCarryOriginalGo(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + var cases []x86AuxiliaryCase + for _, row := range x86AuxiliaryCases("amd64") { + if row.op != "CMPXCHG" { + cases = append(cases, row) + } + } + x86AuxiliaryOriginalGo(t, cases) +} + +func TestCrossLinuxRuntimeMatrixX86AuxiliaryCarryCMPXCHGNativeGo(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skip("original CMPXCHG flags require the native AMD64 cross-runtime job, not Rosetta") + } + var cases []x86AuxiliaryCase + for _, row := range x86AuxiliaryCases("amd64") { + if row.op == "CMPXCHG" { + cases = append(cases, row) + } + } + x86AuxiliaryOriginalGo(t, cases) +} + +func x86AuxiliaryOriginalGo(t *testing.T, cases []x86AuxiliaryCase) { + t.Helper() + var declarations, kernels strings.Builder + kernels.WriteString("var kernels=[]kernel{\n") + for _, row := range cases { + fmt.Fprintf(&declarations, "func %s(*[64]byte,uint64,uint64,uint64)\n", row.name) + fmt.Fprintf(&kernels, "{%s,%q,%d,%d},\n", row.name, row.op, row.bits, row.form) + } + kernels.WriteString("}\n") + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module auxiliary\n\ngo 1.20\n", + "auxiliary.go": "package auxiliary\n" + declarations.String(), + "auxiliary_amd64.s": x86AuxiliaryAssembly("amd64", cases), + "auxiliary_test.go": x86AuxiliaryGoReference + kernels.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original Go complete arithmetic status: %v\n%s", err, output) + } + t.Logf("%d original Go kernels, exhaustive low nibbles, both CF/AF seeds and signed/unsigned width boundaries", len(cases)) +} + +const x86AuxiliaryCReference = ` +#include +#include +#include +typedef void (*kernel)(uint8_t *, uint64_t, uint64_t, uint64_t); + +static int check(kernel fn, const char *op, int width, int form, + uint64_t a, uint64_t b, uint64_t seed) { + _Alignas(8) uint8_t out[64]; + memset(out, 0xa7, sizeof(out)); + fn(out, a, b, seed); + + uint64_t want = seed & 0x8d5; + if (width && !((!strcmp(op, "CMPS") || !strcmp(op, "SCAS")) && form)) { + uint64_t mask = width == 64 ? UINT64_MAX : (UINT64_C(1) << width) - 1; + a &= mask; + b &= mask; + if (form == 3) { + b = 1; + } + unsigned carry = !strcmp(op, "ADC") || !strcmp(op, "SBB") ? seed & 1 : 0; + int add = !strcmp(op, "ADD") || !strcmp(op, "ADC") || + !strcmp(op, "INC") || !strcmp(op, "XADD"); + if (!strcmp(op, "INC") || !strcmp(op, "DEC")) { + b = 1; + } + if (!strcmp(op, "NEG")) { + b = a; + a = 0; + } + if (!strcmp(op, "CMPXCHG")) { + uint64_t swap = a; + a = b; + b = swap; + } + + __uint128_t wide = add ? (__uint128_t)a + b + carry : (__uint128_t)a - b - carry; + uint64_t result = (uint64_t)wide & mask; + unsigned cf = add ? wide > mask : (__uint128_t)a < (__uint128_t)b + carry; + unsigned af = add ? (a & 15) + (b & 15) + carry > 15 : (a & 15) < (b & 15) + carry; + uint64_t sign = UINT64_C(1) << (width - 1); + unsigned of = (add ? (~(a ^ b) & (a ^ result)) : (a ^ b) & (a ^ result)) & sign ? 1 : 0; + if (!strcmp(op, "INC") || !strcmp(op, "DEC")) { + cf = seed & 1; + } + unsigned ones = 0; + for (unsigned i = 0; i < 8; i++) { + ones += (result >> i) & 1; + } + want = cf | ((ones % 2 == 0) << 2) | (af << 4) | ((result == 0) << 6) | + ((!!(result & sign)) << 7) | (of << 11); + } + + uint64_t flags; + memcpy(&flags, out + 24, 8); + flags &= 0x8d5; + if (flags != want || out[32] != ((want & 255) | 2)) { + fprintf(stderr, "%s width=%d form=%d flags=%llx lahf=%x want=%llx\n", + op, width, form, (unsigned long long)flags, out[32], (unsigned long long)want); + return 1; + } + unsigned bits[] = {0, 2, 6, 7, 11}; + for (unsigned i = 0; i < 5; i++) { + if (out[33+i] != ((want >> bits[i]) & 1)) { + return 2; + } + } + if (out[38] != (((want >> 7) ^ (want >> 11)) & 1)) { + return 3; + } + for (unsigned i = 16; i < 24; i++) { + if (out[i] != 0xa7) { + return 4; + } + } + for (unsigned i = 39; i < 64; i++) { + if (out[i] != 0xa7) { + return 5; + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86AuxiliaryCarryLLVM(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + cases := x86AuxiliaryCases("amd64") + file, err := Parse(ArchAMD64, x86AuxiliaryAssembly("amd64", cases)) + if err != nil { + t.Fatal(err) + } + var declarations, checks strings.Builder + for _, row := range cases { + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,uint64_t,uint64_t,uint64_t);\n", row.name) + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,a,b,seed))return 1;\n", row.name, row.op, row.bits, row.form) + } + main := x86AuxiliaryCReference + declarations.String() + ` +int main(void) { + const uint64_t edges[] = { + 0, 1, 15, 16, 17, 127, 128, 255, 256, 32767, 32768, 65535, + 0x7fffffff, 0x80000000, 0xffffffff, 0x7fffffffffffffff, + 0x8000000000000000, UINT64_MAX + }; + for (unsigned s = 0; s < 2; s++) { + for (unsigned i = 0; i < 292; i++) { + uint64_t seed = s ? 0x8d5 : 0, a, b; + if (i < 256) { + a = i / 16; + b = i % 16; + } else { + a = edges[(i-256)/2]; + b = edges[((i-256)/2 + (i%2 ? 7 : 1))%18]; + } +` + checks.String() + " }\n }\n return 0;\n}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + ir, err := Translate(file, x86AuxiliaryOptions("amd64", triple, cases)) + if err != nil { + t.Fatal(err) + } + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "auxiliary-status", triple, ir, main, prefix) + }) + } +} diff --git a/amd64_auxiliary_carry_test.go b/amd64_auxiliary_carry_test.go new file mode 100644 index 000000000..1cf9b575f --- /dev/null +++ b/amd64_auxiliary_carry_test.go @@ -0,0 +1,160 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86AuxiliaryCase struct { + name string + op string + bits int + form int +} + +// Operand axes are tested independently of the existing complete encoder-row +// tests. All arithmetic flag producers, including exchange/string comparisons, +// must feed the same architectural AF bit consumed by LAHF and PUSHF. +func x86AuxiliaryCases(arch string) []x86AuxiliaryCase { + var cases []x86AuxiliaryCase + widths := []int{8, 16, 32} + if arch == "amd64" { + widths = append(widths, 64) + } + for _, op := range []string{"ADD", "SUB", "ADC", "SBB", "CMP", "INC", "DEC", "NEG", "XADD", "CMPXCHG", "CMPS", "SCAS"} { + forms := 4 // Register, memory source, memory destination, immediate. + switch op { + case "INC", "DEC", "NEG", "XADD", "CMPXCHG": + forms = 2 + case "CMPS", "SCAS": + forms = 3 // Single comparison; zero-count REP and REPN with NULL inputs. + } + for _, bits := range widths { + for form := 0; form < forms; form++ { + cases = append(cases, x86AuxiliaryCase{fmt.Sprintf("aux%d", len(cases)), op, bits, form}) + } + } + } + return append(cases, x86AuxiliaryCase{"auxpopf", "POPF", 0, 0}, x86AuxiliaryCase{"auxsahf", "SAHF", 0, 0}) +} + +func x86AuxiliaryAssembly(arch string, cases []x86AuxiliaryCase) string { + move, push, flags, slot := "Q", "Q", "Q", 8 + if arch == "386" { + move, push, flags, slot = "L", "L", "L", 4 + } + var source strings.Builder + for _, row := range cases { + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n", row.name, slot*4) + fmt.Fprintf(&source, "MOV%s out+0(FP),DI\nMOV%s a+%d(FP),BX\nMOV%s b+%d(FP),DX\nMOV%s seed+%d(FP),CX\n", move, move, slot, move, slot*2, move, slot*3) + fmt.Fprintf(&source, "MOV%s BX,0(DI)\nMOV%s DX,8(DI)\n", move, move) + if row.op == "SAHF" { + fmt.Fprintf(&source, "MOV%s CX,AX\nSHL%s $8,AX\n", move, move) + } + if row.op == "CMPXCHG" { + fmt.Fprintf(&source, "MOV%s DX,AX\n", move) + } + if row.op == "CMPS" || row.op == "SCAS" { + // BP retains the output address, not a fabricated caller frame. + fmt.Fprintf(&source, "PUSH%s BP\n", push) + fmt.Fprintf(&source, "MOV%s DI,BP\nMOV%s DI,SI\nADD%s $8,DI\nMOV%s BX,AX\n", move, move, move, move) + } + fmt.Fprintf(&source, "PUSH%s CX\nPOPF%s\n", push, flags) + suffix := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[row.bits] + opcode := row.op + suffix + switch row.op { + case "POPF": // The seed is already installed above. + case "SAHF": + source.WriteString("SAHF\n") + case "CMPS", "SCAS": + if row.form != 0 { + fmt.Fprintf(&source, "MOV%s $0,CX\nMOV%s $0,SI\nMOV%s $0,DI\n", move, move, move) + prefix := "REP" + if row.form == 2 { + prefix = "REPN" + } + fmt.Fprintf(&source, "%s\n", prefix) + } + fmt.Fprintf(&source, "%s\nMOV%s BP,DI\n", opcode, move) + case "INC", "DEC", "NEG": + destination := "BX" + if row.form == 1 { + destination = "0(DI)" + } + fmt.Fprintf(&source, "%s %s\n", opcode, destination) + case "XADD", "CMPXCHG": + destination := "BX" + if row.form == 1 { + destination = "0(DI)" + } + fmt.Fprintf(&source, "%s DX,%s\n", opcode, destination) + case "CMP": + operands := []string{"BX,DX", "BX,8(DI)", "0(DI),DX", "BX,$1"} + fmt.Fprintf(&source, "%s %s\n", opcode, operands[row.form]) + default: + operands := []string{"DX,BX", "8(DI),BX", "DX,0(DI)", "$1,BX"} + fmt.Fprintf(&source, "%s %s\n", opcode, operands[row.form]) + } + fmt.Fprintf(&source, "PUSHF%s\nPOP%s CX\nMOV%s CX,24(DI)\nLAHF\nMOVB AH,32(DI)\n", flags, push, move) + source.WriteString("SETCS 33(DI)\nSETPS 34(DI)\nSETEQ 35(DI)\nSETMI 36(DI)\nSETOS 37(DI)\nSETLT 38(DI)\n") + if row.op == "CMPS" || row.op == "SCAS" { + fmt.Fprintf(&source, "POP%s BP\n", push) + } + source.WriteString("RET\n") + } + return source.String() +} + +func x86AuxiliaryOptions(arch, triple string, cases []x86AuxiliaryCase) Options { + typ, slot := I64, int64(8) + if arch == "386" { + typ, slot = I32, 4 + } + sigs := make(map[string]FuncSig, len(cases)) + for _, row := range cases { + sigs[row.name] = FuncSig{Name: row.name, Args: []LLVMType{Ptr, typ, typ, typ}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: slot, Type: typ, Index: 1, Field: -1}, + {Offset: slot * 2, Type: typ, Index: 2, Field: -1}, + {Offset: slot * 3, Type: typ, Index: 3, Field: -1}, + }}, + } + } + return Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + } +} + +func TestX86AuxiliaryCarryWholeFamilyObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + cases := x86AuxiliaryCases(target.arch) + source := x86AuxiliaryAssembly(target.arch, cases) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86AuxiliaryOptions(target.arch, target.triple, cases)) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "auxiliary.ll", "auxiliary.o", ir) + if !strings.Contains(ir, "%flags_af = alloca i1") { + t.Fatal("arithmetic status consumers silently omit architectural AF") + } + }) + } +} diff --git a/amd64_bittest_effects_test.go b/amd64_bittest_effects_test.go new file mode 100644 index 000000000..42ddcd365 --- /dev/null +++ b/amd64_bittest_effects_test.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86BitTestTypedByteAndFlagEffects(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86BitTestOracleNames(arch) { + width := int64(map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]]) + writes := len(op) != 3 + for _, index := range []string{"$1", "CX"} { + ins := x86PureGPUnknownInstruction(t, op+" "+index+",AX") + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 || effect.dynamicMemory || + effect.flagsDefined != x86ByteFlagCF || effect.flagsRead != 0 || + effect.flagsKilled != x86ByteFlagsAll&^(x86ByteFlagCF|x86ByteFlagZF) || + !effect.operands[1].read || effect.operands[1].write != writes { + t.Fatalf("incomplete %s/%s byte/flag effects: %+v %v", arch, ins.Raw, effect, err) + } + for flags := uint8(0); flags <= x86ByteFlagsAll; flags++ { + state := x86ResultState{flags: flags} + state.gp[0], state.gp[1] = 0xff, 0xff + if !x86ResultTransfer(ins, &state, nil, 0, arch) || state.flags != flags&x86ByteFlagZF|x86ByteFlagCF { + t.Fatalf("%s/%s invented incoming flag state %x: %x", arch, ins.Raw, flags, state.flags) + } + } + for missingOperand := 0; missingOperand < 2; missingOperand++ { + for missingByte := int64(-1); missingByte < width; missingByte++ { + state := x86ResultState{flags: x86ByteFlagsAll} + state.gp[0], state.gp[1] = uint8(1<= 0 && (missingOperand == 1 || index != "$1") + if missing { + register := 1 - missingOperand + state.gp[register] &^= 1 << uint(missingByte) + } + before := state.clone() + accepted := x86ResultTransfer(ins, &state, nil, 0, arch) + if accepted == missing { + t.Fatalf("%s/%s missing operand=%d byte=%d admitted=%v", arch, ins.Raw, missingOperand, missingByte, accepted) + } + if accepted { + want := before.gp[0] + if writes && width == 4 { + want |= 0xf0 + } + if state.gp[0] != want || state.gp[1] != before.gp[1] || state.flags != x86ByteFlagCF|x86ByteFlagZF { + t.Fatalf("%s/%s changed unwritten bytes or preserved undefined flags: %+v", arch, ins.Raw, state) + } + } + } + } + memory := x86PureGPUnknownInstruction(t, op+" "+index+",(DX)") + effect, known, err = x86TypedInstructionByteEffect(memory.Op, memory, arch) + if !known || err != nil || effect.dynamicMemory != (index == "CX") { + t.Fatal("memory bit string was mistaken for a fixed register word", memory, effect, err) + } + } + } + } +} + +func TestX86BitTestFlagsAndMemoryDoNotAcquireAuthority(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, op := range x86BitTestOracleNames(target.arch) { + for _, condition := range []struct { + branch string + valid bool + }{{"JCS", true}, {"JE", true}, {"JOS", false}, {"JMI", false}, {"JPS", false}} { + source := "TEXT ·value(SB),4,$0-4\nMOVL $1,AX\nCMPL AX,AX\n" + op + " $0,AX\n" + + condition.branch + " equal\nMOVL $7,AX\nMOVL AX,X0\nJMP done\nequal:\nPXOR X0,X0\ndone:\nPEXTRD $0,X0,AX\nMOVL AX,ret+0(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "func value() uint32", target.arch, target.triple, api) + if condition.valid && err != nil || !condition.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%s/%s/%s flag initialization differs: %v", target.triple, op, condition.branch, api, err) + } + } + } + for _, memory := range []string{"unbound(SB)", "(DX)", "8(FS)"} { + source := "TEXT ·value(SB),4,$0-4\n" + op + " $0," + memory + "\nPXOR X0,X0\nPEXTRD $0,X0,AX\nMOVL AX,ret+0(FP)\nRET\n" + if _, err := x86ScalarFPBitViewAPI(t, source, "func value() uint32", target.arch, target.triple, "text"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("new native/source read authority: %s/%s/%s: %v", target.triple, op, memory, err) + } + } + } + } +} + +func TestX86BitTestFPCompleteByteSpans(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + var validSource, declarations strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames(target.arch) { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + for _, declaration := range []string{"func value(x uint64)", "func value(x [8]byte)", "func value(x float64)"} { + for offset := -1; offset <= 8; offset++ { + valid := offset >= 0 && offset+width <= 8 + source := fmt.Sprintf("TEXT ·value(SB),4,$0-8\n%s $-1,x%+d(FP)\nRET\n", op, offset) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, api) + if valid && err != nil || !valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%s/%s/%s/%d byte boundary: %v", target.triple, op, declaration, api, offset, err) + } + if valid && x86CMOVFPHasNumericConversion(ir) { + t.Fatal("raw FP bits were numerically converted") + } + } + } + } + for _, declaration := range []string{"func value(x *uint64)", "func value(x uint8,y uint64)", "func value() uint64"} { + source := fmt.Sprintf("TEXT ·value(SB),4,$0-16\n%s $0,x+0(FP)\nRET\n", op) + if _, err := x86ScalarFPBitViewAPI(t, source, declaration, target.arch, target.triple, "text"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("hole/pointer/uninitialized result acquired FP bytes: %s/%s/%s: %v", target.triple, op, declaration, err) + } + } + name := fmt.Sprintf("span%d", count) + fmt.Fprintf(&validSource, "TEXT ·%s(SB),4,$0-8\n%s $1,x+0(FP)\nRET\n", name, op) + fmt.Fprintf(&declarations, "func %s(x [8]byte)\n", name) + count++ + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, validSource.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, validSource.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "bit-test-fp.ll", "bit-test-fp.o", ir) + } + } +} diff --git a/amd64_bittest_encoder_test.go b/amd64_bittest_encoder_test.go new file mode 100644 index 000000000..07c7b23d1 --- /dev/null +++ b/amd64_bittest_encoder_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86BitTestCompleteGoEncoderRows(t *testing.T) { + contents, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + text := string(contents) + start := strings.Index(text, "var ybtl = []ytab{") + if start < 0 { + t.Fatal("actual Go omitted the bit-string encoder table") + } + actual := strings.SplitN(strings.SplitN(text[start:], "\n}", 2)[0], "[]ytab{", 2)[1] + got, err := x86SignedPackedFloatGoOpcodeTokens(actual) + want, wantErr := x86SignedPackedFloatGoOpcodeTokens(`{Zibo_m,2,argList{Yi8,Yml}}, {Zr_m,1,argList{Yrl,Yml}},`) + if err != nil || wantErr != nil || !reflect.DeepEqual(got, want) { + t.Fatal("complete ordered ybtl rows/classes/zcase/offset changed") + } + rows := regexp.MustCompile(`(?m)^\s*\{A(BT(?:C|R|S)?[WLQ]),\s*(\w+),\s*(\w+),\s*opBytes\{([^}]+)\}\},`).FindAllStringSubmatch(text, -1) + if len(rows) != 12 { + t.Fatalf("complete Go family changed: %d rows", len(rows)) + } + seen := make(map[string]bool) + for _, row := range rows { + op, suffix := row[1], row[1][len(row[1])-1] + stem := op[:len(op)-1] + group := map[string]int{"BT": 4, "BTC": 7, "BTR": 6, "BTS": 5}[stem] + registerOpcode := map[string]int{"BT": 0xa3, "BTC": 0xbb, "BTR": 0xb3, "BTS": 0xab}[stem] + prefix := map[byte]string{'W': "Pq", 'L': "Pm", 'Q': "Pw"}[suffix] + encoding := fmt.Sprintf("0xba,0%d,0x%x", group, registerOpcode) + if suffix == 'Q' { + encoding = fmt.Sprintf("0x0f,0xba,0%d,0x0f,0x%x", group, registerOpcode) + } + got, err := x86SignedPackedFloatGoOpcodeTokens(row[4]) + want, wantErr := x86SignedPackedFloatGoOpcodeTokens(encoding) + if seen[op] || row[2] != "ybtl" || row[3] != prefix || err != nil || wantErr != nil || !reflect.DeepEqual(got, want) { + t.Fatal("complete Go family registration/prefix/opcode changed", row) + } + seen[op] = true + } + for _, op := range x86BitTestOracleNames("amd64") { + if !seen[op] { + t.Fatal("independent witness absent from actual Go optab", op) + } + } +} + +func TestX86BitTestAllImmediateBytesActualGo(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + var expected []byte + source.WriteString("TEXT f(SB),4,$0-0\n") + count := 0 + for _, op := range x86BitTestOracleNames(arch) { + stem, suffix := op[:len(op)-1], op[len(op)-1] + group := byte(map[string]int{"BT": 4, "BTC": 7, "BTR": 6, "BTS": 5}[stem]) + for immediate := -128; immediate < 128; immediate++ { + for _, destination := range []string{"AX", "7(DX)"} { + fmt.Fprintf(&source, "%s $%d,%s\n", op, immediate, destination) + if suffix == 'W' { + expected = append(expected, 0x66) + } else if suffix == 'Q' { + expected = append(expected, 0x48) + } + expected = append(expected, 0x0f, 0xba) + if destination == "AX" { + expected = append(expected, 0xc0|group<<3) + } else { + // Go emits exactly the source displacement, not a new + // displacement inferred from high immediate bits. + expected = append(expected, 0x42|group<<3, 7) + } + expected = append(expected, byte(immediate)) + ins := x86PureGPUnknownInstruction(t, fmt.Sprintf("%s $%d,%s", op, immediate, destination)) + form, known, err := x86BitTestProperties(ins.Op, ins, arch) + if !known || err != nil || form.index.Imm != int64(immediate) { + t.Fatal("typed immediate differs from independent original Go byte", ins, err) + } + count++ + } + } + } + source.WriteString("RET\n") + expected = append(expected, 0xc3) + actual := x86PackedShiftActualGoBytes(t, arch, source.String()) + if !bytes.Equal(actual, expected) { + t.Fatalf("%s complete original immediate bytes differ: got %d bytes, want %d", arch, len(actual), len(expected)) + } + t.Logf("%s: %d actual Go bit-string encodings, every imm8, register and memory", arch, count) + } +} + +func TestX86BitTestCompleteFormsFiveTargetsThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames(target.arch) { + rows := []string{"$-128,AX", "$127,CX", "CX,AX", "AX,AX", "SP,BP", "CX,SP", + "$63,7(DX)", "CX,7(DX)", "CX,-3(DX)(SI*4)", "$1,data-3(SB)", "CX,data+3(SB)", + "$1,8(FS)", "CX,8(GS)", "$1,4096", "CX,4096"} + if target.arch == "amd64" { + rows = append(rows, "R8,R15", "R9,-7(R12)(R11*8)") + } else { + rows = append(rows, "$4294967295,AX", "CX,data(SB)(SP)", "$1,data(SB)(BX*4)") + } + registers := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + if target.arch == "amd64" { + registers = append(registers, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + } + for _, register := range registers { + rows = append(rows, register+",AX", "$1,"+register, "AX,"+register) + } + for _, row := range rows { + body := op + " " + row + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86BitTestProperties(ins.Op, ins, target.arch); !known || err != nil { + t.Fatalf("typed shape %s: %v", body, err) + } + fmt.Fprintf(&source, "TEXT ·form%d(SB),4,$0-0\n%s\nRET\n", count, body) + fmt.Fprintf(&declarations, "func form%d()\n", count) + count++ + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "bittest.ll", "bittest.o", ir) + }) + } + t.Logf("%d actual Go witnesses, three LLVM API object consumers", count) + }) + } +} + +func TestX86BitTestMalformedAndModeForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86BitTestOracleNames(arch) { + for _, row := range []string{"$128,AX", "$-129,AX", "$1.0,AX", "AL,AX", "X0,AX", "AX,X0", "AX,$1", "AX", "(AX),BX", "$1,(AX)(SP*2)"} { + body := op + " " + row + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+body+"\nRET\n", "", false) + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86BitTestProperties(ins.Op, ins, arch); !known || err == nil { + t.Errorf("malformed original form admitted: %s %s", arch, body) + } + } + for _, mutate := range []func(*Instr){ + func(i *Instr) { i.Op += ".Z" }, + func(i *Instr) { i.Args[0].ImmRaw = "unknown" }, + func(i *Instr) { i.Args[1].Mem.OffRaw = "unknown" }, + func(i *Instr) { i.Args[1].Mem.Index = CX; i.Args[1].Mem.Scale = 3 }, + } { + ins := x86PureGPUnknownInstruction(t, op+" $1,(DX)") + mutate(&ins) + if _, known, err := x86BitTestProperties(ins.Op, ins, arch); !known || err == nil { + t.Errorf("typed malformed shape admitted: %s %+v", arch, ins) + } + } + } + } + for _, body := range []string{"BTQ AX,BX", "BTCQ $1,(AX)", "BTRQ AX,BX", "BTSQ AX,BX", "BTL R8,AX", "BTSL AX,R15"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+body+"\nRET\n", "", false) + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86BitTestProperties(ins.Op, ins, "386"); !known || err == nil { + t.Error("386 accepted unavailable native encoding", body) + } + } +} diff --git a/amd64_bittest_fault_runtime_unix_test.go b/amd64_bittest_fault_runtime_unix_test.go new file mode 100644 index 000000000..313681c37 --- /dev/null +++ b/amd64_bittest_fault_runtime_unix_test.go @@ -0,0 +1,127 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This root is required by the Linux cross-runtime matrix. Local Rosetta +// observations are diagnostic, not a replacement for native AMD64 execution. +func TestCrossLinuxRuntimeMatrixX86BitTestDeadMemoryFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames("amd64") { + for _, index := range []string{"$0", "CX"} { + name, original := fmt.Sprintf("dead%d", count), fmt.Sprintf("original%d", count) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),AX\nMOVQ index+8(FP),CX\n%s %s,(AX)\nRET\n", name, op, index) + fmt.Fprintf(&declarations, "func %s(p *byte,index int64)\n", name) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\nMOVQ SI,CX\n"+op+" "+index+",(DI)\nRET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,int64_t),%s(uint8_t*,int64_t);\n", name, original) + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + indexed, writes := 0, 1 + if index == "CX" { + indexed = 1 + } + if len(op) == 3 { + writes = 0 + } + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d},\n", original, name, width, indexed, writes) + count++ + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + native.String() + ` +typedef void (*program)(uint8_t*,int64_t); +static const struct { program original,translated; int width,indexed,writes; } pairs[]={ +` + pairs.String() + `}; +static int outcome(program fn,uint8_t *p,int64_t index) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn(p,index); _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)) return 1; + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *data=mmap(0,3*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(data==MAP_FAILED || mprotect(data+2*page,page,PROT_NONE)) return 2; + memset(data,0xff,2*page); + for(unsigned i=0;i 127 { + return fail("bit index is outside Go's signed Yi8 class") + } + } else if form.index.Kind != OpReg || !isX86YrlRegisterForArch(form.index.Reg, arch) { + return fail("bit index is outside Go's Yi8/Yrl rows") + } + if form.destination.Kind == OpReg { + if !isX86YrlRegisterForArch(form.destination.Reg, arch) { + return fail("destination is outside Go's Yml register class") + } + } else if !x86ScalarCompareMemoryValid(form.destination, arch) { + if x86SourceTLSMemoryRole(form.destination, arch) { + return form, true, fmt.Errorf("%w: bit string needs its actual Go TLS address contract", ErrProbeNeedsContext) + } + return fail("destination is outside Go's encodable Yml memory class") + } + return form, true, nil +} + +func x86TypedBitTestByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86BitTestProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.bits / 8) + return x86TypedByteEffect{ + operands: []x86TypedByteOperand{ + {index: 0, operand: form.index, width: width, read: true}, + {index: 1, operand: form.destination, width: width, read: true, write: form.stem != "BT"}, + }, + flagsDefined: x86ByteFlagCF, + flagsKilled: x86ByteFlagOF | x86ByteFlagSF | x86ByteFlagAF | x86ByteFlagPF, + dynamicMemory: form.index.Kind == OpReg && form.destination.Kind != OpReg, + }, true, nil +} diff --git a/amd64_bittest_fp_runtime_test.go b/amd64_bittest_fp_runtime_test.go new file mode 100644 index 000000000..48351829c --- /dev/null +++ b/amd64_bittest_fp_runtime_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86BitTestFPPartialWriteback(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, checks, goChecks strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, op := range x86BitTestOracleNames("amd64") { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + for offset := 0; offset+width <= 8; offset++ { + name := fmt.Sprintf("Partial%d", count) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\n%s $-1,x+%d(FP)\nMOVQ x+0(FP),AX\nMOVQ AX,ret+8(FP)\nRET\n", name, op, offset) + fmt.Fprintf(&declarations, "func %s(x uint64) uint64\n", name) + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint64_t);\n", name) + mask := uint64(1) << uint(8*(offset+width)-1) + cWant, goWant := "x", "x" + switch op[:len(op)-1] { + case "BTC": + cWant, goWant = fmt.Sprintf("(x^UINT64_C(0x%x))", mask), fmt.Sprintf("x^uint64(0x%x)", mask) + case "BTR": + cWant, goWant = fmt.Sprintf("(x&~UINT64_C(0x%x))", mask), fmt.Sprintf("x&^uint64(0x%x)", mask) + case "BTS": + cWant, goWant = fmt.Sprintf("(x|UINT64_C(0x%x))", mask), fmt.Sprintf("x|uint64(0x%x)", mask) + } + fmt.Fprintf(&checks, "if(%s(x)!=%s) {fprintf(stderr,\"FP partial %d mismatch\\n\");return 1;}\n", name, cWant, count) + fmt.Fprintf(&goChecks, "if got,want:=%s(x),%s;got!=want {t.Fatalf(\"FP partial %d got=%%x want=%%x\",got,want)}\n", name, goWant, count) + count++ + } + } + if count != 52 { + t.Fatalf("partial FP byte-span denominator changed: %d", count) + } + const patterns = "0,1,0x8000000000000000,0xffffffffffffffff,0x55aa0123cdef9876,0x7ff8000000000001" + original := "package oracle\nimport \"testing\"\nfunc TestOriginal(t *testing.T) { for _,x:=range []uint64{" + patterns + "} {\n" + goChecks.String() + "}}\n" + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), source.String(), original) + main := "#include \n#include \n" + prototypes.String() + + "int main(void) {const uint64_t patterns[]={" + patterns + "};\nfor(unsigned i=0;i<6;++i) {uint64_t x=patterns[i];\n" + checks.String() + + "}\nputs(\"52 partial FP programs x 6 complete values\");return 0;}\n" + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), strings.TrimPrefix(declarations.String(), "package oracle\n"), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, pipeline := range []struct{ level, ir string }{{"-O0", ir}, {"-O2", x86MulDivRoundOptimize(t, ir)}} { + output := execution.compileAndRunLLVM(t, "bit-test-fp-partial", pipeline.ir, main, pipeline.level) + if string(output) != "52 partial FP programs x 6 complete values\n" { + t.Fatalf("missing complete original/LLVM FP value oracle: %s", output) + } + } + }) + } +} diff --git a/amd64_bittest_runtime_test.go b/amd64_bittest_runtime_test.go new file mode 100644 index 000000000..e98214360 --- /dev/null +++ b/amd64_bittest_runtime_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The original Go machine bytes and this independent whole-byte reference +// agree separately with LLVM. CF and preserved ZF are compared; the ISA's +// undefined OF/SF/AF/PF are deliberately not assigned invented expectations. +func TestCrossLinuxRuntimeMatrixX86BitTestNumericalAndAliases(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + count := 0 + for _, op := range x86BitTestOracleNames("amd64") { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + operation := map[string]int{"BT": 0, "BTC": 1, "BTR": 2, "BTS": 3}[op[:len(op)-1]] + for _, memory := range []bool{false, true} { + for _, index := range []string{"CX", "$-128", "$-1", "$0", "$15", "$16", "$31", "$32", "$63", "$64", "$127", "DX"} { + if memory && index == "DX" { + continue // DX/DX is the separate register alias axis. + } + name, original := fmt.Sprintf("numeric%d", count), fmt.Sprintf("original%d", count) + destination, originalDestination, after := "DX", "DX", "MOVQ DX,(DI)\n" + memoryFlag, mode, immediate := 0, 0, 0 + if memory { + destination, originalDestination, after, memoryFlag = "(DI)", "(DI)", "", 1 + } + if strings.HasPrefix(index, "$") { + mode = 1 + if _, err := fmt.Sscanf(index, "$%d", &immediate); err != nil { + t.Fatal(err) + } + } else if index == "DX" { + mode = 2 + } + body := op + " " + index + "," + destination + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-32\nMOVQ p+0(FP),DI\nMOVQ index+8(FP),CX\nMOVQ value+16(FP),DX\nCMPQ DX,DX\n%s\nPUSHFQ\nPOPQ AX\nANDQ $65,AX\n%sMOVQ AX,ret+24(FP)\nRET\n", name, body, after) + fmt.Fprintf(&declarations, "func %s(p *byte,index int64,value uint64) uint64\n", name) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\nMOVQ SI,CX\nCMPQ DX,DX\n"+op+" "+index+","+originalDestination+"\nPUSHFQ\nPOPQ AX\nANDQ $65,AX\n"+after+"RET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint8_t*,int64_t,uint64_t);\n", name) + prototypes.WriteString(x86OriginalSysVDeclaration("uint64_t", original, "uint8_t*,int64_t,uint64_t")) + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d,%d,%d},\n", original, name, width, operation, memoryFlag, mode, immediate) + count++ + } + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +` + prototypes.String() + native.String() + + x86OriginalSysVPointer("uint64_t", "originalProgram", "uint8_t*,int64_t,uint64_t") + ` +typedef uint64_t (*program)(uint8_t*,int64_t,uint64_t); +static const struct { originalProgram original; program translated; int width,op,memory,mode,immediate; } pairs[]={ +` + pairs.String() + `}; +int main(void) { + const int64_t indexes[]={-255,-129,-128,-65,-64,-33,-32,-17,-16,-1,0,1,15,16,31,32,63,64,65,127,255}; + const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000), + UINT64_C(0x55aa0123cdef9876),UINT64_C(0x12345678ffffffff)}; + for(unsigned i=0;i=0?signedIndex/bits:-1-((-1-signedIndex)/bits))*pairs[i].width; + } + uint64_t old=values[v]; + if(pairs[i].memory) { + old=0; + memcpy(&old,expected+257+offset,pairs[i].width); + } + uint64_t selected=UINT64_C(1)<<(index&(bits-1)); + uint64_t flags=64|!!(old&selected),result=old; + if(pairs[i].op==1) result^=selected; + if(pairs[i].op==2) result&=~selected; + if(pairs[i].op==3) result|=selected; + if(pairs[i].memory) { + if(pairs[i].op) memcpy(expected+257+offset,&result,pairs[i].width); + } else { + if(pairs[i].op && bits==32) result=(uint32_t)result; + memcpy(expected+257,&result,8); + } + uint64_t a=pairs[i].original(original+257,indexes[j],values[v]); + uint64_t b=pairs[i].translated(translated+257,indexes[j],values[v]); + if(a!=flags || b!=flags || memcmp(original,expected,sizeof(expected)) || memcmp(translated,expected,sizeof(expected))) { + fprintf(stderr,"bit test=%u index=%lld value=%llx expected flags=%llx Go=%llx LLVM=%llx\n", + i,(long long)indexes[j],(unsigned long long)values[v],(unsigned long long)flags,(unsigned long long)a,(unsigned long long)b); + return 1; + } + } + } + } + printf("%zu original Go/LLVM bit tests x 126 independent whole-memory, alias and CF/ZF checks\n",sizeof(pairs)/sizeof(*pairs)); + return 0; +} + +` + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, pipeline := range []struct{ name, level, ir string }{ + {"backend_O0", "-O0", ir}, + {"IR_O2/backend_O2", "-O2", x86MulDivRoundOptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "bit-test-numerical", pipeline.ir, main, pipeline.level) + want := fmt.Sprintf("%d original Go/LLVM bit tests x 126 independent whole-memory, alias and CF/ZF checks\n", count) + if string(output) != want { + t.Fatalf("missing independent original/LLVM numerical receipt: %s", output) + } + }) + } + }) + } +} diff --git a/amd64_bittest_typed_test.go b/amd64_bittest_typed_test.go new file mode 100644 index 000000000..eda656e7f --- /dev/null +++ b/amd64_bittest_typed_test.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func x86BitTestOracleNames(arch string) []string { + var names []string + for _, stem := range []string{"BT", "BTC", "BTR", "BTS"} { + for _, suffix := range "WLQ" { + if arch != "386" || suffix != 'Q' { + names = append(names, stem+string(suffix)) + } + } + } + return names +} + +func TestX86BitTestDeadReadSurvivesIROptimization(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + move, word, triple := "MOVQ", 8, "x86_64-unknown-linux-gnu" + if arch == "386" { + move, word, triple = "MOVL", 4, "i386-unknown-linux-gnu" + } + for _, op := range x86BitTestOracleNames(arch) { + t.Run(arch+"/"+op, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·dead(SB),4,$0-%d\n%s p+0(FP),AX\n%s $0,(AX)\nRET\n", word, move, op) + goAssemblerOracleOutput(t, arch, source, "", true) + ir, err := x86ScalarFPBitViewAPI(t, source, "func dead(p *byte)", arch, triple, "text") + if err != nil { + t.Fatal(err) + } + optimized := x86MulDivRoundOptimize(t, ir) + if !strings.Contains(optimized, "load volatile") { + t.Fatalf("Go's required bit-string read/fault was removed by LLVM IR O2:\n%s", optimized) + } + if strings.HasPrefix(op, "BTC") || strings.HasPrefix(op, "BTR") || strings.HasPrefix(op, "BTS") { + if !strings.Contains(optimized, "store volatile") { + t.Fatal("native bit-string write is not independently observable") + } + } + }) + } + } +} + +func TestX86BitTest386ImmediateClassification(t *testing.T) { + for _, op := range x86BitTestOracleNames("386") { + // Go mode32 classifies its int32 value before selecting Yi8. This + // witness is legal despite the original source literal exceeding 32 bits. + source := "TEXT ·value(SB),4,$0-0\n" + op + " $4294967295,AX\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + if _, err := x86ScalarFPBitViewAPI(t, source, "func value()", "386", "i386-unknown-linux-gnu", "text"); err != nil { + t.Errorf("actual Go signed-Yi8 classification %s: %v", op, err) + } + } +} + +func TestX86BitTestDynamicFPDoesNotBecomeModulo(t *testing.T) { + for _, op := range x86BitTestOracleNames("amd64") { + source := "TEXT ·value(SB),4,$0-16\nMOVQ index+8(FP),CX\n" + op + " CX,x+0(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + _, err := x86ScalarFPBitViewAPI(t, source, "func value(x uint64,index int64)", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("dynamic native bit string borrowed fixed FP slot/modulo: %s: %v", op, err) + } + } +} diff --git a/amd64_blend_variable_runtime_test.go b/amd64_blend_variable_runtime_test.go new file mode 100644 index 000000000..8f0817522 --- /dev/null +++ b/amd64_blend_variable_runtime_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86VariableBlendMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" { + t.Skip("execution requires amd64 or Rosetta; native amd64 CI and five-target objects are required") + } + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, family := range x86VariableBlendTestFamilies { + for _, width := range []int{0, 16, 32} { + for _, memory := range []bool{false, true} { + for dst := 0; dst < 4; dst++ { + name := fmt.Sprintf("blend%d", index) + index++ + reg, move, op, size := "X", "MOVOU", family.legacy, 16 + if width > 0 { + op, size = family.vector, width + } + if width == 32 { + reg, move = "Y", "VMOVUPS" + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ base+0(FP),AX\nMOVQ source+8(FP),BX\nMOVQ mask+16(FP),CX\nMOVQ out+24(FP),DI\n") + for r := 0; r < 4; r++ { + fmt.Fprintf(&source, "VMOVUPS (AX),Z%d\n", r) + } + fmt.Fprintf(&source, "%s (CX),%s0\n%s (BX),%s1\n", move, reg, move, reg) + operand := reg + "1" + if memory { + operand = "(BX)" + } + line := fmt.Sprintf("%s %s0,%s,%s%d", op, reg, operand, reg, dst) + if width > 0 { + line = fmt.Sprintf("%s %s0,%s,%s2,%s%d", op, reg, operand, reg, reg, dst) + } + emitX86GoEncodedTestInstruction(t, &source, line, raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\nRET\n", dst, dst, dst) + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void*, const void*, const void*, void*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, %d, %d, %d, %d)) {\n", name, size, family.bits/8, width, dst) + fmt.Fprintf(&checks, " fprintf(stderr, \"%s failed\\n\");\n return 1;\n }\n", name) + } + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + body := "#include \n#include \n#include \n" + + x86TestGuardPagesC + declarations.String() + x86VariableBlendOracleC + + "\nint main(void) {\n if (setup_guard()) return 2;\n" + + checks.String() + " return free_guard();\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "variable-blend", triple, ir, body, prefix) + }) + } +} + +const x86VariableBlendOracleC = ` +static int check(void (*fn)(const void*, const void*, const void*, void*), + int size, int lane, int vex, int dst) { + for (int n = 0; n < 256; n++) { + uint8_t old[65], storage[33], masks[33], out[112], want[112] = {0}; + uint8_t *input = storage + 1, *mask = masks + 1; + for (int i = 0; i < 65; i++) old[i] = (uint8_t)(i * 3 + n); + for (int i = 0; i < size; i++) { + input[i] = (uint8_t)(i * 7 + n + 83); + mask[i] = (uint8_t)(n + i * 31); + } + const uint8_t *base = old + 1; + if (!vex && dst == 0) base = mask; + if (!vex && dst == 1) base = input; + for (int i = 0; i < size; i += lane) { + int enabled = mask[i + lane - 1] >> 7; + memcpy(want + i, (enabled ? input : base) + i, (size_t)lane); + } + if (!vex) memcpy(want + 16, old + 1 + 16, 48); + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + for (int edge = 0; edge < 2; edge++) { + const uint8_t *source = input; + if (edge) { + source = guard + page - size; + memcpy(guard + page - size, input, (size_t)size); + } + memset(out, 0xa5, sizeof(out)); + fn(old + 1, source, mask, out); + if (memcmp(out, want, sizeof(out))) { + fprintf(stderr, "size=%d lane=%d vex=%d dst=%d n=%d edge=%d\n", + size, lane, vex, dst, n, edge); + return 1; + } + } + } + return 0; +} +` diff --git a/amd64_blocks.go b/amd64_blocks.go index 19e9cba47..57e6757b1 100644 --- a/amd64_blocks.go +++ b/amd64_blocks.go @@ -6,38 +6,62 @@ import ( ) type amd64Block struct { - name string - instrs []Instr + name string + instrs []Instr + synthetic bool +} + +func isAMD64ConditionalBranch(op Op) bool { + switch Op(strings.ToUpper(string(op))) { + case "JCXZW", "JCXZL", "JCXZQ": + return true + case "JE", "JEQ", "JZ", "JNE", "JNZ", + "JL", "JLT", "JNGE", "JLE", "JNG", "JG", "JGT", "JNLE", "JGE", "JNL", "JS", "JMI", "JNS", "JPL", + "JA", "JHI", "JNBE", "JAE", "JHS", "JNB", "JB", "JLO", "JNAE", "JBE", "JLS", "JNA", + "JC", "JCS", "JNC", "JCC", + "JO", "JOS", "JNO", "JOC", "JP", "JPE", "JPS", "JNP", "JPO", "JPC": + return true + default: + return false + } +} + +func isAMD64CounterZeroBranch(op Op) bool { + switch Op(strings.ToUpper(string(op))) { + case "JCXZW", "JCXZL", "JCXZQ": + return true + default: + return false + } } func amd64SplitBlocks(fn Func) []amd64Block { - blocks := []amd64Block{{name: "entry"}} + names := newAssemblyBlockNames(fn) + blocks := []amd64Block{{name: names.fresh("entry"), synthetic: true}} cur := 0 anon := 0 isPCRelTarget := func(ins Instr) (off int64, ok bool) { op := strings.ToUpper(string(ins.Op)) - switch Op(op) { - case "JMP", - "JE", "JEQ", "JZ", "JNE", "JNZ", - "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JS", "JNS", - "JB", "JLO", "JBE", "JA", "JHI", "JAE", "JHS", "JLS", "JNA", - "JNC", "JC", "JCC": - default: + if op != "JMP" && op != "LOOP" && op != "XBEGIN" && !isAMD64ConditionalBranch(Op(op)) { return 0, false } - if len(ins.Args) != 1 || ins.Args[0].Kind != OpMem { + args := ins.Args + if op != "JMP" && len(args) == 2 && args[0].Kind == OpImm { + args = args[1:] + } + if len(args) != 1 || args[0].Kind != OpMem { return 0, false } - if !strings.EqualFold(string(ins.Args[0].Mem.Base), "PC") { + if !strings.EqualFold(string(args[0].Mem.Base), "PC") { return 0, false } - return ins.Args[0].Mem.Off, true + return args[0].Mem.Off, true } startAnon := func() { anon++ - blocks = append(blocks, amd64Block{name: fmt.Sprintf("anon_%d", anon)}) + blocks = append(blocks, amd64Block{name: names.fresh(fmt.Sprintf("anon_%d", anon)), synthetic: true}) cur = len(blocks) - 1 } @@ -46,16 +70,16 @@ func amd64SplitBlocks(fn Func) []amd64Block { return true } op := strings.ToUpper(string(ins.Op)) - switch Op(op) { - case "JMP", - "JE", "JEQ", "JZ", "JNE", "JNZ", - "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JS", "JNS", - "JB", "JLO", "JBE", "JA", "JHI", "JAE", "JHS", "JLS", "JNA", - "JNC", "JC", "JCC": + if _, ok := amd64FarReturnSpecs[Op(op)]; ok { return true - default: - return false } + if spec, ok := amd64SystemTransferSpecs[Op(op)]; ok && spec.terminates() { + return true + } + if op == "JMP" || op == "LOOP" || op == "XBEGIN" || isAMD64ConditionalBranch(Op(op)) { + return true + } + return false } linear := make([]Instr, 0, len(fn.Instrs)) @@ -79,8 +103,9 @@ func amd64SplitBlocks(fn Func) []amd64Block { for _, ins := range fn.Instrs { if ins.Op == OpLABEL && len(ins.Args) == 1 && ins.Args[0].Kind == OpLabel { lbl := ins.Args[0].Sym - if len(blocks[cur].instrs) == 0 && strings.HasPrefix(blocks[cur].name, "anon_") { + if cur != 0 && len(blocks[cur].instrs) == 0 && blocks[cur].synthetic { blocks[cur].name = lbl + blocks[cur].synthetic = false continue } blocks = append(blocks, amd64Block{name: lbl}) @@ -97,15 +122,22 @@ func amd64SplitBlocks(fn Func) []amd64Block { } } - if len(blocks) > 1 && len(blocks[len(blocks)-1].instrs) == 0 && strings.HasPrefix(blocks[len(blocks)-1].name, "anon_") { + if len(blocks) > 1 && len(blocks[len(blocks)-1].instrs) == 0 && blocks[len(blocks)-1].synthetic { blocks = blocks[:len(blocks)-1] } return blocks } func amd64LLVMBlockName(src string) string { + return "bb_" + amd64LLVMSafeName(src) +} + +// Keep block names in a namespace distinct from SSA temporaries and register +// allocas. Go permits local labels such as x1 and t1, which otherwise collide +// with the translator's %x1 and %t1 values in LLVM IR. +func amd64LLVMSafeName(src string) string { if src == "" { - return "bb" + return "empty" } var out []byte for i := 0; i < len(src); i++ { @@ -115,16 +147,15 @@ func amd64LLVMBlockName(src string) string { continue } out = append(out, '_') + out = append(out, "0123456789abcdef"[ch>>4]) + out = append(out, "0123456789abcdef"[ch&0xf]) } s := string(out) if s == "" { - return "bb" + return "empty" } if s[0] >= '0' && s[0] <= '9' { - s = "bb_" + s - } - if s == "entry" { - s = "bb_entry" + s = "n_" + s } return s } diff --git a/amd64_bmi2_shift_cross_test.go b/amd64_bmi2_shift_cross_test.go new file mode 100644 index 000000000..ace3f5368 --- /dev/null +++ b/amd64_bmi2_shift_cross_test.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "testing" +) + +// Keep this in the required CI cross-runtime regex, not an optional native test. +func TestCrossLinuxRuntimeMatrixBMI2Shift386(t *testing.T) { + if os.Getenv("PLAN9ASM_CROSS_EXEC") != "1" { + t.Skip("set PLAN9ASM_CROSS_EXEC=1 for the required Linux cross-runtime matrix") + } + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("cross-runtime driver requires linux/amd64") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, tool := range []string{"i686-linux-gnu-gcc", "qemu-i386"} { + if _, err := exec.LookPath(tool); err != nil { + t.Fatalf("required cross-runtime tool %s: %v", tool, err) + } + } + + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range x86BMI2ShiftTestOps { + kind := 0 + if strings.HasPrefix(op, "SHRX") { + kind = 1 + } else if strings.HasPrefix(op, "SARX") { + kind = 2 + } + + for _, raw := range []bool{false, true} { + for _, destination := range []string{"CX", "SP"} { + name := fmt.Sprintf("shift386_%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-12\n", name) + source.WriteString("MOVL count+0(FP),AX\nMOVL source+4(FP),BX\n") + line := fmt.Sprintf("%s AX,(BX),%s\n", op, destination) + if raw { + code := assembleX87ControlBytes(t, "386", "TEXT probe(SB),4,$0-0\n"+line+"RET\n") + for _, b := range code[:len(code)-1] { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + } else { + source.WriteString(line) + } + fmt.Fprintf(&source, "MOVL %s,ret+8(FP)\nRET\n", destination) + + sigs[name] = FuncSig{ + Name: name, + Args: []LLVMType{I32, Ptr}, + Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: I32, Index: 0, Field: -1}, + {Offset: 4, Type: Ptr, Index: 1, Field: -1}, + }, + Results: []FrameSlot{ + {Offset: 8, Type: I32, Index: 0, Field: -1}, + }, + }, + } + fmt.Fprintf(&declarations, "extern uint32_t %s(uint32_t, const uint32_t*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, value, %d)) return %d;\n", name, kind, index) + } + } + } + if index != 24 { + t.Fatalf("generated %d functions, want 24", index) + } + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +#include +#include +` + declarations.String() + x86BMI2ShiftOracleC + ` +static void memory_fault(int signal_number) { + (void)signal_number; + _exit(98); +} + +static int check(uint32_t (*fn)(uint32_t, const uint32_t*), uint32_t *value, int kind) { + uint32_t values[] = {0, 1, UINT32_MAX, 0x80000000U, 0x7fffffffU, 0xfedcba98U}; + for (unsigned n = 0; n < 256; n++) { + *value = values[n % 6]; + uint32_t count = n < 128 ? n : UINT32_MAX - n; + if (fn(count, value) != shift_reference(*value, count, 32, kind)) { + fprintf(stderr, "386 kind=%d case=%u\n", kind, n); + return 1; + } + } + return 0; +} + +int main(void) { + if (signal(SIGSEGV, memory_fault) == SIG_ERR) return 92; + long page = sysconf(_SC_PAGESIZE); + if (page < 4) return 90; + + uint8_t *memory = mmap(NULL, page * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (memory == MAP_FAILED || mprotect(memory + page, page, PROT_NONE)) return 91; + uint32_t *value = (uint32_t *)(memory + page - 4); +` + checks.String() + " return munmap(memory, page * 2) != 0;\n}\n" + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc"}, "bmi2_shift_386", + "i386-unknown-linux-gnu", ir, mainC, []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) +} diff --git a/amd64_bmi2_shift_runtime_test.go b/amd64_bmi2_shift_runtime_test.go new file mode 100644 index 000000000..bed5d08d4 --- /dev/null +++ b/amd64_bmi2_shift_runtime_test.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86BMI2ShiftRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" { + t.Skip("execution requires amd64 or Rosetta; five-target objects remain required") + } + + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range x86BMI2ShiftTestOps { + bits, kind := 32, 0 + if strings.HasSuffix(op, "Q") { + bits = 64 + } + if strings.HasPrefix(op, "SHRX") { + kind = 1 + } else if strings.HasPrefix(op, "SARX") { + kind = 2 + } + + for _, raw := range []bool{false, true} { + for _, memory := range []bool{false, true} { + for _, destination := range []string{"CX", "AX", "DX"} { + name := fmt.Sprintf("bmi2shift%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\n", name) + source.WriteString("MOVQ count+0(FP),AX\nMOVQ source+8(FP),BX\nMOVQ out+16(FP),DI\n") + source.WriteString("MOVQ (BX),DX\nMOVQ $-1,CX\nCMPQ AX,AX\nSTC\n") + input := "DX" + if memory { + input = "(BX)" + } + line := fmt.Sprintf("%s AX,%s,%s", op, input, destination) + emitX86GoEncodedTestInstruction(t, &source, line, raw) + fmt.Fprintf(&source, "MOVQ %s,(DI)\n", destination) + source.WriteString("SETCS 8(DI)\nSETEQ 9(DI)\nSETPS 10(DI)\nSETMI 11(DI)\nSETOS 12(DI)\nRET\n") + + sigs[name] = FuncSig{ + Name: name, + Args: []LLVMType{I64, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(uint64_t, const void*, void*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, %d, %d)) return %d;\n", name, bits, kind, index) + } + } + } + } + if index != 72 { + t.Fatalf("generated %d functions, want 72", index) + } + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := "#include \n#include \n#include \n" + + declarations.String() + x86BMI2ShiftOracleC + ` +static int check(void (*fn)(uint64_t, const void*, void*), int bits, int kind) { + uint64_t values[] = {0, 1, UINT64_MAX, UINT64_C(0x8000000080000000), + UINT64_C(0x7fffffff7fffffff), UINT64_C(0xfedcba9876543210)}; + for (unsigned n = 0; n < 256; n++) { + uint8_t input[9], out[13] = {0}; + uint64_t value = values[n % 6]; + uint64_t count = n < 128 ? n : UINT64_MAX - n; + memcpy(input + 1, &value, sizeof(value)); + + fn(count, input + 1, out); + uint64_t got; + memcpy(&got, out, sizeof(got)); + if (got != shift_reference(value, count, bits, kind) || + out[8] != 1 || out[9] != 1 || out[10] != 1 || out[11] || out[12]) { + fprintf(stderr, "bits=%d kind=%d case=%u\n", bits, kind, n); + return 1; + } + } + return 0; +} + +int main(void) { +` + checks.String() + " return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "bmi2_shift", triple, ir, mainC, prefix) +} + +const x86BMI2ShiftOracleC = ` +static uint64_t shift_reference(uint64_t value, uint64_t count, int bits, int kind) { + uint64_t mask = bits == 64 ? UINT64_MAX : UINT32_MAX; + unsigned shift = (unsigned)count & (bits - 1); + value &= mask; + if (kind == 0) { + return (value << shift) & mask; + } + + uint64_t result = value >> shift; + if (kind == 2 && shift && (value & (UINT64_C(1) << (bits - 1)))) { + result |= mask ^ (mask >> shift); + } + return result; +} +` diff --git a/amd64_broadcast_go_frame_test.go b/amd64_broadcast_go_frame_test.go new file mode 100644 index 000000000..2e00c5015 --- /dev/null +++ b/amd64_broadcast_go_frame_test.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedBroadcastGoFrameScalarWidths(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + pointerBytes, movePointer := 8, "MOVQ" + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + pointerBytes, movePointer = 4, "MOVL" + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, form := range []struct { + op string + bytes int + }{ + {"VPBROADCASTB", 1}, {"VPBROADCASTW", 2}, + {"VPBROADCASTD", 4}, {"VPBROADCASTQ", 8}, + } { + t.Run(arch+"/"+form.op, func(t *testing.T) { + outOffset := (form.bytes + pointerBytes - 1) &^ (pointerBytes - 1) + var source, declarations strings.Builder + declarations.WriteString("package broadcast\n") + for _, width := range []string{"X", "Y", "Z"} { + for index, masking := range []struct{ suffix, mask string }{ + {}, {mask: "K1,"}, {suffix: ".Z", mask: "K7,"}, + } { + name := fmt.Sprintf("Broadcast%s%d", width, index) + fmt.Fprintf(&declarations, "func %s(x uint%d, out *[64]byte)\n", name, form.bytes*8) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n", name, outOffset+pointerBytes) + if masking.mask != "" { + fmt.Fprintf(&source, "%s $1,AX\nKMOVQ AX,%s\n", movePointer, strings.TrimSuffix(masking.mask, ",")) + if masking.suffix == "" { + clear := map[string]string{"X": "PXOR X0,X0", "Y": "VPXOR Y0,Y0,Y0", "Z": "VPXORD Z0,Z0,Z0"}[width] + fmt.Fprintln(&source, clear) + } + } + fmt.Fprintf(&source, "%s%s x+0(FP),%s%s0\n", form.op, masking.suffix, masking.mask, width) + fmt.Fprintf(&source, "%s out+%d(FP),AX\n", movePointer, outOffset) + store := map[string]string{"X": "MOVOU", "Y": "VMOVDQU", "Z": "VMOVDQU64"}[width] + fmt.Fprintf(&source, "%s %s0,(AX)\nRET\n", store, width) + } + } + requireX86GoAssemblerResult(t, arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "example.com/broadcast", declarations.String()) + resolve := testResolveSym(pkg.Path) + sigs, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range triples { + t.Run(triple, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{ + Goarch: arch, TargetTriple: triple, Sigs: sigs, ResolveSym: resolve, + }) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + if err := llvm.VerifyModule(module, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "broadcast-frame.ll", "broadcast-frame.o", module.String()) + }) + } + }) + } + } +} diff --git a/amd64_broadcast_views_test.go b/amd64_broadcast_views_test.go new file mode 100644 index 000000000..7880db6c1 --- /dev/null +++ b/amd64_broadcast_views_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Independent axes from Go 1.27 _yvpbroadcastb: B/W/D/Q, X/Y/Z, +// memory/X/GP source, unmasked/merge/zero. Observe every overlapping view, +// including upper bits, and suppress masked-off scalar memory faults. +func integerBroadcastViews(t *testing.T, triple string) (string, string) { + t.Helper() + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, spec := range []struct { + op string + bits int + }{ + {"VPBROADCASTB", 8}, {"VPBROADCASTW", 16}, {"VPBROADCASTD", 32}, {"VPBROADCASTQ", 64}, + } { + for width, dst := range []string{"X0", "Y0", "Z0"} { + for input := 0; input < 3; input++ { + for mask := 0; mask < 3; mask++ { + name := fmt.Sprintf("integer_broadcast_%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ old+0(FP),BX\nMOVQ src+8(FP),AX\nMOVQ out+16(FP),DI\nMOVQ mask+24(FP),CX\nKMOVQ CX,K7\nVMOVUPS (BX),Z0\n", name) + operand := "(AX)" + if input == 1 { + source.WriteString("VMOVUPS (AX),X1\n") + operand = "X1" + } else if input == 2 { + source.WriteString("MOVQ (AX),R8\n") + operand = "R8" + } + op, predicate := spec.op, "" + if mask != 0 { + predicate = "K7," + } + if mask == 2 { + op += ".Z" + } + fmt.Fprintf(&source, "%s %s,%s%s\nVMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n", op, operand, predicate, dst) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,uint64_t);\n", name) + fmt.Fprintf(&checks, "if (check(%s,%d,%d,%d,%d)) return 1;\n", name, 16< +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,const void *,void *,uint64_t),int width,int lane,int form,int input) { + const uint64_t masks[] = {0, 1, 2, UINT64_C(0xaaaaaaaaaaaaaaaa), UINT64_MAX, UINT64_C(1) << 63}; + for (unsigned m = 0; m < sizeof(masks)/sizeof(masks[0]); m++) { + uint8_t old[64], src[16], output[112]; + for (int i = 0; i < 64; i++) old[i] = (uint8_t)(197-i*3); + for (int i = 0; i < 16; i++) src[i] = (uint8_t)(17+i*7); + uint64_t relevant = width/lane == 64 ? UINT64_MAX : (UINT64_C(1) << (width/lane)) - 1; + int inactiveMemory = input == 0 && form != 0 && (masks[m] & relevant) == 0; + fn(old, inactiveMemory ? 0 : src, output, masks[m]); + for (int view = 0; view < 3; view++) { + int offset = view == 0 ? 0 : view == 1 ? 64 : 96; + for (int i = 0; i < (64 >> view); i++) { + int enabled = form == 0 || ((masks[m] >> (i/lane)) & 1); + uint8_t want = i >= width ? 0 : enabled ? src[i%lane] : form == 2 ? 0 : old[i]; + if (output[offset+i] != want) { + fprintf(stderr,"broadcast width=%d lane=%d form=%d input=%d mask=%u view=%d byte=%d got=%u want=%u\n",width,lane,form,input,m,view,i,output[offset+i],want); + return 1; + } + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + return ir, main +} + +func TestIntegerBroadcastRegisterViews(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"x86_64-apple-macosx", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, main := integerBroadcastViews(t, triple) + compileLLVMToObject(t, llc, triple, "integer_broadcast.ll", "integer_broadcast.o", ir) + var runner []string + execute := runtime.GOARCH == "amd64" && triple == testTargetTriple(runtime.GOOS, runtime.GOARCH) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && triple == "x86_64-apple-macosx" && rosettaAvailable() { + execute = true + runner = []string{"/usr/bin/arch", "-x86_64"} + } + if execute { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang not found") + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "integer_broadcast", triple, ir, main, runner) + } + }) + } +} diff --git a/amd64_byte_alias_test.go b/amd64_byte_alias_test.go new file mode 100644 index 000000000..5a5a21f75 --- /dev/null +++ b/amd64_byte_alias_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Go 1.27's amd64 Yrb class contains BPB/SIB/DIB and R8B..R15B in +// addition to the four low and four high legacy byte-register spellings. +// REG_SPB is deliberately not classified as Yrb by cmd/internal/obj/x86. +func TestTranslateAMD64ExtendedByteRegisterCompleteGoAssemblerClass(t *testing.T) { + aliases := []string{"BPB", "SIB", "DIB", "R8B", "R9B", "R10B", "R11B", "R12B", "R13B", "R14B", "R15B"} + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT extendedbyteforms(SB),NOSPLIT,$0-0\n") + for _, alias := range aliases { + fmt.Fprintf(&source, "\tMOVB %s, (AX)\n", alias) + fmt.Fprintf(&source, "\tMOVB (AX), %s\n", alias) + fmt.Fprintf(&source, "\tMOVBQZX %s, CX\n", alias) + fmt.Fprintf(&source, "\tMOVBQSX %s, DX\n", alias) + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "extendedbyteforms": {Name: "extendedbyteforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "extended-byte-"+target.name+".ll", "extended-byte-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ExtendedByteRegisterRejectsFormsOutsideGoAssemblerClass(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + aliases := []string{"SPB"} + if target.goarch == "386" { + aliases = append(aliases, "BPB", "SIB", "DIB", "R8B", "R9B", "R10B", "R11B", "R12B", "R13B", "R14B", "R15B") + } + for _, alias := range aliases { + instruction := "MOVB " + alias + ", (AX)" + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's byte-register class for %s", instruction, target.goarch) + } + } + }) + } +} + +func TestAMD64ExtendedByteRegisterAliasesShareFullRegisterStorage(t *testing.T) { + for _, test := range []struct { + alias Reg + base Reg + }{ + {BPB, BP}, {SIB, SI}, {DIB, DI}, + {R8B, Reg("R8")}, {R9B, Reg("R9")}, {R10B, Reg("R10")}, {R11B, Reg("R11")}, + {R12B, Reg("R12")}, {R13B, Reg("R13")}, {R14B, Reg("R14")}, {R15B, Reg("R15")}, + } { + base, shift, ok := amd64ByteAlias(test.alias) + if !ok || base != test.base || shift != 0 { + t.Errorf("amd64ByteAlias(%s) = (%s, %d, %v), want (%s, 0, true)", test.alias, base, shift, ok, test.base) + } + } +} diff --git a/amd64_byte_shuffle_caller_test.go b/amd64_byte_shuffle_caller_test.go new file mode 100644 index 000000000..b4a0ab303 --- /dev/null +++ b/amd64_byte_shuffle_caller_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86ByteShuffleActualCallerThreeAPIs(t *testing.T) { + var body strings.Builder + rows := []string{"PSHUFB X0,X1"} + for _, bank := range []string{"X", "Y", "Z"} { + for _, control := range []string{bank + "0", "(DI)"} { + rows = append(rows, + fmt.Sprintf("VPSHUFB %s,%s1,%s1", control, bank, bank), + fmt.Sprintf("VPSHUFB %s,%s1,K1,%s1", control, bank, bank), + fmt.Sprintf("VPSHUFB.Z %s,%s1,K1,%s1", control, bank, bank)) + } + } + for _, row := range rows { + fmt.Fprintf(&body, "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\n%s\nVMOVSS X1,(DX)\n", row) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body.String(), 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "shuffle-text.ll", "shuffle-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "shuffle-module.ll", "shuffle-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86ByteShuffleCallerMemoryCertificateAndAlignment(t *testing.T) { + for _, row := range []string{"VPSHUFB (DI),X0,X2", "VPSHUFB.Z (DI),Z0,K1,Z2", "PSHUFB (DI),X0"} { + result := "X2" + if strings.HasPrefix(row, "PSHUFB") { + result = "X0" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n"+row+"\nVMOVSS "+result+",(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if strings.HasPrefix(row, "PSHUFB") { + if err == nil { + t.Fatal("caller slice acquired legacy m128 alignment") + } + continue + } + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86ByteShuffleProperties(ins.Op, ins, "amd64") + if !known || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.byteWidth) || !x86CallerMemoryCertified(ins, index) { + t.Fatal("caller lost full E4NF tuple") + } + for _, change := range []string{"opcode", "width", "address", "destination", "direction", "mask"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + updated := access + switch change { + case "opcode": + mutated.Op = "VPANDD" + case "width": + updated.width-- + case "address": + mutated.Args[index].Mem.Off++ + case "destination": + mutated.Args[form.destination].Reg = "X3" + case "direction": + updated.write = true + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + } + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: updated} + if x86CallerMemoryCertified(mutated, index) { + t.Fatal("changed shuffle retained caller memory authority", change) + } + } + } + if !found { + t.Fatal("no shuffle consumed the caller proof") + } + } +} diff --git a/amd64_byte_shuffle_effects.go b/amd64_byte_shuffle_effects.go new file mode 100644 index 000000000..699b236f8 --- /dev/null +++ b/amd64_byte_shuffle_effects.go @@ -0,0 +1,65 @@ +package plan9asm + +func x86ResultByteShuffleStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86ByteShuffleProperties(ins.Op, ins, arch) + if !known || err != nil { + return false, known + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false, true + } + } + control := uint64(0) + if form.memory >= 0 { + // Retain the existing FP/AUTO or lowering-owned same-file DATA proof. + // A mask cannot shrink the read, nor prove an arbitrary pointer safe. + if !x86ResultRead(ins.Args[form.memory], int64(form.memoryBytes), state, owned, arch) { + return false, true + } + control = x86CallerVectorByteMask(int64(form.byteWidth)) + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[form.first].Reg, form.byteWidth) + control = state.vectorBytes(index) + } + data, _ := amd64VectorRegisterIndex(ins.Args[form.second].Reg, form.byteWidth) + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + initialized := x86ByteShuffleDefinedBytes(form.byteWidth, control, state.vectorBytes(data)) + if form.mask >= 0 && !form.zeroing { + initialized &= state.vectorBytes(destination) + } + state.writeVector(destination, form.byteWidth, initialized, form.legacy) + return true, true +} + +func x86CallerByteShuffleStep(ins *Instr, form x86PackedBinaryForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + control := uint64(0) + if form.memory >= 0 { + // A bounded slice is not a witness for legacy m128 alignment. + // E4NF vector reads require the full tuple even with a zero mask. + if form.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + control = width + } else { + control = state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] + } + data := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86ByteShuffleDefinedBytes(form.byteWidth, control, data) + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_byte_shuffle_fault_unix_test.go b/amd64_byte_shuffle_fault_unix_test.go new file mode 100644 index 000000000..c4f7aa213 --- /dev/null +++ b/amd64_byte_shuffle_fault_unix_test.go @@ -0,0 +1,177 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ByteShuffleMemoryFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, entries strings.Builder + count := 0 + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for legacy := 0; legacy < 2; legacy++ { + if legacy != 0 && width != 16 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy != 0 && masking != 0 { + continue + } + for dead := 0; dead < 2; dead++ { + name := fmt.Sprint("Fault", count) + count++ + row := "VPSHUFB" + if legacy != 0 { + row = "PSHUFB" + } + if masking == 2 { + row += ".Z" + } + row += " (SI)," + bank + "1" + if masking != 0 { + row += ",K1" + } + if legacy == 0 { + row += "," + bank + "1" + } + fmt.Fprintf(&declarations, "func %s(out,control,data *[64]byte)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nMOVQ out+0(FP),DI\nMOVQ control+8(FP),SI\nMOVQ data+16(FP),DX\nVMOVUPS (DX),Z1\nMOVL $0,AX\nKMOVQ AX,K1\n%s\n", name, row) + if dead == 0 { + fmt.Fprintf(&source, "VMOVUPS %s1,(DI)\n", bank) + } + source.WriteString("RET\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", name) + original := "0" + if legacy != 0 { + // Original Go bytes use the native C argument registers. + // This does not claim native VEX/EVEX execution. + original = "Original" + name + native := "TEXT native(SB),4,$0-0\nMOVOU (DX),X1\nPSHUFB (SI),X1\n" + if dead == 0 { + native += "MOVOU X1,(DI)\n" + } + code := x86OrderingHintGoBytes(t, "amd64", native+"RET\n") + var bytes []string + for _, value := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", original) + } + fmt.Fprintf(&entries, "{%s,%s,%d,%d,%d},\n", name, original, width, masking, dead) + } + } + } + } + if count != 20 { + t.Fatal("incomplete fault family", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86ByteShuffleFaultReference, prototypes.String(), originals.String(), entries.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + text := ir + if level == "-O2" { + text = x86MulDivRoundOptimize(t, text) + } + output := execution.compileAndRunLLVM(t, "byte-shuffle-faults", text, driver, level) + if string(output) != "byte shuffle: 20 kernels, 110 address outcomes, failures 0\n" { + t.Fatal("incomplete fault oracle", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} + +const x86ByteShuffleFaultReference = ` +#include +#include +#include +#include +#include +#include +#include +#include +%s +%s +typedef void (*Program)(uint8_t*,const uint8_t*,const uint8_t*); +typedef struct { Program translated, original; int width, masking, dead; } Kernel; +static const Kernel kernels[] = {%s}; + +static int observe(Program run, Kernel k, const uint8_t *control) { + pid_t child = fork(); + if (child < 0) return -1; + if (child == 0) { + struct rlimit no_core = {0,0}; + if (setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + _Alignas(64) uint8_t data[64], out[66]; + for (int i=0;i<64;i++) data[i]=(uint8_t)(i*17+31); + memset(out,0xa7,sizeof(out)); + run(out+1,control,data); + for (int i=0;i<66;i++) { + unsigned want=0xa7; + if (!k.dead && i>=1 && i<=k.width) { + int offset=i-1; + if (k.masking) want=k.masking==2 ? 0 : data[offset]; + else want=control[offset]&0x80 ? 0 : data[offset/16*16+(control[offset]&15)]; + } + if (out[i]!=want) _exit(121); + } + _exit(0); + } + int status; + if (waitpid(child,&status,0)!=child) return -2; + if (WIFEXITED(status)) return WEXITSTATUS(status)==0 ? 0 : -3; + if (WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)) return 1; + return -4; +} + +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if (page<128) return 1; + uint8_t *memory=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if (memory==MAP_FAILED || mprotect(memory+page,page,PROT_NONE)) return 2; + for (long i=0;i1 && j!=3 && want!=0)) return 3; + outcomes++; + } + if (got<0 || got!=want) { + fprintf(stderr,"kernel %%u address %%u got %%d want %%d\n",i,j,got,want); + return 4; + } + outcomes++; + } + } + if (munmap(memory,2*page)) return 5; + printf("byte shuffle: %%zu kernels, %%u address outcomes, failures 0\n",sizeof(kernels)/sizeof(kernels[0]),outcomes); + return 0; +} +` diff --git a/amd64_byte_shuffle_forms.go b/amd64_byte_shuffle_forms.go new file mode 100644 index 000000000..9d9512d54 --- /dev/null +++ b/amd64_byte_shuffle_forms.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's ymshufb has only X/m128, X, not the ISA's separate MMX form. +// Both vector encodings share the binary grammar, with no broadcast or SAE. +var x86ByteShuffleSpecs = map[Op]x86PackedBinarySpec{ + "PSHUFB": {encoding: x86PackedLogicalLegacyFP}, + "VPSHUFB": {encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, +} + +func x86ByteShuffleProperties(op Op, ins Instr, arch string) (x86PackedBinaryForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86ByteShuffleSpecs[Op(base)] + if !known { + return x86PackedBinaryForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedBinaryForm{}, true, fmt.Errorf("byte shuffle dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + grammarArch := arch + if arch == "386" && ins.x86Encoded && spec.encoding != x86PackedLogicalLegacyFP { + // Physical EVEX masking does not inherit cmd/asm's named operand cap. + grammarArch = "amd64" + } + form, err := x86PackedBinaryProperties(local, grammarArch, spec) + if err == nil && grammarArch != arch && form.memory >= 0 && !x86PackedShiftMemoryValid(local.Args[form.memory], arch) { + err = fmt.Errorf("386 byte shuffle physical source has an unbound address") + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && form.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: byte shuffle has no bound address-size override", ErrProbeNeedsContext) + } + return form, true, err +} + +// A dynamic control byte may select any byte of its own 128-bit data lane. +// This is numerical definedness, never a memory, alignment or caller grant. +func x86ByteShuffleDefinedBytes(width int, control, data uint64) uint64 { + var known uint64 + for start := 0; start < width; start += 16 { + lane := uint64(0xffff) << uint(start) + if data&lane == lane { + known |= control & lane + } + } + return known +} + +func x86TypedByteShuffleEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86ByteShuffleProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: 16, read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: 16, read: true, write: true}, + }}, true, nil +} diff --git a/amd64_byte_shuffle_forms_test.go b/amd64_byte_shuffle_forms_test.go new file mode 100644 index 000000000..310ec0403 --- /dev/null +++ b/amd64_byte_shuffle_forms_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86ByteShuffleCompleteGoEncoderRows(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APSHUFB, ymshufb, Pq, opBytes{0x38, 0x00}}", + "var ymshufb = []ytab{\n\t{Zm2_r, 2, argList{Yxm, Yxr}},\n}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual legacy Go row changed", row) + } + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if len(x86ByteShuffleSpecs) != 2 { + t.Fatal("byte shuffle family changed") + } + for _, row := range []struct{ begin, end, want string }{ + {"{as: AVPSHUFB,", "\n\t}},", `{as: AVPSHUFB, ytab: _yvandnpd, prefix: Pavx, op: opBytes{ + avxEscape | vex128 | vex66 | vex0F38 | vexW0, 0x00, + avxEscape | vex256 | vex66 | vex0F38 | vexW0, 0x00, + avxEscape | evex128 | evex66 | evex0F38 | evexW0, evexN16 | evexZeroingEnabled, 0x00, + avxEscape | evex256 | evex66 | evex0F38 | evexW0, evexN32 | evexZeroingEnabled, 0x00, + avxEscape | evex512 | evex66 | evex0F38 | evexW0, evexN64 | evexZeroingEnabled, 0x00,`}, + {"var _yvandnpd =", "\n}", `var _yvandnpd = []ytab{ + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}}, + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yym, Yyr, Yyr}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YxmEvex, YxrEvex, YxrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YxmEvex, YxrEvex, Yknot0, YxrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YymEvex, YyrEvex, YyrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YymEvex, YyrEvex, Yknot0, YyrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{Yzm, Yzr, Yzr}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{Yzm, Yzr, Yknot0, Yzr}},`}, + } { + start := strings.Index(string(avx), row.begin) + if start < 0 || strings.SplitN(string(avx)[start:], row.end, 2)[0] != row.want { + t.Fatal("actual Go encoder classes, offsets or attributes changed", row.begin) + } + } +} + +func x86ByteShuffleFormatRows(arch string) []string { + var rows []string + for _, source := range []string{"X0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "PSHUFB "+source+",X1") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + for mode := 0; mode < 3; mode++ { + if arch == "386" && mode != 0 { + continue + } + name, args := "VPSHUFB", source+","+bank+"1" + if mode != 0 { + args += ",K7" + } + if mode == 2 { + name += ".Z" + } + rows = append(rows, name+" "+args+","+bank+"2") + } + } + } + return rows +} + +func TestX86ByteShuffleGoFormsAndMemoryShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86ByteShuffleFormatRows(arch) { + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86ByteShuffleProperties(ins.Op, ins, arch) + if !known || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatal("Go shape missing in a consumer", known, err) + } + if form.memory >= 0 { + width, write, known := x86InstructionMemoryShape(ins, form.memory, arch) + if !known || write || width != int64(form.byteWidth) { + t.Fatal("lost full E4NF control tuple", width, write, known) + } + } + }) + } + } +} + +func TestX86ByteShuffleMode32HighBanksRemainContext(t *testing.T) { + for _, bank := range []string{"X", "Y"} { + for _, index := range []int{8, 15, 16, 31} { + row := fmt.Sprintf("VPSHUFB %s%d,%s1,%s2", bank, index, bank, bank) + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("source acceptance invented physical high-bank identity", row, err) + } + } + } +} + +func TestX86ByteShuffleInvalidFormsAndSourceIdentity(t *testing.T) { + for _, row := range []string{ + "PSHUFB M0,M1", "PSHUFB X16,X1", "PSHUFB.Z X0,X1", + "VPSHUFB X0,Y1,Y2", "VPSHUFB X0,(AX),X2", "VPSHUFB $1,X1,X2", + "VPSHUFB X0,X1,K0,X2", "VPSHUFB.BCST (AX),X1,X2", + "VPSHUFB.SAE X0,X1,X2", "VPSHUFB.RN_SAE X0,X1,X2", + "VPSHUFB (AX)(CX*3),X1,X2", "VPSHUFB 2147483648(AX),X1,X2", + } { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, parseErr := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if parseErr != nil { + // The original Go rejection above still owns this source axis. + continue + } + ins := file.Funcs[0].Instrs[1] + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, "amd64"); !known || err == nil || x86CallerVectorMoveKnown(ins) { + t.Fatal("invalid Go form acquired an effect", row, known, err) + } + } + for _, row := range []string{"PSHUFB (AX),X1", "VPSHUFB (AX),X1,X2"} { + for _, mutation := range []string{"dispatch", "address-size", "unbound-override"} { + ins := x86ScalarLaneInstruction(t, row) + op := ins.Op + switch mutation { + case "dispatch": + ins.Op = "MOVOU" + case "address-size": + ins.x86AddressBits = 32 + case "unbound-override": + ins.x86UnboundAddressOverride = true + } + var out strings.Builder + ctx := amd64Ctx{goarch: "amd64", b: &out} + if known, _, err := ctx.lowerPackedByteShuffle(op, ins); !known || err == nil || out.Len() != 0 { + t.Fatal("changed source identity emitted IR", row, mutation, err, out.String()) + } + } + } +} + +func TestX86ByteShuffleVectorDispatcherUsesCompleteGrammar(t *testing.T) { + for _, row := range []string{ + "PSHUFB Y0,X0", "PSHUFB X0", "PSHUFB.Z X0,X1", + "VPSHUFB Y0,Y1,AX", "VPSHUFB X0,Y1,Y2", + "VPSHUFB X0,X1,K0,X2", "VPSHUFB.BCST (AX),X1,X2", + } { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + for _, keepOpcode := range []bool{false, true} { + ins := x86ScalarLaneInstruction(t, row) + op := ins.Op + if !keepOpcode { + ins.Op = "" + } + var output strings.Builder + ctx := amd64Ctx{goarch: "amd64", b: &output} + known, terminated, err := ctx.lowerVec(op, ins) + if !known || terminated || err == nil || output.Len() != 0 { + t.Fatalf("vector dispatcher bypassed byte-shuffle grammar for %q: (%v, %v, %v), IR %q", row, known, terminated, err, output.String()) + } + } + } + for _, row := range []string{ + "PSHUFB X0,X1", "VPSHUFB X0,X1,X2", "VPSHUFB Y0,Y1,Y2", + "VPSHUFB Z0,Z1,Z2", "VPSHUFB X0,X1,K7,X2", "VPSHUFB.Z Z0,Z1,K7,Z2", + } { + ins := x86ScalarLaneInstruction(t, row) + function := Func{Instrs: []Instr{ins}} + family, familyOutput := newAMD64CtxWithFuncForTest(t, function, FuncSig{Name: "shuffle", Ret: Void}, nil) + dispatch, dispatchOutput := newAMD64CtxWithFuncForTest(t, function, FuncSig{Name: "shuffle", Ret: Void}, nil) + if known, terminated, err := family.lowerPackedByteShuffle(ins.Op, ins); !known || terminated || err != nil { + t.Fatal("independent family entry", row, known, terminated, err) + } + if known, terminated, err := dispatch.lowerVec(ins.Op, ins); !known || terminated || err != nil { + t.Fatal("vector family entry", row, known, terminated, err) + } + if familyOutput.String() != dispatchOutput.String() { + t.Fatal("vector dispatcher changed the family's emitted effects", row) + } + } +} diff --git a/amd64_byte_shuffle_objects_test.go b/amd64_byte_shuffle_objects_test.go new file mode 100644 index 000000000..357e96beb --- /dev/null +++ b/amd64_byte_shuffle_objects_test.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86ByteShuffleThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + rows := x86ByteShuffleFormatRows(target.arch) + for begin := 0; begin < len(rows); begin += 64 { + end := begin + 64 + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprint(begin), func(t *testing.T) { + var source, declarations strings.Builder + for index, row := range rows[begin:end] { + if strings.HasPrefix(row, "PSHUFB ") && (strings.Contains(row, "(FP)") || strings.Contains(row, "(SP)")) { + // Explicit native-storage rejection controls below. + continue + } + fmt.Fprintf(&declarations, "func Shuffle%d(value [64]byte)\n", index) + fmt.Fprintf(&source, "TEXT ·Shuffle%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "byte-shuffle-"+api) + x86ResultInitializationObject(t, llc, target.triple, x86MulDivRoundOptimize(t, ir), "byte-shuffle-"+api+"-optimized") + } + }) + } + }) + } +} + +func TestX86ByteShuffleVirtualLegacyStorageRemainsContext(t *testing.T) { + for _, row := range []string{"PSHUFB value+0(FP),X1", "PSHUFB local-16(SP),X1", "PSHUFB (SP),X1", "PSHUFB (BP),X1"} { + source := "TEXT ·Shuffle(SB),4,$64-64\n" + row + "\nRET\n" + for _, target := range x86RawReturnTargets { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "func Shuffle(value [64]byte)", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("logical storage acquired original SSE alignment", row, target.triple, api, err) + } + } + } + } +} diff --git a/amd64_byte_shuffle_raw_test.go b/amd64_byte_shuffle_raw_test.go new file mode 100644 index 000000000..e5da92d4e --- /dev/null +++ b/amd64_byte_shuffle_raw_test.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ByteShuffleRawWIGAndPhysicalMasks(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, width := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + code := []byte{0xc4, 0xe2, 0x71 | width | w, 0x00, 0xd0} + ins, size, known, err := decodedX86VPSHUFBInstruction(code, mode) + if !known || err != nil || size != len(code) || ins.Op != "VPSHUFB" { + t.Errorf("VEX.WIG mode=%d code=%x: size=%d known=%t err=%v", mode, code, size, known, err) + } + } + } + for width := byte(0); width < 3; width++ { + for mask := byte(0); mask < 8; mask++ { + for _, zero := range []byte{0, 0x80} { + if mask == 0 && zero != 0 { + continue + } + for _, w := range []byte{0, 0x80} { + code := []byte{0x62, 0xf2, 0x75 | w, 8 | width<<5 | mask | zero, 0x00, 0xd0} + t.Run(fmt.Sprintf("%d/%x", mode, code), func(t *testing.T) { + ins, size, known, err := decodedX86VPSHUFBInstruction(code, mode) + if !known || err != nil || size != len(code) { + t.Fatalf("physical EVEX inherited named operand limit or W0: size=%d known=%t err=%v", size, known, err) + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + if !ins.x86Encoded { + t.Fatal("decoder lost physical encoding identity") + } + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, arch); !known || err != nil { + t.Fatal("typed physical consumer", known, err) + } + ins.x86Encoded = false + if _, _, err := x86ByteShuffleProperties(ins.Op, ins, arch); mode == 32 && mask != 0 && err == nil { + t.Fatal("named mode32 form acquired physical decoder authority") + } + }) + } + } + } + } + } +} + +func TestX86ByteShuffleRawThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var codes [][]byte + for _, width := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + codes = append(codes, []byte{0xc4, 0xe2, 0x71 | width | w, 0x00, 0xd0}) + } + } + for width := byte(0); width < 3; width++ { + for _, mask := range []byte{1, 7} { + for _, zero := range []byte{0, 0x80} { + for _, w := range []byte{0, 0x80} { + codes = append(codes, []byte{0x62, 0xf2, 0x75 | w, 8 | width<<5 | mask | zero, 0x00, 0xd0}) + } + } + } + } + var source, declarations strings.Builder + for index, code := range codes { + fmt.Fprintf(&declarations, "func RawShuffle%d()\n", index) + fmt.Fprintf(&source, "TEXT ·RawShuffle%d(SB),4,$0-0\n", index) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("RET\n") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "raw-byte-shuffle-"+api) + } + }) + } +} + +func TestX86ByteShuffleReservedRawForms(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x70, 0, 0xd0}, // Wrong mandatory prefix. + {0xc4, 0xe2, 0x71, 0}, // Truncated ModRM. + {0x62, 0xf2, 0x75, 0x68, 0, 0xd0}, // Reserved LL. + {0x62, 0xf2, 0x75, 0x18, 0, 0xd0}, // No broadcast/rounding. + {0x62, 0xf2, 0x75, 0x88, 0, 0xd0}, // Zeroing without mask. + {0x67, 0x62, 0xf2, 0x75, 0x08, 0, 0x10}, // Unbound address size. + } { + for _, mode := range []int{32, 64} { + if _, _, known, err := decodedX86VPSHUFBInstruction(code, mode); !known || err == nil { + t.Fatalf("reserved byte shuffle accepted: mode=%d code=%x known=%t err=%v", mode, code, known, err) + } + } + } +} diff --git a/amd64_byte_shuffle_result_test.go b/amd64_byte_shuffle_result_test.go new file mode 100644 index 000000000..997975159 --- /dev/null +++ b/amd64_byte_shuffle_result_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ByteShuffleIndependentResultConsumers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + const data = "DATA control<>+0(SB)/8,$0x0001020304050607\nDATA control<>+8(SB)/8,$0x08090a0b0c0d0e0f\nGLOBL control<>(SB),8,$16\n" + for _, row := range []string{ + "PSHUFB X0,X1", "VPSHUFB X0,X1,X1", + "PSHUFB control<>(SB),X1", "VPSHUFB control<>(SB),X1,X1", + } { + t.Run(row, func(t *testing.T) { + source := data + "TEXT ·view(SB),4,$0-20\nMOVQ a+0(FP),X0\nMOVQ b+8(FP),X1\n" + row + "\nPEXTRD $0,X1,AX\nMOVL AX,ret+16(FP)\nRET\n" + for _, target := range x86RawReturnTargets { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b uint64) uint32\n", target.arch, target.triple, api) + if err != nil { + t.Fatalf("%s/%s complete numerical shuffle has no result consumer: %v", target.triple, api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "byte-shuffle-result-"+api) + } + } + }) + } +} + +func TestX86ByteShuffleDeadMemoryRetainsOriginalAccess(t *testing.T) { + for _, row := range []struct { + body string + width int + }{ + {"PSHUFB (DX),X0", 16}, + {"VPSHUFB (DX),X0,X1", 16}, + {"VPSHUFB (DX),Y0,Y1", 32}, + {"VPSHUFB (DX),Z0,Z1", 64}, + {"VPSHUFB.Z (DX),Z0,K1,Z1", 64}, + } { + t.Run(row.body, func(t *testing.T) { + source := "TEXT ·view(SB),4,$0-8\nMOVQ p+0(FP),DX\nVPXORQ Z0,Z0,Z0\nMOVL $0,AX\nKMOVW AX,K1\n" + row.body + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte)\n", "amd64", "x86_64-unknown-linux-gnu", "Go") + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + want := fmt.Sprintf("load volatile <%d x i8>", row.width) + if !strings.HasPrefix(row.body, "V") { + want = "asm sideeffect \"pshufb " + } + if !strings.Contains(text, want) { + t.Errorf("dead shuffle lost its full native read/alignment effect: want %s", want) + } + } + }) + } +} + +func TestX86ByteShuffleSparseResultAndCallerBytes(t *testing.T) { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, legacy := range []bool{false, true} { + if legacy && width != 16 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy && masking != 0 { + continue + } + row, dst := "VPSHUFB", 2 + if legacy { + row, dst = "PSHUFB", 1 + } + if masking == 2 { + row += ".Z" + } + row += " " + bank + "0," + bank + "1" + if masking != 0 { + row += ",K1" + } + if !legacy { + row += "," + bank + "2" + } + ins := x86ScalarLaneInstruction(t, row) + for input := 0; input < 3; input++ { + for missing := -1; missing < 64; missing++ { + initial := [3]uint64{^uint64(0), ^uint64(0), ^uint64(0)} + if missing >= 0 { + initial[input] &^= uint64(1) << uint(missing) + } + result := x86ResultState{flags: 0x3f} + caller := x86SourceStackState{vectors: map[Reg]uint64{}, masks: map[Reg]bool{"K1": true}} + for i, value := range initial { + result.writeVector(i, 64, value, false) + caller.vectors[Reg(fmt.Sprintf("Z%d", i))] = value + } + result.masks[1] = true + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") || !x86CallerVectorMoveStep(&ins, &caller, nil) { + t.Fatal("missing byte consumer", row) + } + var want uint64 + for output := 0; output < 64; output++ { + known := !legacy || initial[dst]>>uint(output)&1 != 0 + if output < width { + known = initial[0]>>uint(output)&1 != 0 + for selected := 0; selected < 16; selected++ { + known = known && initial[1]>>uint(output/16*16+selected)&1 != 0 + } + if masking == 1 { + known = known && initial[dst]>>uint(output)&1 != 0 + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if result.vectorBytes(dst) != want || caller.vectors[Reg(fmt.Sprintf("Z%d", dst))] != want || result.flags != 0x3f { + t.Fatalf("%s input=%d missing=%d: result=%x caller=%x want=%x", row, input, missing, result.vectorBytes(dst), caller.vectors[Reg(fmt.Sprintf("Z%d", dst))], want) + } + } + } + } + } + } +} + +func TestX86ByteShuffleResultMemoryAndMaskBoundaries(t *testing.T) { + for _, bank := range []string{"X", "Y", "Z"} { + width := map[string]int{"X": 16, "Y": 32, "Z": 64}[bank] + row := "VPSHUFB.Z value+0(FP)," + bank + "1,K1," + bank + "2" + ins := x86ScalarLaneInstruction(t, row) + for missing := -1; missing < 64; missing++ { + for _, maskKnown := range []bool{false, true} { + state := x86ResultState{fp: make([]bool, 64)} + owned := make([]bool, 64) + for i := range owned { + owned[i], state.fp[i] = true, i != missing + } + state.writeVector(1, 64, ^uint64(0), false) + state.masks[1] = maskKnown + accepted := x86ResultTransfer(ins, &state, owned, 64, "amd64") + want := maskKnown && (missing < 0 || missing >= width) + if accepted != want || accepted && state.vectorBytes(2) != ^uint64(0) || len(ins.x86CallerMemory) != 0 { + t.Fatal("mask/tuple observation invented bytes or caller permission", row, missing, maskKnown, accepted) + } + } + } + } + for _, source := range []string{"(AX)", "·unknown(SB)", "value-1(FP)"} { + ins := x86ScalarLaneInstruction(t, "VPSHUFB.Z "+source+",X1,K1,X2") + state := x86ResultState{} + state.xmm[1], state.masks[1], state.pointerGP[0] = 0xffff, true, true + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("unproved memory acquired numerical authority", source) + } + } +} diff --git a/amd64_byte_shuffle_runtime_test.go b/amd64_byte_shuffle_runtime_test.go new file mode 100644 index 000000000..e2bb101d0 --- /dev/null +++ b/amd64_byte_shuffle_runtime_test.go @@ -0,0 +1,136 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This observes portable LLVM semantics, not execution of original AVX512. +// Every output byte, including wider aliases and surrounding canaries, is +// compared with an independent scalar reference over all 256 control values. +func TestCrossLinuxRuntimeMatrixX86ByteShuffleSoftware(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, reference, checks strings.Builder + reference.WriteString(x86ByteShuffleReference) + count := 0 + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for legacy := 0; legacy < 2; legacy++ { + for memory := 0; memory < 2; memory++ { + for mask := 0; mask < 3; mask++ { + for dst := 0; dst < 3; dst++ { + if legacy != 0 && (width != 16 || mask != 0 || dst != 1) { + continue + } + name := fmt.Sprint("Shuffle", count) + count++ + row, control := "VPSHUFB", bank+"0" + if legacy != 0 { + row = "PSHUFB" + } + if memory != 0 { + control = "(DI)" + } + if mask == 2 { + row += ".Z" + } + row += " " + control + "," + bank + "1" + if mask != 0 { + row += ",K1" + } + if legacy == 0 { + row += fmt.Sprintf(",%s%d", bank, dst) + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\n", name) + source.WriteString("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVQ mask+48(FP),AX\nKMOVQ AX,K1\nPUSHQ seed+40(FP)\nPOPFQ\n") + fmt.Fprintf(&source, "%s\nVMOVUPS Z%d,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", row, dst) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", name) + fmt.Fprintf(&checks, " if (check(%s,%q,%d,%d,%d,%d)) return 1;\n", name, row, width, legacy, mask, dst) + } + } + } + } + } + if count != 56 || count > 64 { + t.Fatal("incomplete or oversized runtime batch", count) + } + reference.WriteString("int main(void) {\n" + checks.String() + " return 0;\n}\n") + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "byte-shuffle", ir, reference.String(), level) + }) + } + }) + } +} + +const x86ByteShuffleReference = ` +#include +#include +#include +#include + +typedef void (*kernel)(uint8_t *, const uint8_t *, const uint8_t *, + uint64_t *, uint32_t *, uint64_t, uint64_t); + +static int check(kernel fn, const char *name, int width, int legacy, + int masking, int dst) { + _Alignas(64) uint8_t a[64], b[64], out[192], oldA[64], oldB[64]; + const uint64_t masks[] = {0, UINT64_MAX, UINT64_C(0x5555555555555555), UINT64_C(1) << 63}; + for (int sample = 0; sample < 256; ++sample) { + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(sample + i * 53); + b[i] = (uint8_t)(sample * 17 - i * 91); + } + memcpy(oldA, a, 64); + memcpy(oldB, b, 64); + for (int seedIndex = 0; seedIndex < 2; ++seedIndex) { + uint64_t seed = seedIndex ? 0x8d5 : 0; + for (unsigned mi = 0; mi < sizeof(masks)/sizeof(masks[0]); ++mi) { + uint64_t mask = masks[mi]; + struct { uint64_t before, value, after; } flags = {11, 0, 13}; + struct { uint32_t before, value, after; } csr = {17, 0, 19}; + uint32_t initialCSR = _mm_getcsr(); + memset(out, 0xa7, sizeof(out)); + fn(out + 64, a, b, &flags.value, &csr.value, seed, mask); + if ((flags.value & 0x8d5) != seed || flags.before != 11 || flags.after != 13 || + csr.value != initialCSR || csr.before != 17 || csr.after != 19 || + memcmp(a, oldA, 64) || memcmp(b, oldB, 64)) { + fprintf(stderr, "%s state/input/canary mismatch\n", name); + return 1; + } + for (int i = 0; i < 192; ++i) { + unsigned want = 0xa7; + int offset = i - 64; + if (offset >= 0 && offset < 64) { + const uint8_t *old = dst == 0 ? a : b; + want = legacy ? old[offset] : 0; + if (offset < width) { + if (!masking || ((mask >> offset) & 1)) { + want = a[offset] & 0x80 ? 0 : b[(offset / 16) * 16 + (a[offset] & 15)]; + } else { + want = masking == 2 ? 0 : old[offset]; + } + } + } + if (out[i] != want) { + fprintf(stderr, "%s sample=%d mask=%llx byte=%d got=%x want=%x\n", + name, sample, (unsigned long long)mask, i, out[i], want); + return 1; + } + } + } + } + } + return 0; +} +` diff --git a/amd64_clmul_definedness_test.go b/amd64_clmul_definedness_test.go new file mode 100644 index 000000000..7eeaa7c00 --- /dev/null +++ b/amd64_clmul_definedness_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "errors" + "fmt" + "testing" +) + +func TestX86CLMULSelectedQwordResultBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, vector := range []bool{false, true} { + if arch == "386" && vector { + continue // Stock Go rejects the four-operand source form. + } + for _, immediate := range []int{0, 1, 16, 17, 238, 239, 254, 255} { + name, tail := "PCLMULQDQ", "X0,X1" + if vector { + name, tail = "VPCLMULQDQ", "X0,X1,X2" + } + t.Run(fmt.Sprintf("%s/%s/%d", arch, name, immediate), func(t *testing.T) { + row := fmt.Sprintf("%s $%d,%s", name, immediate, tail) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + state := x86ResultState{flags: x86ByteFlagCF | x86ByteFlagZF} + state.xmm[0] = uint16(255) << uint((immediate>>4&1)*8) + state.xmm[1] = uint16(255) << uint((immediate&1)*8) + if !x86ResultTransfer(ins, &state, nil, 0, arch) { + t.Fatal("complete selected qwords lack their registered result effect") + } + destination := 1 + if vector { + destination = 2 + } + if state.xmm[destination] != 0xffff || state.flags != x86ByteFlagCF|x86ByteFlagZF { + t.Fatal("carryless product did not define exactly the full X result while preserving flags") + } + }) + } + } + } +} + +func TestX86CLMULMissingByteAndAliasEffects(t *testing.T) { + for _, bank := range []string{"X", "Y", "Z"} { + for _, legacy := range []bool{true, false} { + if legacy && bank != "X" { + continue + } + for immediate := 0; immediate < 256; immediate++ { + row := fmt.Sprintf("PCLMULQDQ $%d,X0,X1", immediate) + if !legacy { + row = fmt.Sprintf("VPCLMULQDQ $%d,%s0,%s1,%s2", immediate, bank, bank, bank) + } + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86CLMULProperties(ins.Op, ins, "amd64") + if !known || err != nil { + t.Fatal(err) + } + for missing := -1; missing < form.byteWidth; missing++ { + for source := 0; source < 2; source++ { + inputs := [2]uint64{^uint64(0), ^uint64(0)} + if missing >= 0 { + inputs[source] &^= uint64(1) << uint(missing) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": inputs[0], "Z1": inputs[1], "Z2": 1 << 60}} + if !x86CallerVectorMoveKnown(ins) || !x86CallerVectorMoveStep(&ins, &state, nil) { + t.Fatal("complete register effect is unregistered", row) + } + want := ^uint64(0) + if missing >= 0 { + selector := immediate & 1 + if source == 0 { + selector = immediate >> 4 & 1 + } + if missing%16/8 == selector { + want &^= uint64(65535) << uint(missing/16*16) + } + } + destination := Reg("Z2") + if legacy { + destination = "Z1" + } + if state.vectors[destination] != want { + t.Fatalf("%s missing %d operand %d got %016x want %016x", row, missing, source, state.vectors[destination], want) + } + result := x86ResultState{} + result.writeVector(0, 64, inputs[0], false) + result.writeVector(1, 64, inputs[1], false) + result.writeVector(2, 64, 1<<60, false) + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") { + t.Fatal("partial known bytes lost the effect registration") + } + index := 2 + if legacy { + index = 1 + } + if result.vectorBytes(index) != want { + t.Fatal("caller/result complete numerical byte selectors differ", row) + } + } + } + } + } + } +} + +func TestX86CLMULNoMemoryOrUnknownResultPermission(t *testing.T) { + for _, row := range []string{ + "PCLMULQDQ $0,(AX),X1", "PCLMULQDQ $0,·data(SB),X1", + "VPCLMULQDQ $0,(AX),X1,X2", "VPCLMULQDQ $0,(AX),Y1,Y2", "VPCLMULQDQ $0,(AX),Z1,Z2", + } { + ins := x86ScalarLaneInstruction(t, row) + state := x86SourceStackState{vectors: map[Reg]uint64{"Z1": ^uint64(0)}} + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || len(ins.x86CallerMemory) != 0 { + t.Fatal("shape or selected qword invented caller memory authority", row) + } + result := x86ResultState{pointerGP: [16]bool{true}} + result.xmm[1] = 0xffff + if x86ResultTransfer(ins, &result, nil, 0, "amd64") { + t.Fatal("pointer identity or SB spelling invented readable bytes", row) + } + } + for _, arch := range []string{"amd64", "386"} { + for _, missing := range []int{0, 1, 2, 3} { + body := "MOVL $7,AX\n" + for lane := 0; lane < 4; lane++ { + if lane != missing { + body += fmt.Sprintf("PINSRD $%d,AX,X%d\n", lane%2, lane/2) + } + } + body += "PCLMULQDQ $0,X0,X1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+0(FP)\nRET" + _, file, sig, source := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + goAssemblerOracleOutput(t, arch, source, "", true) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a partially missing selected qword acquired numerical result bytes", arch, missing, err) + } + } + } +} + +func TestX86CLMULScalarResultWholeSource(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, immediate := range []int{0, 1, 16, 17} { + t.Run(fmt.Sprintf("%s/%d", arch, immediate), func(t *testing.T) { + // Separate 32-bit insertions initialize only each selected qword; + // the opposite halves deliberately retain unknown input bytes. + body := "MOVL $7,AX\n" + for operand, selector := range []int{immediate >> 4 & 1, immediate & 1} { + for half := 0; half < 2; half++ { + body += fmt.Sprintf("PINSRD $%d,AX,X%d\n", selector*2+half, operand) + } + } + body += fmt.Sprintf("PCLMULQDQ $%d,X0,X1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+0(FP)\nRET", immediate) + _, file, sig, source := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + goAssemblerOracleOutput(t, arch, source, "", true) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); err != nil { + t.Errorf("selected qword result failed: %v", err) + } + } + }) + } + } +} diff --git a/amd64_clmul_fault_runtime_test.go b/amd64_clmul_fault_runtime_test.go new file mode 100644 index 000000000..9c528265b --- /dev/null +++ b/amd64_clmul_fault_runtime_test.go @@ -0,0 +1,95 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CLMULOriginalDeadReadFaults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, references, pairs strings.Builder + sigs := map[string]FuncSig{} + for index, immediate := range []int{0, 1, 16, 17} { + name := fmt.Sprint("clmulfault", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ p+0(FP),AX\nPXOR X1,X1\nPCLMULQDQ $%d,(AX),X1\nRET\n", name, immediate) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + code := x86OrderingHintGoBytes(t, "amd64", fmt.Sprintf("TEXT f(SB),4,$0-0\nPCLMULQDQ $%d,(DI),X1\nRET\n", immediate)) + if len(code) != 7 || code[len(code)-1] != 0xc3 { + t.Fatal("original memory opcode or its RET changed", code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&references, "extern void %s(const void*);\nstatic void original%d(const void *p) {\n", name, index) + fmt.Fprintf(&references, " __asm__ volatile(\"pxor %%%%xmm1, %%%%xmm1; .byte %s\"::\"D\"(p):\"xmm1\",\"memory\");\n}\n", strings.Join(bytes, ",")) + if index > 0 { + pairs.WriteString(",") + } + fmt.Fprintf(&pairs, "{original%d,%s}", index, name) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +` + references.String() + ` +typedef void (*operation)(const void*); +struct pair {operation original,translated;}; +static const struct pair pairs[]={ ` + pairs.String() + ` }; +static int outcome(operation fn,const void *p) { + pid_t pid=fork(); if(pid<0) return -1; + if(!pid) { + struct rlimit limit={0,0}; if(setrlimit(RLIMIT_CORE,&limit)) _exit(120); + fn(p); _exit(0); + } + int status; if(waitpid(pid,&status,0)!=pid) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) return WTERMSIG(status); + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); if(page<128) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op<4;op++) for(unsigned extent=0;extent<=32;extent++) { + const void *p=region+page-extent; + int original=outcome(pairs[op].original,p),translated=outcome(pairs[op].translated,p); + if(original<0 || original!=translated || (extent==32 && original!=0) || + (extent==0 && original!=SIGSEGV && original!=SIGBUS)) { + fprintf(stderr,"CLMUL selector=%u extent=%u original=%d translated=%d\n",op,extent,original,translated); + return 3; + } + } + for(unsigned op=0;op<4;op++) { + int original=outcome(pairs[op].original,0),translated=outcome(pairs[op].translated,0); + if(original!=translated || (original!=SIGSEGV && original!=SIGBUS)) return 4; + } + return munmap(region,2*page)!=0; +} +` + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "clmul-original-dead-faults", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_clmul_forms.go b/amd64_clmul_forms.go new file mode 100644 index 000000000..c61d5503c --- /dev/null +++ b/amd64_clmul_forms.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's yxshuf and _yvpclmulqdq rows select one qword from each 128-bit +// lane. The complete memory tuple is distinct from those numerical inputs. +type x86CLMULForm struct { + legacy bool + byteWidth, source, second int + destination, memory int + immediate uint8 +} + +func x86CLMULProperties(op Op, ins Instr, arch string) (x86CLMULForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + if name != "PCLMULQDQ" && name != "VPCLMULQDQ" { + return x86CLMULForm{}, false, nil + } + f := x86CLMULForm{legacy: name == "PCLMULQDQ", source: 1, second: 2, destination: 3, memory: -1} + if f.legacy { + f.destination = 2 + } + fail := func() (x86CLMULForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete Go carryless-multiply grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || dotted || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || + len(ins.Args) != f.destination+1 || arch == "386" && !f.legacy { + return fail() + } + constant, valid := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !valid || !amd64UnsignedImmediate(constant, 8) { + return fail() + } + f.immediate = uint8(constant.Imm) + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + if f.legacy && f.byteWidth != 16 || f.byteWidth != 16 && f.byteWidth != 32 && f.byteWidth != 64 { + return fail() + } + register := func(operand Operand) bool { + index, valid := amd64VectorRegisterIndex(operand.Reg, f.byteWidth) + limit := 32 + if f.legacy { + limit = 16 + if arch == "386" { + limit = 8 + } + } + return operand.Kind == OpReg && valid && index < limit + } + if !register(ins.Args[f.second]) || !register(ins.Args[f.destination]) { + return fail() + } + source := ins.Args[f.source] + if register(source) { + return f, true, nil + } + if !isAMD64MemoryOperand(source) || !x86PackedShiftMemoryValid(source, arch) { + return fail() + } + if source.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return f, true, err + } + if source.Mem.OffRaw != "" || source.Mem.IndexExt != "" { + return fail() + } + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return f, true, fmt.Errorf("%w: carryless multiply has no bound native address-size override", ErrProbeNeedsContext) + } + f.memory = f.source + return f, true, nil +} + +// Each product requires all bytes of exactly two selected qwords. No mask, +// unselected qword, destination alias or numerical zero supplies a missing input. +func x86CLMULDefinedBytes(f x86CLMULForm, source, second uint64) uint64 { + var known uint64 + for lane := 0; lane < f.byteWidth; lane += 16 { + sourceMask := uint64(255) << uint(lane+int(f.immediate>>4&1)*8) + secondMask := uint64(255) << uint(lane+int(f.immediate&1)*8) + if source&sourceMask == sourceMask && second&secondMask == secondMask { + known |= uint64(65535) << uint(lane) + } + } + return known +} + +func x86ResultCLMULStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + f, known, err := x86CLMULProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil { + return false, true + } + source := x86CallerVectorByteMask(int64(f.byteWidth)) + if f.memory >= 0 { + if !x86ResultRead(ins.Args[f.memory], int64(f.byteWidth), state, owned, arch) { + return false, true + } + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[f.source].Reg, f.byteWidth) + source = state.vectorBytes(index) + } + second, _ := amd64VectorRegisterIndex(ins.Args[f.second].Reg, f.byteWidth) + destination, _ := amd64VectorRegisterIndex(ins.Args[f.destination].Reg, f.byteWidth) + state.writeVector(destination, f.byteWidth, x86CLMULDefinedBytes(f, source, state.vectorBytes(second)), f.legacy) + return true, true +} + +func x86CallerCLMULStep(ins Instr, f x86CLMULForm, state *x86SourceStackState) bool { + if f.memory >= 0 { + return false // A registered byte effect grants no new native read proof. + } + source := state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] + second := state.vectors[x86CallerVectorRoot(ins.Args[f.second].Reg)] + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + width := x86CallerVectorByteMask(int64(f.byteWidth)) + known := x86CLMULDefinedBytes(f, source, second) + if f.legacy { + state.vectors[destination] = state.vectors[destination]&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_clmul_forms_test.go b/amd64_clmul_forms_test.go new file mode 100644 index 000000000..17e874bc4 --- /dev/null +++ b/amd64_clmul_forms_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86CLMULIndependentCompleteGoRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "{APCLMULQDQ, yxshuf, Pq, opBytes{0x3a, 0x44, 0}}") { + t.Fatal("complete Go legacy prefix/map/opcode/offset changed") + } + start := strings.Index(string(legacy), "var yxshuf = []ytab{") + if start < 0 { + t.Fatal("Go omitted yxshuf") + } + actual := strings.SplitN(string(legacy)[start:], "\n}", 2)[0] + actual = strings.SplitN(actual, "[]ytab{", 2)[1] + if strings.Join(strings.Fields(actual), " ") != "{Zibm_r, 2, argList{Yu8, Yxm, Yxr}}," { + t.Fatal("complete Go legacy classes/zcase/zoffset changed", actual) + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + classes := []string{ + "Yu8,Yxm,Yxr,Yxr", "Yu8,Yym,Yyr,Yyr", + "Yu8,YxmEvex,YxrEvex,YxrEvex", "Yu8,YymEvex,YyrEvex,YyrEvex", "Yu8,Yzm,Yzr,Yzr", + } + opbytes := []string{ + "avxEscape | vex128 | vex66 | vex0F3A | vexW0", "0x44", + "avxEscape | vex256 | vex66 | vex0F3A | vexW0", "0x44", + "avxEscape | evex128 | evex66 | evex0F3A | evexW0", "evexN16", "0x44", + "avxEscape | evex256 | evex66 | evex0F3A | evexW0", "evexN32", "0x44", + "avxEscape | evex512 | evex66 | evex0F3A | evexW0", "evexN64", "0x44", + } + count := 0 + for _, rule := range rules { + if rule.Opcode != "VPCLMULQDQ" { + continue + } + zcase := "Zevex_i_rm_v_r" + if count < 2 { + zcase = "Zvex_i_rm_v_r" + } + if count >= 5 || rule.RowIndex != count || rule.Ytab != "_yvpclmulqdq" || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != []int{2, 2, 3, 3, 3}[count] || + rule.EncodingOffset != []int{0, 2, 4, 7, 10}[count] || + strings.Join(rule.Classes, ",") != classes[count] || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatal("complete independent Go vector row changed", rule) + } + count++ + } + if count != 5 { + t.Fatal("incomplete vector family", count) + } +} + +func x86CLMULIndependentRows(arch string) []string { + var rows []string + limit := 16 + if arch == "386" { + limit = 8 + } + for immediate := 0; immediate < 256; immediate++ { + rows = append(rows, fmt.Sprintf("PCLMULQDQ $%d,X0,X1", immediate)) + } + for index := 0; index < limit; index++ { + rows = append(rows, fmt.Sprintf("PCLMULQDQ $17,X%d,X1", index), fmt.Sprintf("PCLMULQDQ $17,X0,X%d", index)) + } + for _, memory := range []string{"(AX)", "16(BX)(CX*2)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "PCLMULQDQ $255,"+memory+",X1") + } + if arch == "386" { + // cmd/asm classifies TYPE_CONST after mode32 signed truncation. + rows = append(rows, "PCLMULQDQ $4294967296,X0,X1", "PCLMULQDQ $4294967551,X0,X1") + return rows + } + for _, bank := range []string{"X", "Y", "Z"} { + for immediate := 0; immediate < 256; immediate++ { + rows = append(rows, fmt.Sprintf("VPCLMULQDQ $%d,%s0,%s1,%s2", immediate, bank, bank, bank)) + } + for index := 0; index < 32; index++ { + for operand := 0; operand < 3; operand++ { + registers := []string{bank + "0", bank + "1", bank + "2"} + registers[operand] = fmt.Sprintf("%s%d", bank, index) + rows = append(rows, "VPCLMULQDQ $255,"+strings.Join(registers, ",")) + } + } + for _, memory := range []string{"(AX)", "16(BX)(CX*2)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "VPCLMULQDQ $17,"+memory+","+bank+"1,"+bank+"2") + } + } + return rows +} + +func TestX86CLMULCompleteGoOperandAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + rows := x86CLMULIndependentRows(arch) + var source strings.Builder + for index, row := range rows { + fmt.Fprintf(&source, "TEXT f%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for index, fn := range file.Funcs { + ins := fn.Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86CLMULProperties(ins.Op, ins, arch) + if !known || err != nil { + t.Fatal(rows[index], err) + } + for operand := range ins.Args { + width, write, memory := x86InstructionMemoryShape(ins, operand, arch) + if memory != (operand == form.memory) || memory && (write || width != int64(form.byteWidth)) { + t.Fatal("native tuple differs from complete operand form", rows[index], operand, width, write, memory) + } + if ins.Args[operand].Kind == OpFP { + width, write, known := X86FPAccessBytes(ins, operand, arch) + if !known || write || width != int64(form.byteWidth) { + t.Fatal("FP diagnostic lost full native tuple", rows[index], width, write, known) + } + } + } + if fmt.Sprintf("%#v", ins) != before { + t.Fatal("grammar changed its original source") + } + } + t.Logf("%d complete independent Go forms accepted", len(rows)) + }) + } +} + +func TestX86CLMULInvalidGrammarAndPrivateBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + rows := []string{ + "PCLMULQDQ $-1,X0,X1", "PCLMULQDQ $256,X0,X1", "PCLMULQDQ $1.5,X0,X1", + "PCLMULQDQ $1,X16,X0", "PCLMULQDQ $1,Y0,Y1", "PCLMULQDQ $1,M0,M1", + "PCLMULQDQ.Z $1,X0,X1", "PCLMULQDQ $1,X0,(AX)", "PCLMULQDQ $1,X0", + "VPCLMULQDQ $-1,X0,X1,X2", "VPCLMULQDQ $256,X0,X1,X2", + "VPCLMULQDQ.Z $1,X0,X1,X2", "VPCLMULQDQ.BCST $1,(AX),X1,X2", + "VPCLMULQDQ $1,X0,Y1,Y2", "VPCLMULQDQ $1,X0,X1,K1,X2", + "VPCLMULQDQ $1,X0,(AX),X2", "VPCLMULQDQ $1,X0,X1,(AX)", + } + if arch == "386" { + rows = append(rows, "PCLMULQDQ $1,X8,X0", "VPCLMULQDQ $1,X0,X1,X2", "VPCLMULQDQ $1,Z0,Z1,Z2") + } + for _, row := range rows { + goAssemblerOracleOutput(t, arch, "TEXT ·invalid(SB),4,$0-0\n"+row+"\nRET\n", "", false) + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86CLMULProperties(ins.Op, ins, arch); !known || err == nil { + t.Error("unbound grammar accepted", arch, row) + } + } + } + for _, mutation := range []string{"float", "unresolved", "dispatch", "address-size", "unknown-displacement", "unknown-index"} { + ins := x86ScalarLaneInstruction(t, "PCLMULQDQ $1,(AX),X1") + op := ins.Op + switch mutation { + case "float": + ins.Args[0].ImmIsFloat = true + case "unresolved": + ins.Args[0].ImmRaw = "missing" + case "dispatch": + ins.Op = "AESENC" + case "address-size": + ins.x86AddressBits = 32 + case "unknown-displacement": + ins.Args[1].Mem.OffRaw = "missing" + case "unknown-index": + ins.Args[1].Mem.IndexExt = "unknown" + } + if _, known, err := x86CLMULProperties(op, ins, "amd64"); !known || err == nil { + t.Error("constructed source bypassed complete grammar", mutation) + } + } +} diff --git a/amd64_clmul_objects_test.go b/amd64_clmul_objects_test.go new file mode 100644 index 000000000..03cd91c16 --- /dev/null +++ b/amd64_clmul_objects_test.go @@ -0,0 +1,44 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86CLMULThreeAPIFiveTargetResultObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + for _, vector := range []bool{false, true} { + if target.arch == "386" && vector { + continue // Source mode, independently checked against Go. + } + for _, selector := range []int{0, 1, 16, 17} { + var body strings.Builder + body.WriteString("MOVL a+0(FP),AX\nMOVL b+4(FP),BX\n") + for half := 0; half < 2; half++ { + fmt.Fprintf(&body, "PINSRD $%d,AX,X0\nPINSRD $%d,BX,X1\n", (selector>>4&1)*2+half, (selector&1)*2+half) + } + if vector { + fmt.Fprintf(&body, "VPCLMULQDQ $%d,X0,X1,X2\nPEXTRD $0,X2,AX\n", selector) + } else { + fmt.Fprintf(&body, "PCLMULQDQ $%d,X0,X1\nPEXTRD $0,X1,AX\n", selector) + } + source := "TEXT ·view(SB),4,$0-12\n" + body.String() + "MOVL AX,ret+8(FP)\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b uint32) uint32\n", target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal(api, vector, selector, err) + } + compileLLVMToObject(t, llc, target.triple, "clmul-result.ll", "clmul-result.o", ir) + } + } + } + }) + } +} diff --git a/amd64_clmul_runtime_test.go b/amd64_clmul_runtime_test.go new file mode 100644 index 000000000..2d6357b83 --- /dev/null +++ b/amd64_clmul_runtime_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Polynomial long multiplication is independent of the production selectors, +// descriptors and LLVM intrinsic. Both input qwords and both output qwords are +// observed, including the otherwise ignored immediate bits. +const x86CLMULIndependentC = `#include +#include +#include +#include +static void product(uint64_t a,uint64_t b,uint64_t *out) { + uint64_t lo=0,hi=0; + for(unsigned bit=0;bit<64;bit++) { + if((b>>bit)&1) { + lo^=a<>(64-bit); + } + } + out[0]=lo; out[1]=hi; +} +static const uint64_t edges[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000), + UINT64_C(0x123456789abcdef0),UINT64_C(0xfedcba9876543210), + UINT64_C(0x1111111111111111),UINT64_C(0xaaaaaaaaaaaaaaaa)}; +struct Buffer {_Alignas(64) uint8_t before[64],value[80],after[64];}; +typedef void (*Program)(uint8_t*,const uint8_t*,const uint8_t*); +static int check(Program program,const char *name,unsigned width,int legacy,int selector,int alias) { + for(unsigned sample=0;sample<73;sample++) { + struct Buffer a,b,out; + memset(&a,0xa7,sizeof(a)); memset(&b,0xb9,sizeof(b)); memset(&out,0xc3,sizeof(out)); + for(unsigned word=0;word<8;word++) { + uint64_t av=edges[(sample+word)%8],bv=edges[(sample/8+word*3)%8]; + if(sample>=64) {av^=(uint64_t)sample*713781;bv^=(uint64_t)sample*391531;} + memcpy(a.value+word*8,&av,8); memcpy(b.value+word*8,&bv,8); + } + struct Buffer originalA=a,originalB=b,want=out; + for(unsigned lane=0;lane>4)&1)*8,8); + memcpy(&bv,b.value+lane+(selector&1)*8,8); + product(bv,av,pair); memcpy(want.value+lane,pair,16); + } + memcpy(want.value+width,legacy?b.value+width:(uint8_t[64]){0},64-width); + want.value[64]=1; want.value[65]=0; + unsigned saved=_mm_getcsr(),profile=0x1f80|((sample%4)<<13)|(sample%64); + memcpy(want.value+68,&profile,4); _mm_setcsr(profile); + program(out.value,a.value,b.value); unsigned after=_mm_getcsr(); _mm_setcsr(saved); + if(after!=profile||memcmp(&a,&originalA,sizeof(a))||memcmp(&b,&originalB,sizeof(b))||memcmp(&out,&want,sizeof(out))) { + for(unsigned i=0;i= width) { + t.Fatal("each potential byte must exist, regardless of predicate", valid, width, effect) + } + if !valid && state.known[input] != before { + t.Fatal("failed proof washed unknown bytes") + } + result := x86ResultState{flags: x86ByteFlagsAll} + result.gp[0], result.gp[3] = 0xff, 0xff + index := 0 + if input == BX { + index = 3 + } + result.gp[index] &^= 1 << uint(missing) + beforeResult := result + validResult := x86ResultTransfer(file.Funcs[0].Instrs[1], &result, nil, 0, "amd64") + if validResult != (missing >= width) || !validResult && !reflect.DeepEqual(result, beforeResult) { + t.Fatal("result observer borrowed potential undefined bytes or mutated a failed proof", validResult) + } + if validResult && width == 4 && result.gp[3] != 0xff { + t.Fatal("L false-or-true writes did not define upper zero bytes") + } + }) + } + } + } +} + +func TestX86CMOVFamilyExactRequiredFlagsAndUnconditionalFrameBytes(t *testing.T) { + for _, name := range x86CMOVOracleNames("amd64") { + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[name[4]] + for _, source := range []string{"AX", "x+0(FP)", "local-8(SP)"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$8-8\n"+name+" "+source+",BX\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + flags := uint8(0) + for _, condition := range x86CMOVOracleConditions { + if strings.HasSuffix(name, condition.name) { + flags = condition.flags + } + } + for bit := uint8(1); bit <= x86ByteFlagOF; bit <<= 1 { + state := x86ResultState{flags: x86ByteFlagsAll &^ bit, fp: make([]bool, 8), auto: make([]bool, 8)} + state.gp[0], state.gp[3] = 0xff, 0xff + owned := make([]bool, 8) + for index := 0; index < 8; index++ { + state.fp[index], state.auto[index], owned[index] = true, true, true + } + valid := x86ResultTransfer(ins, &state, owned, 8, "amd64") + if valid != (flags&bit == 0) || valid && state.flags != x86ByteFlagsAll&^bit { + t.Fatal("CMOV exact flag dependency/preservation", name, source, bit, valid) + } + } + if source == "AX" { + continue + } + for missing := 0; missing < 8; missing++ { + state := x86ResultState{flags: x86ByteFlagsAll, fp: make([]bool, 8), auto: make([]bool, 8)} + state.gp[0], state.gp[3] = 0xff, 0xff + owned := make([]bool, 8) + for index := 0; index < 8; index++ { + state.fp[index], state.auto[index], owned[index] = true, true, true + } + state.fp[missing], state.auto[missing] = false, false + if valid := x86ResultTransfer(ins, &state, owned, 8, "amd64"); valid != (missing >= width) { + t.Fatal("CMOV condition cannot suppress a missing FP/AUTO source byte", name, source, missing, valid) + } + } + } + } +} + +func TestX86CMOVFamilyCountedComparisonVersionGuards(t *testing.T) { + base := x86CallerCountedLoopScalarAssembly(false) + for _, destination := range []string{"BX", "CX", "AX"} { + t.Run(destination, func(t *testing.T) { + assembly := strings.Replace(base, "CMPL CX,AX\nJNE again", "CMPL CX,AX\nCMOVLLE R11,"+destination+"\nJNE again", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("CMOV relaxed the exact adjacent comparison/latch or retained a stale counter/bound version", destination, err) + } + }) + } +} + +func TestX86CMOVFamilyCountedNumberObserver(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-0\nCMOVLLE BX,AX\nRET\n") + if err != nil { + t.Fatal(err) + } + flags := x86CallerCountedFlags{pc: 7, op: "CMPL", known: true, equal: true} + facts := x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{ + AX: {version: 2, value: 17, known: true}, BX: {version: 3, value: 23, known: true}, + }, flags: flags, source: -1} + facts.step(8, 0, file.Funcs[0].Instrs[1], x86SourceStackState{}, nil) + if facts.flags != flags { + t.Fatal("CMOV changed existing numerical flag facts", facts.flags) + } + if _, retained := facts.regs[AX]; retained { + t.Fatal("CMOV invented an unconditional selected numerical value or retained a stale target version") + } + if !facts.regs[BX].known || facts.regs[BX].version != 3 { + t.Fatal("CMOV erased its unchanged source version") + } +} + +func TestX86CMOVFamilyIndependentResultByteEffect(t *testing.T) { + for _, row := range []string{"CMOVWCC AX,BX", "CMOVLLE AX,BX", "CMOVQPS AX,BX", "CMOVLLE AX,AX"} { + t.Run(row, func(t *testing.T) { + result := "BX" + if row == "CMOVLLE AX,AX" { + result = "AX" + } + body := "MOVQ x+0(FP),AX\nMOVQ $9,BX\nCMPQ AX,BX\n" + row + "\nMOVQ " + result + ",ret0+8(FP)\nRET" + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value(x uint64) uint64", body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); err != nil { + t.Fatal("actual-Go conditional move has no complete result byte effect", err) + } + }) + } +} + +func TestX86CMOVFamilyIndependentDeadNativeRead(t *testing.T) { + for _, row := range []string{"CMOVWCC (DI),BX", "CMOVLLE (DI),BX", "CMOVQPS (DI),BX"} { + t.Run(row, func(t *testing.T) { + source := "TEXT ·Dead(SB),4,$0-0\nMOVQ $7,AX\nMOVQ $9,BX\nCMPQ AX,BX\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + ir := x86NarrowOracleIR(t, source, "package cmov\nfunc Dead()\n", "x86_64-unknown-linux-gnu") + optimized := x86DwordProductIROptimize(t, ir) + t.Logf("original loads=%t; actual IR default retains any read=%t", strings.Contains(ir, " = load "), strings.Contains(optimized, " = load ")) + if !strings.Contains(ir, "load volatile") || !strings.Contains(optimized, "load volatile") { + t.Fatal("unconditional source read can disappear with a dead CMOV result under real IR default") + } + }) + } +} + +func TestX86CMOVFamilyExactSourceMemoryCertificates(t *testing.T) { + for _, name := range x86CMOVOracleNames("amd64") { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "CMPQ AX,AX\n"+name+" (DI),AX\nMOVL AX,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + if ins.Op != Op(name) { + continue + } + found = true + access, exists := ins.x86CallerMemory[0] + width := map[byte]int64{'W': 2, 'L': 4, 'Q': 8}[name[4]] + if !exists || access.width != width || access.write || !x86CallerMemoryCertified(ins, 0) { + t.Fatal("false condition shrank or omitted native source read", name, access) + } + for _, mutation := range []string{"opcode", "width", "direction", "offset", "destination", "source", "address-size", "segment"} { + changed, certificate := ins, access + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{0: access} + switch mutation { + case "opcode": + changed.Op = "CMOV" + Op(string(name[4])) + "CS" + if changed.Op == ins.Op { + changed.Op = "CMOV" + Op(string(name[4])) + "CC" + } + case "width": + certificate.width-- + changed.x86CallerMemory[0] = certificate + case "direction": + certificate.write = true + changed.x86CallerMemory[0] = certificate + case "offset": + changed.Args[0].Mem.Off++ + case "destination": + changed.Args[1].Reg = BX + case "source": + changed.Args[0].Mem.Base = SI + case "address-size": + changed.x86AddressBits = 32 + case "segment": + changed.Args[0].Mem.Segment = FS + } + if x86CallerMemoryCertified(changed, 0) { + t.Fatal("stale source or native extent survived", name, mutation) + } + } + } + if !found { + t.Fatal("source normalization omitted family instruction", name) + } + } +} diff --git a/amd64_cmov_fault_runtime_unix_test.go b/amd64_cmov_fault_runtime_unix_test.go new file mode 100644 index 000000000..127eb792b --- /dev/null +++ b/amd64_cmov_fault_runtime_unix_test.go @@ -0,0 +1,117 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CMOVUnconditionalFaultEnvelopes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, originals, rows strings.Builder + declarations.WriteString("package faultoracle\n") + count := 0 + for _, width := range []string{"W", "L", "Q"} { + for condition, row := range x86CMOVOracleConditions { + for _, dead := range []bool{false, true} { + fn := fmt.Sprintf("F%d", count) + count++ + name := "CMOV" + width + row.name + store, nativeStore := "MOVQ out+16(FP),DX\nMOVQ BX,(DX)\n", "MOVQ AX,(DX)\n" + if dead { + store, nativeStore = "", "" + } + fmt.Fprintf(&declarations, "func %s(p *[8]byte,seed uint64,out *uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ p+0(FP),DI\nMOVQ seed+8(FP),AX\nMOVQ $0x87654321fedcba98,BX\nPUSHQ AX\nPOPFQ\n%s (DI),BX\n%sRET\n", fn, name, store) + original := "Original" + fn + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\nMOVQ $0x87654321fedcba98,AX\nPUSHQ SI\nPOPFQ\n"+name+" (DI),AX\n"+nativeStore+"RET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint64_t,uint64_t*);\nextern void %s(const uint8_t*,uint64_t,uint64_t*);\n", fn, original) + fmt.Fprintf(&rows, "{%s,%s,%d,%d},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + reference := fmt.Sprintf(x86CMOVFaultC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fault", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov live/dead kernels %d translated/original fault outcomes %d false/live/dead tripwires 4 failures 0\n", count, count*2*7*2) + if string(output) != want { + t.Fatal("unconditional exact-width source fault disagreement", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86CMOVFaultC = ` +#include +#include +#include +#include +#include +#include +#include +#include +` + x86CMOVConditionC + ` +%s +%s +typedef void(*Program)(const uint8_t*,uint64_t,uint64_t*); +typedef struct {Program translated,original;unsigned width,condition;} Kernel; +static Kernel kernels[]={%s}; +static int observe(Program p,unsigned width,unsigned address,uint64_t flags) { + pid_t child=fork();if(child<0) return -1; + if(child==0) { + struct rlimit no_core={0,0};if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + uint8_t data[16];memset(data,0xa7,sizeof(data));const uint8_t *input=data; + if(address==1) input++; + if(address==2) input=NULL; + if(address>=3) { + long page=sysconf(_SC_PAGESIZE);if(page<=0) _exit(121); + uint8_t *memory=mmap(NULL,2*(size_t)page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(memory==MAP_FAILED) _exit(122); + memset(memory,0xa7,(size_t)page); + if(mprotect(memory+page,(size_t)page,PROT_NONE)) _exit(123); + unsigned available[]={0,width-1,width,width+1};input=memory+page-available[address-3]; + } + uint64_t out=0;p(input,flags,&out);_exit(0); + } + int status=0;if(waitpid(child,&status,0)!=child) return -2; + if(WIFEXITED(status)) return WEXITSTATUS(status)?-3:0; + if(WIFSIGNALED(status)&&(WTERMSIG(status)==SIGSEGV||WTERMSIG(status)==SIGBUS)) return 1; + return -4; +} +static void missing_read(const uint8_t*p,uint64_t f,uint64_t*out) {(void)p;(void)f;(void)out;} +static void too_wide(const uint8_t*p,uint64_t f,uint64_t*out) {(void)f;*out=*(volatile const uint64_t*)p;} +int main(void) { + unsigned count=0; + // Negative observers prove faults are real, not a forged list of outcomes. + if(observe(missing_read,8,2,0)!=0||observe(missing_read,8,4,0)!=0||observe(too_wide,2,5,0)!=1||observe(too_wide,4,5,0)!=1) return 4; + for(unsigned i=0;i=0;native--) { + int want=address==2||address==3||address==4; + int got=observe(native?k.original:k.translated,k.width,address,flags);count++; + if(got!=want) {fprintf(stderr,"kernel%%u width%%u taken%%u address%%u native%%u got%%d want%%d\n",i,k.width,taken,address,native,got,want);return 1;} + } + } + printf("cmov live/dead kernels %%zu translated/original fault outcomes %%u false/live/dead tripwires 4 failures 0\n",sizeof(kernels)/sizeof(kernels[0]),count); + return 0; +} +` diff --git a/amd64_cmov_fixed_local_runtime_test.go b/amd64_cmov_fixed_local_runtime_test.go new file mode 100644 index 000000000..57b5524d9 --- /dev/null +++ b/amd64_cmov_fixed_local_runtime_test.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CMOVFixedLocalAUTOWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + target := "linux/amd64" + triple := "x86_64-unknown-linux-gnu" + if strings.Contains(execution.triple, "apple") { + target = "darwin/amd64" + triple = "x86_64-apple-darwin" + } + var assembly, declarations, prototypes, rows, goRows strings.Builder + count := 0 + prefix := "" + if target == "darwin/amd64" { + prefix = "_" + } + for _, width := range []string{"W", "L", "Q"} { + for condition, row := range x86CMOVOracleConditions { + for _, source := range append([]string{"AX", "8(SP)"}, x86CMOVAUTOSources...) { + fn := fmt.Sprintf("F%d", count) + count++ + assembly.WriteString(x86CMOVFixedLocalAssembly(fn, "CMOV"+width+row.name, source)) + fmt.Fprintf(&declarations, "func %s(a,b uint64) uint64\n", fn) + symbol := prefix + "example.invalid/framefixture." + fn + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint64_t,uint64_t) __asm__(%s);\n", fn, runtimeCAsmLabel(symbol)) + fmt.Fprintf(&rows, "{%s,%d,%d,0},\n", fn, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + fmt.Fprintf(&goRows, "{%s,%d,%d,0},\n", fn, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + } + } + } + for mode, operation := range []string{"MOVQ tmp-8(SP),BX", "MOVQ AX,tmp-8(SP)\nMOVQ tmp-8(SP),BX", "MOVOU tmp-16(SP),X0\nMOVQ X0,BX"} { + fn := fmt.Sprintf("F%d", count) + count++ + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$16-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ AX,0(SP)\nMOVQ BX,8(SP)\n%s\nMOVQ BX,ret+16(FP)\nRET\n", fn, operation) + fmt.Fprintf(&declarations, "func %s(a,b uint64) uint64\n", fn) + symbol := prefix + "example.invalid/framefixture." + fn + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint64_t,uint64_t) __asm__(%s);\n", fn, runtimeCAsmLabel(symbol)) + fmt.Fprintf(&rows, "{%s,8,0,%d},\n", fn, mode+1) + fmt.Fprintf(&goRows, "{%s,8,0,%d},\n", fn, mode+1) + } + pkg, request, file, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), target, triple) + observer := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observer.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, bound) + if err != nil { + t.Fatal(err) + } + reference := runtimeCAsmLabelPreamble + fmt.Sprintf(x86CMOVFixedLocalWholeBytesC, prototypes.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fixed-local-auto", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov/private GP/SIMD AUTO kernels %d whole-byte outcomes %d failures 0\n", count, count*64) + if string(output) != want { + t.Fatal("actual native AUTO displacement/predicate/writeback mismatch", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + x86WordMultiplyRunOriginalGo(t, execution, "package oracle\n"+declarations.String(), assembly.String(), fmt.Sprintf(x86CMOVFixedLocalOriginalGo, goRows.String())) + t.Logf("%d independently executed actual Go ABI0 local CMOV/GP/SIMD kernels ×64 full-width input pairs; host native/Rosetta is not strict Linux evidence", count) +} + +const x86CMOVFixedLocalWholeBytesC = ` +#include +#include +` + x86CMOVConditionC + ` +%s +typedef struct {uint64_t(*fn)(uint64_t,uint64_t);unsigned width,condition,mode;} Kernel; +static Kernel kernels[]={%s}; +int main(void) { + const uint64_t edges[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000),UINT64_C(0x7fffffffffffffff),UINT64_C(0x87654321fedcba98),UINT64_C(0x80000000),UINT64_C(0x7fc123457f800001)}; + unsigned count=0; + for(unsigned k=0;k>bit)&1; + uint64_t flags=(a>63)<<7)|((((a^b)&(a^difference))>>63)<<11); + uint64_t selected=predicate(kernel.condition,flags)?a:b; + uint64_t mask=kernel.width==8?UINT64_MAX:((UINT64_C(1)<<(kernel.width*8))-1); + uint64_t want=kernel.width==2?((b&~mask)|(selected&mask)):(selected&mask); + if(kernel.mode) want=kernel.mode==1?b:a; + uint64_t got=kernel.fn(a,b); + if(got!=want) {fprintf(stderr,"kernel%%u a%%llx b%%llx got%%llx want%%llx\n",k,(unsigned long long)a,(unsigned long long)b,(unsigned long long)got,(unsigned long long)want);return 1;} + count++; + } + printf("cmov/private GP/SIMD AUTO kernels %%zu whole-byte outcomes %%u failures 0\n",sizeof(kernels)/sizeof(kernels[0]),count); + return 0; +} +` + +const x86CMOVFixedLocalOriginalGo = `package oracle +import "testing" +func TestOriginal(t *testing.T) { + kernels:=[]struct{fn func(uint64,uint64)uint64;width,condition,mode int}{%s} + edges:=[]uint64{0,1,0xffffffffffffffff,0x8000000000000000,0x7fffffffffffffff,0x87654321fedcba98,0x80000000,0x7fc123457f800001} + count:=0 + for k,kernel:=range kernels {for _,a:=range edges {for _,b:=range edges { + d:=a-b;ones:=0;for bit:=uint(0);bit<8;bit++ {ones+=int(d>>bit&1)} + cf,pf,zf,sf,of:=a>63!=0,((a^b)&(a^d))>>63!=0 + predicates:=[]bool{!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf} + selected:=b;if predicates[kernel.condition] {selected=a} + mask:=uint64(0xffffffffffffffff);if kernel.width<8 {mask=(uint64(1)< 0 { + ins, err := x86asm.Decode(code, mode) + if err != nil || ins.Len == 0 { + t.Fatal("independent actual Go object decode", err) + } + if ins.Op == x86asm.CMOVLE { + memory, ok := ins.Args[1].(x86asm.Mem) + want := int64(16) + if strings.Contains(source, "-") { + want = 8 + } + if !ok || memory.Disp != want || memory.Index != 0 || ins.MemBytes != map[string]int{"W": 2, "L": 4, "Q": 8}[width] { + t.Fatal("NAME_AUTO is frame+offset, not virtual/native zero", source, ins) + } + found++ + } + code = code[ins.Len:] + } + if found != 1 { + t.Fatal("actual Go did not emit one CMOV", found) + } + }) + } + } + } +} + +func TestX86CMOVFamilyAUTOGrammarThreeAPIFiveTargets(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, source := range x86CMOVAUTOSources { + var assembly strings.Builder + for _, name := range x86CMOVOracleNames(target.arch) { + fmt.Fprintf(&assembly, "%s %s,AX\n", name, source) + file, err := Parse(ArchAMD64, "TEXT Value(SB),4,$16-0\n"+name+" "+source+",AX\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + t.Run(target.triple+"/"+name+"/"+source, func(t *testing.T) { + if _, known, err := x86ConditionalMoveProperties(ins.Op, ins, target.arch); !known || err != nil { + t.Fatal("accepted NAME_AUTO descriptive offset is not an undefined macro", err) + } + }) + } + original := "TEXT ·Value(SB),4,$16-0\n" + assembly.String() + "RET\n" + goAssemblerOracleOutput(t, target.arch, original, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+source+"/"+api, func(t *testing.T) { + _, err := x86OrderingHintTranslateAPI(t, api, original, "package ordering\nfunc Value()\n", target.arch, target.triple) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("legal grammar is not original native storage authority", err) + } + }) + } + } + } +} + +func x86CMOVFixedLocalAssembly(name, op, source string) string { + shift, restore := "", "" + if source != "AX" && source != "8(SP)" && source != "tmp-8(SP)" && source != "tmp-(2*4)(SP)" { + shift, restore = "MOVQ SP,R11\nADDQ $-16,SP\n", "MOVQ R11,SP\n" + } + return fmt.Sprintf("TEXT ·%s(SB),4,$16-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ AX,0(SP)\nMOVQ AX,8(SP)\n%sCMPQ AX,BX\n%s %s,BX\n%sMOVQ BX,ret+16(FP)\nRET\n", name, shift, op, source, restore) +} + +func TestX86CMOVFamilyFixedLocalSourceRoleQueryNotPermission(t *testing.T) { + for _, op := range x86CMOVOracleNames("amd64") { + for _, source := range []string{"tmp(SP)", "UNKNOWN(SP)", "tmp-8(SP)", "tmp-(2*4)(SP)", "8(SP)", "(BP)", "AX", "a+0(FP)", "slot+UNKNOWN(SP)"} { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$16-24\n"+op+" "+source+",BX\nRET\n") + if err != nil { + t.Fatal(err) + } + want := source != "AX" && source != "a+0(FP)" && source != "slot+UNKNOWN(SP)" + if got := GoAMD64FixedLocalFrameBindingNeeded(file, "amd64"); got != want { + t.Errorf("typed source binding query %s %s got=%t want=%t", op, source, got, want) + } + } + } +} + +func x86CMOVFixedLocalFixture(t *testing.T, assembly, declarations, target, triple string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + pkg, request := goAMD64FrameTypedFixture(t, declarations, assembly) + request.Target = target + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, name := range pkg.Types.Scope().Names() { + sig, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[sig.Name] = sig + } + return pkg, request, file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }} +} + +func TestX86CMOVFamilyFixedLocalAUTOActualBoundWholeFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + var assembly, declarations strings.Builder + count := 0 + for _, op := range x86CMOVOracleNames("amd64") { + for _, source := range append([]string{"AX", "8(SP)"}, x86CMOVAUTOSources...) { + name := fmt.Sprintf("F%d", count) + count++ + assembly.WriteString(x86CMOVFixedLocalAssembly(name, op, source)) + fmt.Fprintf(&declarations, "func %s(a,b uint64) uint64\n", name) + } + } + for _, target := range []struct{ name, triple string }{{"linux/amd64", "x86_64-unknown-linux-gnu"}, {"darwin/amd64", "x86_64-apple-darwin"}} { + t.Run(target.name, func(t *testing.T) { + pkg, request, file, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + observer := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observer.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("actual numeric CMOV source retains the existing bounded private local contract", err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "fixed_local_native") { + t.Fatal("exact sealed original local storage", err) + } + compileLLVMToObject(t, llc, target.triple, "cmov-fixed-local.ll", "cmov-fixed-local.o", ir) + }) + t.Logf("%d complete actual-object condition/width/register/physical/AUTO kernels; NAME_AUTO source bias is not caller or saved-BP storage", count) + }) + } +} + +func TestX86CMOVFamilyFixedLocalEveryReadAndAuthorityNegatives(t *testing.T) { + for _, op := range x86CMOVOracleNames("amd64") { + base := x86CMOVFixedLocalAssembly("Value", op, "tmp(SP)") + pkg := mustGoPackage(t, "cmovlocal", "package cmovlocal\nfunc Value(a,b uint64) uint64\n") + sig := x86WordMultiplySignature(t, pkg, "Value") + width := map[byte]int{'W': 2, 'L': 4, 'Q': 8}[op[4]] + mutations := map[string]string{ + "unknown flags": strings.Replace(base, "CMPQ AX,BX", "MOVQ AX,DX", 1), + "old target pointer": strings.Replace(base, "CMPQ AX,BX", "CMPQ AX,BX\nMOVQ SP,BX", 1), + "g target": strings.Replace(base, "tmp(SP),BX", "tmp(SP),R14", 1), + "BP target": strings.Replace(base, "tmp(SP),BX", "tmp(SP),BP", 1), + "AUTO saved BP": strings.Replace(base, "ADDQ $-16,SP", "ADDQ $0,SP", 1), + "AUTO return": strings.Replace(base, "tmp(SP)", "tmp+24(SP)", 1), + "undefined expression": strings.Replace(base, "tmp(SP)", "slot+UNKNOWN(SP)", 1), + "AUTO address input": strings.Replace(base, op+" tmp(SP),BX", "MOVQ $tmp(SP),AX\n"+op+" AX,BX", 1), + "FP pointer permission": strings.Replace(base, "tmp(SP)", "p+0(FP)", 1), + "source private pointer": strings.Replace(base, "MOVQ AX,0(SP)", "MOVQ SP,0(SP)", 1), + } + for missing := 0; missing < width; missing++ { + var stores, old strings.Builder + for byteIndex := 0; byteIndex < width; byteIndex++ { + if byteIndex != missing { + fmt.Fprintf(&stores, "MOVB $1,%d(SP)\n", byteIndex) + } + } + // Old destination reads are potential even when every flag seed + // makes the predicate true. Byte-state is checked independently. + old.WriteString("MOVQ b+8(FP),DX\n") + mutations[fmt.Sprintf("source byte %d", missing)] = strings.Replace(base, "MOVQ AX,0(SP)\n", stores.String(), 1) + changed, err := Parse(ArchAMD64, strings.Replace(base, "MOVQ b+8(FP),BX\n", old.String(), 1)) + if err != nil { + t.Fatal(err) + } + t.Run(op+fmt.Sprintf("/old byte %d", missing), func(t *testing.T) { + state := x86FixedLocalState{pointers: map[Reg]int64{SP: -16}, known: map[Reg]uint8{AX: 255, BX: 255 &^ (1 << uint(missing))}, bytes: make([]bool, 16), flags: true} + for index := range state.bytes { + state.bytes[index] = true + } + for _, ins := range changed.Funcs[0].Instrs { + if ins.Op == Op(op) { + if err := fixedLocalStep(&ins, &state, sig, &x86FixedLocalProof{frame: 16}, 0); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing old byte became a defined potential input", err) + } + } + } + }) + } + for name, source := range mutations { + t.Run(op+"/"+name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("fixed local memory/flags/pointer/frame authority grew", err) + } + }) + } + } +} + +func TestX86CMOVFamilyFixedLocalSharedAUTOExistingFalseCertificate(t *testing.T) { + // This is never executed: Go's named AUTO displacement is 160+16, + // while the old private proof mistakenly certified virtual offset 16. + pkg, _, file, options := fixedLocalFrameFixture(t, "PADDQ tmp+16(SP),X0") + goAssemblerOracleOutput(t, "amd64", fixedLocalVoidAssembly(file), "", true) + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("named AUTO native out-of-frame bytes acquired an offset-zero virtual certificate", err) + } +} + +func TestX86CMOVFamilyFixedLocalSharedAUTOGPAndSIMD(t *testing.T) { + for _, operation := range []string{"MOVQ tmp-8(SP),BX", "MOVQ AX,tmp-8(SP)\nMOVQ tmp-8(SP),BX", "MOVOU tmp-16(SP),X0\nMOVQ X0,BX"} { + t.Run(operation, func(t *testing.T) { + assembly := "TEXT ·Value(SB),4,$16-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ AX,0(SP)\nMOVQ BX,8(SP)\n" + operation + "\nMOVQ BX,ret+16(FP)\nRET\n" + pkg, request, file, options := x86CMOVFixedLocalFixture(t, assembly, "func Value(a,b uint64) uint64", "linux/amd64", "x86_64-unknown-linux-gnu") + observer := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observer.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("the shared original AUTO role needs its actual native displacement", err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "fixed_local_native") { + t.Fatal("shared scalar/SIMD native offset consumer", err) + } + }) + }) + } +} diff --git a/amd64_cmov_forms.go b/amd64_cmov_forms.go new file mode 100644 index 000000000..eb175bfe7 --- /dev/null +++ b/amd64_cmov_forms.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ConditionalMoveSpec struct { + bits int + condition string +} + +// Each entry is a Go yml_rl row, not a conditional memory-access permission. +// W preserves the other GP bytes. L clears the upper half even on a false +// predicate. Both the source and the old destination are potential inputs. +var x86ConditionalMoveSpecs = map[Op]x86ConditionalMoveSpec{ + "CMOVWCC": {16, "CC"}, "CMOVWCS": {16, "CS"}, "CMOVWEQ": {16, "EQ"}, "CMOVWGE": {16, "GE"}, + "CMOVWGT": {16, "GT"}, "CMOVWHI": {16, "HI"}, "CMOVWLE": {16, "LE"}, "CMOVWLS": {16, "LS"}, + "CMOVWLT": {16, "LT"}, "CMOVWMI": {16, "MI"}, "CMOVWNE": {16, "NE"}, "CMOVWOC": {16, "OC"}, + "CMOVWOS": {16, "OS"}, "CMOVWPC": {16, "PC"}, "CMOVWPL": {16, "PL"}, "CMOVWPS": {16, "PS"}, + "CMOVLCC": {32, "CC"}, "CMOVLCS": {32, "CS"}, "CMOVLEQ": {32, "EQ"}, "CMOVLGE": {32, "GE"}, + "CMOVLGT": {32, "GT"}, "CMOVLHI": {32, "HI"}, "CMOVLLE": {32, "LE"}, "CMOVLLS": {32, "LS"}, + "CMOVLLT": {32, "LT"}, "CMOVLMI": {32, "MI"}, "CMOVLNE": {32, "NE"}, "CMOVLOC": {32, "OC"}, + "CMOVLOS": {32, "OS"}, "CMOVLPC": {32, "PC"}, "CMOVLPL": {32, "PL"}, "CMOVLPS": {32, "PS"}, + "CMOVQCC": {64, "CC"}, "CMOVQCS": {64, "CS"}, "CMOVQEQ": {64, "EQ"}, "CMOVQGE": {64, "GE"}, + "CMOVQGT": {64, "GT"}, "CMOVQHI": {64, "HI"}, "CMOVQLE": {64, "LE"}, "CMOVQLS": {64, "LS"}, + "CMOVQLT": {64, "LT"}, "CMOVQMI": {64, "MI"}, "CMOVQNE": {64, "NE"}, "CMOVQOC": {64, "OC"}, + "CMOVQOS": {64, "OS"}, "CMOVQPC": {64, "PC"}, "CMOVQPL": {64, "PL"}, "CMOVQPS": {64, "PS"}, +} + +type x86ConditionalMoveForm struct { + x86ConditionalMoveSpec + source, destination Operand +} + +func x86ConditionalMoveProperties(op Op, ins Instr, arch string) (x86ConditionalMoveForm, bool, error) { + name := strings.ToUpper(string(op)) + base, _, suffixed := strings.Cut(name, ".") + spec, known := x86ConditionalMoveSpecs[Op(base)] + if !known { + return x86ConditionalMoveForm{}, false, nil + } + fail := func(reason string) (x86ConditionalMoveForm, bool, error) { + return x86ConditionalMoveForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 || suffixed || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || len(ins.Args) != 2 { + return fail("has an unbound mode, suffix, dispatch identity or operand count") + } + source, destination := ins.Args[0], ins.Args[1] + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, arch) { + return fail("expects a Yrl destination") + } + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, arch) { + return fail("register source is outside the Go Yml class") + } + } else if source.Kind == OpMem { + if !x86ConditionalMoveOffsetKnown(source.Mem) || source.Mem.IndexExt != "" || + source.Mem.Segment != "" && source.Mem.Segment != FS && source.Mem.Segment != GS { + return fail("has an unresolved or unencodable memory source") + } + if err := x86SymbolIndexedMemoryError(source.Mem, arch); err != nil { + return x86ConditionalMoveForm{}, true, err + } + validation := source + if arch == "386" && strings.HasSuffix(source.Mem.Sym, "(SB)") { + validation.Mem.Base = "" // Go ignores Reg, not Index, for named memory. + } + if !x86PackedShiftMemoryValid(validation, arch) { + return fail("has an unencodable source address") + } + } else if !x86PackedShiftMemoryValid(source, arch) { + return fail("expects a Yml register or memory source") + } + if source.Kind == OpSym { + if address, err := parseInt(strings.TrimSpace(source.Sym)); err == nil && + address != int64(int32(address)) && !(arch == "386" && address == int64(uint32(address))) { + return fail("absolute source address is outside Go's physical disp32 extent") + } + } + return x86ConditionalMoveForm{spec, source, destination}, true, nil +} + +func x86TypedConditionalMoveByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86ConditionalMoveProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.bits / 8) + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: form.source, width: width, read: true}, + {index: 1, operand: form.destination, width: width, read: true, write: true}, + }, flagsRead: x86ByteConditionFlags(form.condition)}, true, nil +} + +func x86CallerConditionalMoveStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (bool, bool) { + form, known, err := x86ConditionalMoveProperties(ins.Op, *ins, "amd64") + if !known || err != nil || !x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(form.condition)) { + return known, false + } + width := int64(form.bits / 8) + if !x86CallerNumericRead(form.destination, width, state) { + return true, false // Neither possible old pointer bits nor missing bytes are washed away. + } + if form.source.Kind == OpMem { + if !x86CallerOrdinaryMemoryStep(ins, 0, state, proof, width, false) { + return true, false + } + } else if !x86CallerNumericRead(form.source, width, state) { + return true, false // FP pointer slots are imports only, never CMOV inputs. + } + return true, x86CallerNumericWrite(form.destination.Reg, width, state) +} diff --git a/amd64_cmov_forms_test.go b/amd64_cmov_forms_test.go new file mode 100644 index 000000000..7f83527c0 --- /dev/null +++ b/amd64_cmov_forms_test.go @@ -0,0 +1,403 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +// Independent Intel condition/opcode/flag inventory, not the production map. +var x86CMOVOracleConditions = []struct { + name string + opcode byte + flags uint8 +}{ + {"CC", 0x43, x86ByteFlagCF}, {"CS", 0x42, x86ByteFlagCF}, + {"EQ", 0x44, x86ByteFlagZF}, {"GE", 0x4d, x86ByteFlagSF | x86ByteFlagOF}, + {"GT", 0x4f, x86ByteFlagZF | x86ByteFlagSF | x86ByteFlagOF}, {"HI", 0x47, x86ByteFlagCF | x86ByteFlagZF}, + {"LE", 0x4e, x86ByteFlagZF | x86ByteFlagSF | x86ByteFlagOF}, {"LS", 0x46, x86ByteFlagCF | x86ByteFlagZF}, + {"LT", 0x4c, x86ByteFlagSF | x86ByteFlagOF}, {"MI", 0x48, x86ByteFlagSF}, + {"NE", 0x45, x86ByteFlagZF}, {"OC", 0x41, x86ByteFlagOF}, + {"OS", 0x40, x86ByteFlagOF}, {"PC", 0x4b, x86ByteFlagPF}, + {"PL", 0x49, x86ByteFlagSF}, {"PS", 0x4a, x86ByteFlagPF}, +} + +func TestX86CMOVFamilyActualGoCompletePhysicalBytesAndCaller(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, width := range []string{"W", "L", "Q"} { + if arch == "386" && width == "Q" { + continue + } + for _, condition := range x86CMOVOracleConditions { + name := "CMOV" + width + condition.name + for _, row := range []struct { + source, destination string + modrm, rex byte + }{ + {"BX", "AX", 0xc3, 0}, {"AX", "AX", 0xc0, 0}, {"(DI)", "AX", 0x07, 0}, {"R8", "R9", 0xc8, 0x45}, + } { + if arch == "386" && row.rex != 0 { + continue + } + t.Run(arch+"/"+name+"/"+row.source, func(t *testing.T) { + instruction := name + " " + row.source + "," + row.destination + actual := x86OrderingHintGoBytes(t, arch, "TEXT original(SB),4,$0-0\n"+instruction+"\nRET\n") + var want []byte + if width == "W" { + want = append(want, 0x66) + } + rex := row.rex + if width == "Q" { + if rex == 0 { + rex = 0x48 + } else { + rex |= 8 + } + } + if rex != 0 { + want = append(want, rex) + } + want = append(want, 0x0f, condition.opcode, row.modrm, 0xc3) + if !bytes.Equal(actual, want) { + t.Fatalf("independent exact Go prefix/opcode/ModRM bytes %s got %x want %x", instruction, actual, want) + } + file, err := Parse(ArchAMD64, "TEXT original(SB),4,$0-0\n"+x86DwordProductDirectives(actual[:len(actual)-1])+"\nRET\n") + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + if decoded.Instrs[1].Op != Op(name) { + t.Fatal("physical condition alias/width normalization", decoded.Instrs[1].Op, name) + } + if arch != "amd64" || row.rex != 0 { + return + } + prefix := "MOVQ $9,BX\nCMPQ AX,BX\n" + body := x86DwordProductDirectives(actual[:len(actual)-1]) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", prefix+body+"\nMOVL AX,(DX)", 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("actual Go physical source/caller effect", err) + } + }) + } + } + } + } +} + +func x86CMOVOracleNames(arch string) []string { + var names []string + for _, width := range []string{"W", "L", "Q"} { + if arch == "386" && width == "Q" { + continue + } + for _, condition := range x86CMOVOracleConditions { + names = append(names, "CMOV"+width+condition.name) + } + } + return names +} + +func TestX86CMOVFamilyCompleteIndependentEncoderRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + table := string(data) + if !regexp.MustCompile(`(?s)var yml_rl = \[\]ytab\{\s*\{Zm_r, 1, argList\{Yml, Yrl\}\},\s*\}`).MatchString(table) { + t.Fatal("complete Go conditional-move zcase/zoffset/classes changed") + } + rows := regexp.MustCompile(`\{A(CMOV[A-Z]+), ([^,]+), ([^,]+), opBytes\{([^}]+)\}\}`).FindAllStringSubmatch(table, -1) + if len(rows) != 48 || len(x86ConditionalMoveSpecs) != 48 { + t.Fatal("complete family inventory changed", len(rows), len(x86ConditionalMoveSpecs)) + } + seen := map[string]bool{} + for _, width := range []string{"W", "L", "Q"} { + for _, condition := range x86CMOVOracleConditions { + name, prefix := "CMOV"+width+condition.name, map[string]string{"W": "Pq", "L": "Pm", "Q": "Pw"}[width] + bytes := fmt.Sprintf("0x%02x", condition.opcode) + if width == "Q" { + bytes = "0x0f, " + bytes + } + want := fmt.Sprintf("{A%s, yml_rl, %s, opBytes{%s}}", name, prefix, bytes) + if !strings.Contains(table, want) { + t.Fatal("independent complete encoder attributes changed", want) + } + if _, found := x86ConditionalMoveSpecs[Op(name)]; !found { + t.Fatal("missing actual Go member", name) + } + seen[name] = true + } + } + for _, row := range rows { + if !seen[row[1]] { + t.Fatal("unaccounted actual Go family row", row) + } + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(aliases), `instructions["CMOV`) { + t.Fatal("actual Go source alias inventory changed") + } +} + +func TestX86CMOVFamilyIndependentAbsoluteAddressExtent(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86CMOVOracleNames(arch) { + for _, test := range []struct { + address string + valid bool + }{ + {"0", true}, {"-1", true}, {"2147483647", true}, {"-2147483648", true}, + {"2147483648", arch == "386"}, {"4294967295", arch == "386"}, {"4294967296", false}, {"-2147483649", false}, + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+test.address+",AX\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + _, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch) + if !known || (err == nil) != test.valid { + t.Errorf("%s %s absolute address %s valid=%t err=%v", arch, name, test.address, test.valid, err) + } + } + } + for _, address := range []string{"0", "-1", "2147483647", "-2147483648", "2147483648", "4294967295", "4294967296", "-2147483649"} { + valid := address != "4294967296" && address != "-2147483649" && (arch == "386" || address != "2147483648" && address != "4294967295") + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\nCMOVLLE "+address+",AX\nRET\n", "", valid) + } + } +} + +func TestX86CMOVFamilyWholeFormatsEffectsAndShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + registers := strings.Fields("AX BX CX DX SP BP SI DI") + if arch == "amd64" { + registers = append(registers, strings.Fields("R8 R9 R10 R11 R12 R13 R14 R15")...) + } + sources := append(append([]string(nil), registers...), "(AX)", "-1(AX)(CX*2)", "127(AX)(DX*8)", "value+0(FP)", "local-8(SP)", "·data(SB)", "(FS)", "(GS)") + var assembly strings.Builder + count := 0 + for _, name := range x86CMOVOracleNames(arch) { + for _, source := range sources { + row := name + " " + source + ",AX" + fmt.Fprintf(&assembly, "TEXT f%d(SB),4,$16-8\n%s\nRET\n", count, row) + count++ + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$16-8\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch) + if !known || err != nil { + t.Fatal(arch, row, err) + } + width := int64(form.bits / 8) + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + flags := uint8(0) + for _, condition := range x86CMOVOracleConditions { + if strings.HasSuffix(name, condition.name) { + flags = condition.flags + } + } + if !known || err != nil || effect.flagsRead != flags || effect.flagsDefined != 0 || effect.flagsKilled != 0 || effect.dynamicMemory || + len(effect.operands) != 2 || effect.operands[0].index != 0 || effect.operands[1].index != 1 || + !effect.operands[0].read || effect.operands[0].write || !effect.operands[1].read || !effect.operands[1].write || + effect.operands[0].width != width || effect.operands[1].width != width { + t.Fatal(row, effect, err) + } + if !x86SourceStackInstructionKnown(ins, "MOVQ", "Q", arch) { + t.Fatal("source-stack omitted typed member", row) + } + for index, arg := range ins.Args { + bytes, write, valid := x86InstructionMemoryShape(ins, index, arch) + memory := index == 0 && isAMD64MemoryOperand(arg) + if valid != memory || valid && (bytes != width || write || x86ScalarMoveMemoryBytes(ins, index, arch) != width) { + t.Fatal("source extent/direction", row, bytes, write, valid) + } + if arg.Kind == OpFP { + if bytes, write, valid := X86FPAccessBytes(ins, index, arch); !valid || write || bytes != width { + t.Fatal("FP extent disagreement", row) + } + } + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("source identity mutated", row) + } + } + // Destination Yrl is independent of the source class, including self aliases. + for _, destination := range registers { + fmt.Fprintf(&assembly, "TEXT f%d(SB),4,$0-0\n%s %s,%s\nRET\n", count, name, destination, destination) + count++ + } + } + goAssemblerOracleOutput(t, arch, assembly.String(), "", true) + t.Logf("%s independent actual-Go forms %d", arch, count) + } +} + +func TestX86CMOVFamilyDispatchAndGoRejectedForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86CMOVOracleNames("amd64") { + if arch == "386" && name[4] == 'Q' { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+name+" AX,BX\nRET\n", "", false) + continue + } + operands := []string{"$1,AX", "AX,(BX)", "AL,AX", "AH,AX", "AX,AL", "X1,AX", "K1,AX", "M0,AX", "(AX)(SP*2),BX"} + if arch == "386" { + operands = append(operands, "R8,AX", "AX,R8") + } + for _, operands := range operands { + source := "TEXT f(SB),4,$0-0\n" + name + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, known, err := x86ConditionalMoveProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], arch); !known || err == nil { + t.Fatal("Go rejection obtained an effect", arch, source, err) + } + } + } + } + for _, identity := range []Op{"", "CMOVLLE", "cmovlle", "CMOVLGE", "CMOVLLE.Z"} { + ins := Instr{Op: identity, Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}} + before := append([]Operand(nil), ins.Args...) + _, known, err := x86ConditionalMoveProperties("CMOVLLE", ins, "amd64") + valid := identity == "" || strings.EqualFold(string(identity), "CMOVLLE") + if !known || (err == nil) != valid || !reflect.DeepEqual(ins.Args, before) { + t.Fatal("dispatch identity", identity, err) + } + if !valid { + var out strings.Builder + ctx := amd64Ctx{b: &out, goarch: "amd64"} + if _, _, err := ctx.lowerConditionalMove("CMOVLLE", ins); err == nil || out.Len() != 0 { + t.Fatal("contradictory identity emitted IR") + } + } + } +} + +func TestX86CMOVFamilyThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, target := range x86OrderingHintTargets { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw=%t", target.triple, raw), func(t *testing.T) { + var body strings.Builder + for _, name := range x86CMOVOracleNames(target.arch) { + for _, source := range []string{"AX", "BX", "(DI)", "-1(DI)(SI*2)"} { + fmt.Fprintf(&body, "%s %s,BX\n", name, source) + } + } + if raw { + bytes := x86OrderingHintGoBytes(t, target.arch, "TEXT original(SB),4,$0-0\n"+body.String()+"RET\n") + body.Reset() + if bytes[len(bytes)-1] != 0xc3 { + t.Fatal("actual Go body lacks its exact trailing RET") + } + for _, value := range bytes[:len(bytes)-1] { + fmt.Fprintf(&body, "BYTE $%d\n", value) + } + } else { + for _, name := range x86CMOVOracleNames(target.arch) { + fmt.Fprintf(&body, "%s value+0(FP),BX\n%s ·data(SB),BX\n%s (FS),BX\n%s (GS),BX\n", name, name, name, name) + } + } + source := "TEXT ·Forms(SB),4,$0-8\n" + body.String() + "RET\nGLOBL ·data(SB),8,$8\nDATA ·data+0(SB)/8,$0x1020304050607080\n" + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\nfunc Forms(value uint64)\n", target.arch, target.triple) + if err != nil { + t.Fatal(api, err) + } + compileLLVMToObject(t, llc, target.triple, "cmov-"+api+".ll", "cmov-"+api+".o", ir) + } + }) + } + } +} + +func TestX86CMOVFamilyBoundFrameNumericalExtent(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, width := range []string{"W", "L", "Q"} { + if target.arch == "386" && width == "Q" { + continue + } + for _, test := range []struct { + declaration, field string + valid bool + }{ + {"func Value(x uint64)", "x+0(FP)", true}, + {"func Value(x uint64)", "x+1(FP)", width != "Q"}, + {"func Value(x uint8,y uint8)", "x+0(FP)", width == "W"}, + {"func Value(x uint8,y uint32)", "x+0(FP)", false}, + {"func Value(x float64)", "x+0(FP)", true}, + {"func Value(x *uint64)", "x+0(FP)", false}, + {"func Value()", "ret+0(FP)", false}, + } { + for _, api := range []string{"text", "module", "Go"} { + size := 8 + if test.declaration == "func Value(x uint8,y uint8)" { + size = 2 + } + if test.declaration == "func Value(x *uint64)" && target.arch == "386" { + size = 4 + } + if test.declaration == "func Value()" { + size = 0 + } + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\nCMOV%sLE %s,AX\nRET\n", size, width, test.field) + _, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\n"+test.declaration+"\n", target.arch, target.triple) + if test.valid && err != nil || !test.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("numerical FP bit extent/hole/type permission", target.triple, width, test.declaration, api, err) + } + } + } + } + } +} + +func TestX86CMOVFamilyNativeStackAndFPAuthorityUnchanged(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, operand := range []string{"(SP)", "(BP)", "value+7(FP)", "missing+8(FP)", "p+0(FP)"} { + t.Run(target.triple+"/"+operand, func(t *testing.T) { + declaration := "package ordering\nfunc Value(value uint64)\n" + size, width := 8, "Q" + if target.arch == "386" { + width = "L" + } + if operand == "p+0(FP)" { + declaration = "package ordering\nfunc Value(p *uint64)\n" + if target.arch == "386" { + size = 4 + } + } + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\nCMOV%sLE %s,AX\nRET\n", size, width, operand) + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + _, err := x86OrderingHintTranslateAPI(t, api, source, declaration, target.arch, target.triple) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("new native-stack, pointer or missing-FP input authority", api, err) + } + } + }) + } + } +} diff --git a/amd64_cmov_fp_bitview_runtime_test.go b/amd64_cmov_fp_bitview_runtime_test.go new file mode 100644 index 000000000..3c1b5db64 --- /dev/null +++ b/amd64_cmov_fp_bitview_runtime_test.go @@ -0,0 +1,150 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The native oracle contains actual Go bytes and receives raw IEEE bits by +// MOVSS/MOVSD only. Its predicates and byte reference are independent of the +// typed grammar; signaling NaNs never undergo arithmetic or conversion. +func TestCrossLinuxRuntimeMatrixX86CMOVFPBitViewsWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, originals, rows, goRows strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, typ := range []string{"float32", "float64"} { + for _, width := range []string{"W", "L", "Q"} { + if typ == "float32" && width == "Q" { + continue + } + for condition, row := range x86CMOVOracleConditions { + fn := fmt.Sprintf("F%d", count) + count++ + name := "CMOV" + width + row.name + fmt.Fprintf(&declarations, "func %s(out *[16]byte,f %s,old,seed uint64)\n", fn, typ) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ old+16(FP),BX\nPUSHQ seed+24(FP)\nPOPFQ\n%s f+8(FP),BX\nMOVQ BX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nRET\n", fn, name) + move, ctype, size := "MOVSS", "float", 32 + if typ == "float64" { + move, ctype, size = "MOVSD", "double", 64 + } + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\nPUSHQ BX\nSUBQ $8,SP\n"+move+" X0,(SP)\nMOVQ SI,BX\nPUSHQ DX\nPOPFQ\n"+name+" (SP),BX\nMOVQ BX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nADDQ $8,SP\nPOPQ BX\nRET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + original, symbol := "Original"+fn, "Original"+fn + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,%s,uint64_t,uint64_t);\n", fn, ctype) + prototypes.WriteString(x86OriginalSysVDeclaration("void", original, "uint8_t*,"+ctype+",uint64_t,uint64_t")) + if size == 32 { + fmt.Fprintf(&rows, "{%s,%s,0,0,%d,%d,32},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + fmt.Fprintf(&goRows, "{%s,nil,%d,%d,32},\n", fn, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + } else { + fmt.Fprintf(&rows, "{0,0,%s,%s,%d,%d,64},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + fmt.Fprintf(&goRows, "{nil,%s,%d,%d,64},\n", fn, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition) + } + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + if x86CMOVFPHasNumericConversion(ir) { + t.Fatal("CMOV floating bit views numerically converted") + } + reference := fmt.Sprintf(x86CMOVFPWholeBytesC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fp-whole-bytes", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov FP kernels %d translated/original whole-byte outcomes %d NaN/zero/flags/mxcsr/canary failures 0\n", count, count*64*8*2) + if string(output) != want { + t.Fatal("independent FP payload/predicate mismatch", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), fmt.Sprintf(x86CMOVFPOriginalGo, goRows.String())) + t.Logf("%d actual original Go ABI0 floating-source kernels ×64 flag seeds ×8 IEEE edge payloads; native host execution is not strict native Linux evidence", count) +} + +var x86CMOVFPWholeBytesC = ` +#include +#include +#include +#include +` + x86CMOVConditionC + + x86OriginalSysVPointer("void", "OriginalProgram32", "uint8_t*,float,uint64_t,uint64_t") + + x86OriginalSysVPointer("void", "OriginalProgram64", "uint8_t*,double,uint64_t,uint64_t") + ` +%s +%s +typedef void(*Program32)(uint8_t*,float,uint64_t,uint64_t); +typedef void(*Program64)(uint8_t*,double,uint64_t,uint64_t); +typedef struct {Program32 f32;OriginalProgram32 o32;Program64 f64;OriginalProgram64 o64;unsigned width,condition,bits;} Kernel; +static Kernel kernels[]={%s}; +int main(void) { + const uint32_t bits32[]={0,0x80000000,0x7fc00001,0x7f800001,0xffc12345,0x7f800000,0xffffffff,0x00000001}; + const uint64_t bits64[]={0,UINT64_C(0x8000000000000000),UINT64_C(0x7ff8000000000001),UINT64_C(0x7ff0000000000001),UINT64_C(0xfff8123456789abc),UINT64_C(0x7ff0000000000000),UINT64_MAX,1}; + unsigned count=0; + for(unsigned k=0;k>1&1)<<2)|((f>>2&1)<<4)|((f>>3&1)<<6)|((f>>4&1)<<7)|((f>>5&1)<<11) + cf,pf,zf,sf,of:=flags&1!=0,flags&4!=0,flags&64!=0,flags&128!=0,flags&2048!=0 + predicates:=[]bool{!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf} + source:=bits64[v];if kernel.bits==32 {source=uint64(bits32[v])} + old:=uint64(0x87654321fedcba98)^(uint64(v)<<40);selected:=old;if predicates[kernel.condition] {selected=source} + mask:=uint64(0xffffffffffffffff);if kernel.width<8 {mask=(uint64(1)<>16"}, + {"f float64,old uint64", "W", "f+6(FP)", "double,uint64_t", "d,old", "math.Float64frombits(q),old", "q>>48"}, + {"f float64,old uint64", "L", "f+4(FP)", "double,uint64_t", "d,old", "math.Float64frombits(q),old", "q>>32"}, + {"f,g float32,old uint64", "Q", "f+0(FP)", "float,float,uint64_t", "f,g,old", "math.Float32frombits(a),math.Float32frombits(b),old", "uint64_t(a)|(uint64_t(b)<<32)"}, + {"f,g float32,old uint64", "L", "f+2(FP)", "float,float,uint64_t", "f,g,old", "math.Float32frombits(a),math.Float32frombits(b),old", "(uint64_t(a)|(uint64_t(b)<<32))>>16"}, + {"f float32,g uint32,old uint64", "Q", "f+0(FP)", "float,uint32_t,uint64_t", "f,b,old", "math.Float32frombits(a),b,old", "uint64_t(a)|(uint64_t(b)<<32)"}, + } { + fn := fmt.Sprintf("S%d", index) + fmt.Fprintf(&declarations, "func %s(%s) uint64\n", fn, row.declaration) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ old+8(FP),AX\nCLC\nCMOV%sCC %s,AX\nMOVQ AX,ret+16(FP)\nRET\n", fn, row.width, row.source) + fmt.Fprintf(&prototypes, "extern uint64_t %s(%s);\n", fn, row.ctype) + raw := strings.ReplaceAll(row.raw, "uint64_t(", "uint64(") + craw := strings.ReplaceAll(row.raw, "uint64_t(", "(uint64_t)(") + want, goWant := "("+craw+")", "("+raw+")" + if row.width == "W" { + want, goWant = "((old&~UINT64_C(0xffff))|(("+craw+")&0xffff))", "((old&^uint64(0xffff))|(("+raw+")&0xffff))" + } else if row.width == "L" { + want, goWant = "(("+craw+")&UINT64_C(0xffffffff))", "(("+raw+")&uint64(0xffffffff))" + } + fmt.Fprintf(&checks, "if(%s(%s)!=%s) {fprintf(stderr,\"span%d mismatch\\n\");return 1;} count++;\n", fn, row.call, want, index) + fmt.Fprintf(&goChecks, "if got,want:=%s(%s),%s;got!=want {t.Fatalf(\"span%d got=%%x want=%%x\",got,want)};count++\n", fn, row.goCall, goWant, index) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + if x86CMOVFPHasNumericConversion(ir) { + t.Fatal("partial/cross-slot bits numerically converted") + } + reference := fmt.Sprintf(x86CMOVFPSpanC, prototypes.String(), checks.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-fp-spans", pipeline.ir, reference, pipeline.backend) + if string(output) != "cmov FP partial/cross-slot whole-byte outcomes 384 failures 0\n" { + t.Fatal("independent exact FP bytes", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), fmt.Sprintf(x86CMOVFPSpanGo, goChecks.String())) +} + +const x86CMOVFPSpanC = ` +#include +#include +#include +%s +int main(void) { + const uint32_t bits[]={0,0x80000000,0x7fc00001,0x7f800001,0xffc12345,0x7f800000,0xffffffff,1}; + const uint64_t wide[]={0,UINT64_C(0x8000000000000000),UINT64_C(0x7ff8000000000001),UINT64_C(0x7ff0000000000001),UINT64_C(0xfff8123456789abc),UINT64_C(0x7ff0000000000000),UINT64_MAX,1}; + unsigned count=0; + for(unsigned x=0;x<8;x++) for(unsigned y=0;y<8;y++) { + uint32_t a=bits[x],b=bits[y];uint64_t q=wide[x],old=UINT64_C(0x87654321fedcba98)^((uint64_t)y<<40); + float f,g;double d;memcpy(&f,&a,4);memcpy(&g,&b,4);memcpy(&d,&q,8); + %s + } + printf("cmov FP partial/cross-slot whole-byte outcomes %%u failures 0\n",count);return 0; +} +` + +const x86CMOVFPSpanGo = `package oracle +import("math";"testing") +func TestOriginal(t *testing.T) { + bits:=[]uint32{0,0x80000000,0x7fc00001,0x7f800001,0xffc12345,0x7f800000,0xffffffff,1} + wide:=[]uint64{0,0x8000000000000000,0x7ff8000000000001,0x7ff0000000000001,0xfff8123456789abc,0x7ff0000000000000,0xffffffffffffffff,1} + count:=0 + for x,a:=range bits {for y,b:=range bits {q:=wide[x];old:=uint64(0x87654321fedcba98)^(uint64(y)<<40) + %s + }} + if count!=384 {t.Fatal(count)};t.Logf("actual Go partial/cross-slot outcomes %%d failures 0",count) +} +` diff --git a/amd64_cmov_runtime_test.go b/amd64_cmov_runtime_test.go new file mode 100644 index 000000000..423467102 --- /dev/null +++ b/amd64_cmov_runtime_test.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Actual Go-emitted bytes are embedded in independent native wrappers; none +// of the C arithmetic or condition reference is derived from the typed forms. +func TestCrossLinuxRuntimeMatrixX86CMOVIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, originals, prototypes, rows strings.Builder + declarations.WriteString("package narroworacle\n") + count := 0 + for _, width := range []string{"W", "L", "Q"} { + for condition, row := range x86CMOVOracleConditions { + name := "CMOV" + width + row.name + for mode, source := range []string{"BX", "(SI)", "AX"} { + fn := fmt.Sprintf("C%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[16]byte,src *[8]byte,old,seed uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ src+8(FP),SI\nMOVQ old+16(FP),AX\nMOVQ (SI),BX\nPUSHQ seed+24(FP)\nPOPFQ\n%s %s,AX\nMOVQ AX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nRET\n", fn, name, source) + original := "Original" + fn + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\nPUSHQ BX\nMOVQ DX,AX\nMOVQ (SI),BX\nPUSHQ CX\nPOPFQ\n"+name+" "+source+",AX\nMOVQ AX,(DI)\nPUSHFQ\nPOPQ AX\nMOVQ AX,8(DI)\nPOPQ BX\nRET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,uint64_t,uint64_t);\n", fn) + prototypes.WriteString(x86OriginalSysVDeclaration("void", original, "uint8_t*,const uint8_t*,uint64_t,uint64_t")) + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d},\n", fn, original, map[string]int{"W": 2, "L": 4, "Q": 8}[width], condition, mode) + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + reference := fmt.Sprintf(x86CMOVWholeBytesC, prototypes.String(), originals.String(), rows.String()) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmov-whole-bytes", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("cmov kernels %d translated/original whole-byte outcomes %d flags/mxcsr/input/canary failures 0\n", count, count*64*3*2) + if string(output) != want { + t.Fatal("independent full bytes, flags and native predicate mismatch", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86CMOVConditionC = ` +static int predicate(unsigned c,uint64_t f) { + int cf=(f>>0)&1,pf=(f>>2)&1,zf=(f>>6)&1,sf=(f>>7)&1,of=(f>>11)&1; + switch(c) { + case 0:return !cf;case 1:return cf;case 2:return zf;case 3:return sf==of; + case 4:return !zf&&sf==of;case 5:return !cf&&!zf;case 6:return zf||sf!=of; + case 7:return cf||zf;case 8:return sf!=of;case 9:return sf;case 10:return !zf; + case 11:return !of;case 12:return of;case 13:return !pf;case 14:return !sf;case 15:return pf; + }return -1; +} +static uint64_t seed(unsigned n) { + return ((uint64_t)(n&1))|((uint64_t)((n>>1)&1)<<2)|((uint64_t)((n>>2)&1)<<4)| + ((uint64_t)((n>>3)&1)<<6)|((uint64_t)((n>>4)&1)<<7)|((uint64_t)((n>>5)&1)<<11); +} +` + +var x86CMOVWholeBytesC = ` +#include +#include +#include +#include +` + x86CMOVConditionC + + x86OriginalSysVPointer("void", "OriginalProgram", "uint8_t*,const uint8_t*,uint64_t,uint64_t") + ` +%s +%s +typedef void(*Program)(uint8_t*,const uint8_t*,uint64_t,uint64_t); +typedef struct {Program translated;OriginalProgram original;unsigned width,condition,mode;} Kernel; +static Kernel kernels[]={%s}; +struct Guard {uint8_t before[32],value[16],after[32];}; +int main(void) { + unsigned count=0; + const uint64_t edges[]={UINT64_C(0x87654321fedcba98),UINT64_C(0xffff000080000001),UINT64_C(0x0123456789abcdef)}; + for(unsigned k=0;k+0(SB)/4,$0x1f80 +GLOBL cmpxchgCSRseed<>(SB),8,$4 +TEXT ·resetCSR(SB),4,$0-0 +LDMXCSR cmpxchgCSRseed<>(SB) +RET +TEXT ·restoreCSR(SB),4,$0-4 +LDMXCSR value+0(FP) +RET +` + +func x86CMPXCHGCSRControlObject(t *testing.T) { + t.Helper() + directory := t.TempDir() + source, object := filepath.Join(directory, "control.s"), filepath.Join(directory, "control.o") + if err := os.WriteFile(source, []byte(x86CMPXCHGCSRControlAssembly), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-S", + "-p", "ordering", "-o", object, source) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64", "GOFLAGS=", "GOENV=off", "GOWORK=off") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(source, object, output, err, ctx.Err(), true, "amd64"); failure != nil { + t.Fatalf("actual CSR control object: %v\n%s", failure, output) + } + if !strings.Contains(string(output), "0f ae 15 00 00 00 00 c3") || + !strings.Contains(string(output), "0f ae 54 24 08 c3") || + !strings.Contains(string(output), "80 1f 00 00") || strings.Count(string(output), "LDMXCSR") != 2 { + t.Fatalf("actual DATA source/load/restore bytes changed:\n%s", output) + } + bytes, err := os.ReadFile(object) + if err != nil { + t.Fatal(err) + } + t.Logf("actual source/Go CSR object SHA256 %x/%x; complete listing:\n%s", sha256.Sum256([]byte(x86CMPXCHGCSRControlAssembly)), sha256.Sum256(bytes), output) +} + +func x86CMPXCHGObserverReplace(t *testing.T, source, from, to string) string { + t.Helper() + if strings.Count(source, from) != 1 { + t.Fatalf("independent observer seam disappeared or became ambiguous: %q", from) + } + return strings.Replace(source, from, to, 1) +} + +// The exact assembly and strict mathematical drivers are not changed. These +// diagnostic drivers only emit every actual observation and separate canary/ +// CSR transport failure from the already-preserved mathematical deviation. +func x86CMPXCHGObserveGoDriver(t *testing.T, source string) string { + t.Helper() + source = x86CMPXCHGObserverReplace(t, source, "observations, failures := 0, 0", "observations, failures, transportFailures := 0, 0, 0") + source = x86CMPXCHGObserverReplace(t, source, "if failures < 24 {", "if false && failures < 24 {") + source = x86CMPXCHGObserverReplace(t, source, "if kernelFailures != 0 {", "if false && kernelFailures != 0 {") + source = x86CMPXCHGObserverReplace(t, source, " before := csr()", " originalCSR := csr()\n resetCSR()\n before := csr()") + source = x86CMPXCHGObserverReplace(t, source, " after := csr()", " after := csr()\n restoreCSR(originalCSR)\n restoredCSR := csr()") + source = x86CMPXCHGObserverReplace(t, source, " observations++", ` transportBad := out[6]>>32 != canary>>32 || csr != before || after != before || restoredCSR != originalCSR || raw != expectedRaw + for i := 7; i < len(out); i++ { transportBad = transportBad || out[i] != canary } + if transportBad { transportFailures++ } + fmt.Printf("OBS %d %d %d %d", index, observations%48/4, map[bool]int{false:0, true:1}[equal], observations%2) + for i, value := range out { + if i == 4 { value &= 0x8d5 } + fmt.Printf(" %x", value) + } + fmt.Printf(" %x %x %x %x %x\n", before, after, originalCSR, restoredCSR, raw) + observations++`) + source = x86CMPXCHGObserverReplace(t, source, + ` fmt.Printf("original CMPXCHG numerical observations %d failures %d\n", observations, failures) + if failures != 0 { os.Exit(1) }`, + ` fmt.Printf("END %d %d %d\n", observations, failures, transportFailures) + if transportFailures != 0 { os.Exit(1) }`) + return source +} + +func x86CMPXCHGObserveCDriver(t *testing.T, source string) string { + t.Helper() + source = x86CMPXCHGObserverReplace(t, source, "unsigned observations = 0, failures = 0;", "unsigned observations = 0, failures = 0, transport_failures = 0;") + source = x86CMPXCHGObserverReplace(t, source, "if (kernel_failures <= 2) {", "if (0 && kernel_failures <= 2) {") + source = x86CMPXCHGObserverReplace(t, source, "if (kernel_failures) {", "if (0 && kernel_failures) {") + source = x86CMPXCHGObserverReplace(t, source, ` __asm__ volatile("stmxcsr %0" : "=m"(before));`, + ` unsigned original_csr, controlled_csr = 0x1f80; + __asm__ volatile("stmxcsr %0" : "=m"(original_csr)); + __asm__ volatile("ldmxcsr %0" : : "m"(controlled_csr)); + __asm__ volatile("stmxcsr %0" : "=m"(before));`) + source = x86CMPXCHGObserverReplace(t, source, ` __asm__ volatile("stmxcsr %0" : "=m"(after));`, + ` __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(original_csr)); + unsigned restored_csr; + __asm__ volatile("stmxcsr %0" : "=m"(restored_csr));`) + source = x86CMPXCHGObserverReplace(t, source, " observations++;", ` int transport_bad = (out[6]>>32) != (canary>>32) || + (uint32_t)out[6] != before || after != before || restored_csr != original_csr || memcmp(raw, expected, sizeof(raw)); + for (unsigned i = 7; i < 16; i++) transport_bad |= out[i] != canary; + transport_failures += transport_bad != 0; + printf("OBS %u %u %u %u", index, edge, equal, seed); + for (unsigned i = 0; i < 16; i++) { + uint64_t value = i == 4 ? out[i]&0x8d5 : out[i]; + printf(" %llx", (unsigned long long)value); + } + printf(" %x %x %x %x ", before, after, original_csr, restored_csr); + for (unsigned i = 0; i < sizeof(raw); i++) printf("%02x", raw[i]); + printf("\n"); + observations++;`) + source = x86CMPXCHGObserverReplace(t, source, + ` printf("actual C opcode CMPXCHG numerical observations %u failures %u\n", observations, failures); + return failures ? 1 : 0;`, + ` printf("END %u %u %u\n", observations, failures, transport_failures); + return transport_failures ? 1 : 0;`) + return source +} + +func x86CMPXCHGParseActualPacket(output []byte) (x86CMPXCHGActualPacket, error) { + var packet x86CMPXCHGActualPacket + lines := strings.Split(string(output), "\n") + if len(output) > 1<<20 || len(lines) != 578 || lines[577] != "" { + return packet, fmt.Errorf("incomplete, extra or over-bound actual observation output") + } + parse := func(text string, base, bits int) (uint64, error) { + value, err := strconv.ParseUint(text, base, bits) + if err != nil || strconv.FormatUint(value, base) != text { + return 0, fmt.Errorf("noncanonical actual value %q", text) + } + return value, nil + } + for ordinal, line := range lines[:576] { + fields := strings.Split(line, " ") + if len(fields) != 26 || fields[0] != "OBS" { + return packet, fmt.Errorf("actual observation %d has incomplete/unknown fields", ordinal) + } + want := []int{ordinal / 48, ordinal % 48 / 4, ordinal % 4 / 2, ordinal % 2} + for index, value := range want { + if fields[index+1] != strconv.Itoa(value) { + return packet, fmt.Errorf("duplicate, missing or reordered actual observation %d", ordinal) + } + } + row := &packet.rows[ordinal] + for index := range row.output { + value, err := parse(fields[index+5], 16, 64) + if err != nil { + return packet, err + } + row.output[index] = value + } + before, err := parse(fields[21], 16, 32) + if err != nil { + return packet, err + } + after, err := parse(fields[22], 16, 32) + if err != nil || before != 0x1f80 || before != after || uint32(row.output[6]) != uint32(before) { + return packet, fmt.Errorf("actual MXCSR transport changed at observation %d", ordinal) + } + row.csr = uint32(before) + // Ambient input differs between independent Go/C processes. Preserve + // both complete values in the raw trace and require exact restoration; + // opcode equality concerns the identical controlled MXCSR input. + originalCSR, err := parse(fields[23], 16, 32) + if err != nil { + return packet, err + } + restoredCSR, err := parse(fields[24], 16, 32) + if err != nil || originalCSR != restoredCSR { + return packet, fmt.Errorf("actual ambient MXCSR restoration failed at observation %d", ordinal) + } + if row.output[4]&^uint64(0x8d5) != 0 || row.output[5] != row.output[4]&0xd5|2 || row.output[6]>>32 != 0xa7a7a7a7 { + return packet, fmt.Errorf("actual status/LAHF or output canary was incomplete at observation %d", ordinal) + } + for index := 7; index < 16; index++ { + if row.output[index] != 0xa7a7a7a7a7a7a7a7 { + return packet, fmt.Errorf("actual output canary changed at observation %d", ordinal) + } + } + if len(fields[25]) != 192 || strings.ToLower(fields[25]) != fields[25] { + return packet, fmt.Errorf("actual complete byte observation missing at %d", ordinal) + } + bytes, err := hex.DecodeString(fields[25]) + if err != nil { + return packet, err + } + copy(row.bytes[:], bytes) + family := ordinal / 96 + start := 17 + if family == 5 { + start = 16 + } + width := []int{1, 2, 4, 8, 8, 16}[family] + for index, value := range row.bytes { + if (index < start || index >= start+width) && value != 0xa7 { + return packet, fmt.Errorf("actual input canary changed at observation %d", ordinal) + } + } + } + footer := strings.Split(lines[576], " ") + if len(footer) != 4 || footer[0] != "END" || footer[1] != "576" || footer[3] != "0" { + return packet, fmt.Errorf("actual observer has missing/unknown/nonzero transport terminal") + } + deviations, err := parse(footer[2], 10, 16) + if err != nil || deviations > 576 { + return packet, fmt.Errorf("actual observer has invalid mathematical deviation count") + } + packet.deviations = int(deviations) + return packet, nil +} + +func TestX86CMPXCHGActualObservationPacketRejectsIncompleteTransport(t *testing.T) { + // Synthetic output-protocol tripwires, not actual opcode execution evidence. + var source strings.Builder + for ordinal := 0; ordinal < 576; ordinal++ { + fmt.Fprintf(&source, "OBS %d %d %d %d", ordinal/48, ordinal%48/4, ordinal%4/2, ordinal%2) + for index := 0; index < 16; index++ { + value := uint64(0xa7a7a7a7a7a7a7a7) + switch index { + case 4: + value = 64 + case 5: + value = 66 + case 6: + value = 0xa7a7a7a700001f80 + } + fmt.Fprintf(&source, " %x", value) + } + fmt.Fprintf(&source, " 1f80 1f80 1f80 1f80 %s\n", strings.Repeat("a7", 96)) + } + source.WriteString("END 576 0 0\n") + valid := source.String() + packet, err := x86CMPXCHGParseActualPacket([]byte(valid)) + if err != nil || packet.deviations != 0 { + t.Fatalf("complete synthetic protocol control: %v", err) + } + for name, mutation := range map[string]string{ + "missing": strings.TrimSuffix(valid, "END 576 0 0\n"), + "unknown": valid + "unknown\n", + "duplicate": strings.Replace(valid, "OBS 0 0 0 1", "OBS 0 0 0 0", 1), + "truncated": strings.Replace(valid, strings.Repeat("a7", 96), "a7", 1), + "canary": strings.Replace(valid, strings.Repeat("a7", 96), "a6"+strings.Repeat("a7", 95), 1), + "changed-CSR": strings.Replace(valid, "1f80 1f80", "1f80 1f81", 1), + "uncontrolled": strings.Replace(valid, "1f80 1f80", "1fa0 1fa0", 1), + "unrestored": strings.Replace(valid, "1f80 1f80 1f80 1f80", "1f80 1f80 1fa0 1f80", 1), + "unknown-LAHF": strings.Replace(valid, " 40 42 ", " 40 43 ", 1), + "transport-exit": strings.Replace(valid, "END 576 0 0", "END 576 0 1", 1), + "wrong-count": strings.Replace(valid, "END 576", "END 575", 1), + "overflow": strings.Replace(valid, "END 576 0 0", "END 576 65536 0", 1), + "diagnostic-only": "END 576 240 0\n", + } { + t.Run(name, func(t *testing.T) { + if _, err := x86CMPXCHGParseActualPacket([]byte(mutation)); err == nil { + t.Fatal("incomplete/unknown actual transport got equivalence credit") + } + }) + } + mutated := packet + mutated.rows[575].output[0] ^= 1 + if reflect.DeepEqual(packet, mutated) { + t.Fatal("full observation comparison did not retain a late register byte") + } +} diff --git a/amd64_cmpxchg_boundaries_test.go b/amd64_cmpxchg_boundaries_test.go new file mode 100644 index 000000000..b1f281767 --- /dev/null +++ b/amd64_cmpxchg_boundaries_test.go @@ -0,0 +1,96 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86CMPXCHGDirectHelpersCannotBypassOrderedGrammar(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + op Op + identity Op + bits int + pair bool + args []Operand + context bool + }{ + {"CMPXCHG8B", "CMPXCHG8B", 64, true, []Operand{{Kind: OpReg, Reg: BX}}, true}, + {"CMPXCHGL", "MOVL", 32, false, []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: BX}}, false}, + {"CMPXCHGL", "CMPXCHGL", 16, false, []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: BX}}, false}, + {"UNKNOWN", "UNKNOWN", 32, false, []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: BX}}, false}, + } { + t.Run(fmt.Sprintf("%s/%s/%s/width=%d", arch, row.op, row.identity, row.bits), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + ins := Instr{Op: row.identity, Args: row.args} + var known, terminated bool + var err error + if row.pair { + known, terminated, err = ctx.lowerPairCompareExchange(string(row.op), row.bits, ins) + } else { + known, terminated, err = ctx.lowerScalarCompareExchange(string(row.op), row.bits, ins) + } + if !known || terminated || err == nil || errors.Is(err, ErrProbeNeedsContext) != row.context || + before != output.String() || temporary != ctx.tmp { + t.Fatalf("direct helper bypassed complete grammar or emitted partial IR: %t/%t/%v", known, terminated, err) + } + }) + } + } +} + +func TestX86CMPXCHGFPRequiresCompleteNativeExtentAndIdentity(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + body, declaration string + valid bool + }{ + {"CMPXCHGB CL,value+0(FP)", "func locked(value uint8)", true}, + {"CMPXCHGW CX,value+0(FP)", "func locked(value uint8)", false}, + {"CMPXCHGL CX,value+0(FP)", "func locked(value uint16)", false}, + {"CMPXCHG8B value+0(FP)", "func locked(value uint32)", false}, + {"CMPXCHGL CX,value+2(FP)", "func locked(value uint32)", false}, + {"CMPXCHGL CX,value+8(FP)", "func locked(value uint32)", false}, + } { + t.Run(arch+"/"+row.body+"/"+row.declaration, func(t *testing.T) { + source := "TEXT ·locked(SB),4,$0-16\nMOVL $1,AX\nMOVL $2,BX\nMOVL $3,CX\nMOVL $4,DX\n" + row.body + "\nRET\n" + actual := x86OrderingHintGoBytes(t, arch, source) + t.Logf("actual Go accepts the source spelling: %x; this does not prove its typed FP extent", actual) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\n"+row.declaration+"\n", arch, "") + if row.valid { + if err != nil || !strings.Contains(ir, "cmpxchgb") { + t.Fatalf("complete pre-existing one-byte FP extent was rejected in %s: %v", api, err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "fp-control.ll", "fp-control.o", ir) + continue + } + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("incomplete FP native extent acquired an atomic write in %s: %v", api, err) + } + } + }) + } + } + for _, api := range []string{"text", "module", "Go"} { + t.Run("amd64/16B/shadow/"+api, func(t *testing.T) { + source := "TEXT ·locked(SB),4,$0-16\nMOVL $1,AX\nMOVL $2,BX\nMOVL $3,CX\nMOVL $4,DX\nCMPXCHG16B value+0(FP)\nRET\n" + actual := x86OrderingHintGoBytes(t, "amd64", source) + t.Logf("actual original FP16 opcode: %x; no native alignment identity supplied by a logical shadow", actual) + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(value [2]uint64)\n", "amd64", "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("virtual FP shadow fabricated CMPXCHG16B native address/alignment: %v", err) + } + }) + } +} diff --git a/amd64_cmpxchg_fault_observer_test.go b/amd64_cmpxchg_fault_observer_test.go new file mode 100644 index 000000000..e2007a276 --- /dev/null +++ b/amd64_cmpxchg_fault_observer_test.go @@ -0,0 +1,166 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os/exec" + "runtime" + "strconv" + "syscall" + "testing" + "time" +) + +func x86CMPXCHGObserveFault(t *testing.T, execution x86FloatCompareExecution, executable string, index, address, original int) (string, error) { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + return x86CMPXCHGObserveFaultContext(ctx, execution, executable, index, address, original) +} + +func x86CMPXCHGObserveFaultContext(ctx context.Context, execution x86FloatCompareExecution, executable string, index, address, original int) (string, error) { + // Each leaf executable is the owned child: no fork, descendant, exception + // handler or native unwinder can hide the actual instruction's termination. + child, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + args := execution.command(executable, strconv.Itoa(index), strconv.Itoa(address), strconv.Itoa(original)) + stdout, stderr, err := runtimeOracleMachineOutput(child, exec.CommandContext(child, args[0], args[1:]...)) + if child.Err() != nil { + return "", child.Err() + } + witness := fmt.Sprintf("CMPXCHG enter %d %d %d\n", index, address, original) + result, cause := x86CMPXCHGFaultResult(runtime.GOOS, string(stdout), witness, err) + if cause != nil { + return "", fmt.Errorf("kernel %d address %d original %d: %w\nstdout: %q\nstderr: %q", index, address, original, cause, stdout, stderr) + } + return result, nil +} + +func x86CMPXCHGFaultResult(goos, stdout, witness string, err error) (string, error) { + if err == nil { + if stdout == witness+"CMPXCHG complete\n" { + return "return", nil + } + return "", fmt.Errorf("missing exact completed state/canary receipt") + } + // Do not unwrap joined errors: cancellation, pipes and ownership cleanup + // failures must not become accepted native exceptions. + exit, ok := err.(*exec.ExitError) + if !ok || exit.ProcessState == nil || stdout != witness { + return "", fmt.Errorf("not an isolated native fault after the exact entry receipt: %w", err) + } + status, ok := exit.ProcessState.Sys().(syscall.WaitStatus) + if !ok { + return "", fmt.Errorf("missing actual process termination status") + } + if goos == "windows" { + return x86CMPXCHGWindowsFault(uint32(exit.ExitCode())) + } + if status.Signaled() { + switch status.Signal() { + case syscall.SIGSEGV: + return "segv", nil + case syscall.SIGBUS: + return "bus", nil + case syscall.SIGILL: + return "ill", nil + } + } + return "", fmt.Errorf("setup, state or unexpected process failure: %w", err) +} + +func x86CMPXCHGWindowsFault(code uint32) (string, error) { + // GetExitCodeProcess returns the unhandled exception value, not 128+signal. + // Keep only these three native exception classes; ordinary exits never + // count as faults, even when original and translated setup fail identically. + switch code { + case 0xc0000005: // EXCEPTION_ACCESS_VIOLATION + return "segv", nil + case 0x80000002: // EXCEPTION_DATATYPE_MISALIGNMENT + return "bus", nil + case 0xc000001d: // EXCEPTION_ILLEGAL_INSTRUCTION + return "ill", nil + } + return "", fmt.Errorf("not a registered native Windows exception: %#x", code) +} + +func TestX86CMPXCHGWindowsFaultCodes(t *testing.T) { + for code, want := range map[uint32]string{0xc0000005: "segv", 0x80000002: "bus", 0xc000001d: "ill"} { + if got, err := x86CMPXCHGWindowsFault(code); err != nil || got != want { + t.Fatalf("exception %#x: %s, %v", code, got, err) + } + } + for _, code := range []uint32{0, 1, 80, 82, 83, 84, 85, 86, 87, 88, 120, 125, 132, 135, 137, 139, 259, 0xc00000fd, 0xffffffff} { + if _, err := x86CMPXCHGWindowsFault(code); err == nil { + t.Fatalf("ordinary/setup/stack-overflow exit %#x became native opcode evidence", code) + } + } +} + +func TestX86CMPXCHGFaultObserverRejectsFalseEvidence(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + const driver = ` +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif +int main(int argc, char **argv) { +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); +#else + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) return 120; +#endif + if (argc != 4) return 80; + int mode = atoi(argv[1]); + if (mode == 2) return 82; + printf("CMPXCHG enter %d 0 0\n", mode == 7 ? 99 : mode); + if (fflush(stdout)) return 87; + if (mode == 1 || mode == 7) __asm__ volatile("movl $0,(%0)" :: "r"((uintptr_t)0) : "memory"); + if (mode == 3) return 139; + if (mode == 4) return 0; + if (mode == 5) __asm__ volatile("ud2"); + if (mode == 6) return 85; + if (mode == 8) for (;;) __asm__ volatile("" ::: "memory"); + puts("CMPXCHG complete"); + return 0; +} +` + executable := execution.compileLLVM(t, "fault-controls", "", driver, "-O0") + for _, test := range []struct { + mode int + want string + }{{0, "return"}, {1, "segv"}, {2, ""}, {3, ""}, {4, ""}, {5, "ill"}, {6, ""}, {7, ""}} { + t.Run(strconv.Itoa(test.mode), func(t *testing.T) { + got, err := x86CMPXCHGObserveFault(t, execution, executable, test.mode, 0, 0) + if got != test.want || (err == nil) != (test.want != "") { + t.Fatalf("got %s, %v; want %s", got, err, test.want) + } + }) + } + t.Run("cancellation", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + if _, err := x86CMPXCHGObserveFaultContext(ctx, execution, executable, 8, 0, 0); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("cancellation became a native opcode outcome: %v", err) + } + }) + t.Run("joined-cleanup", func(t *testing.T) { + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + args := execution.command(executable, "5", "0", "0") + stdout, _, err := runtimeOracleMachineOutput(ctx, exec.CommandContext(ctx, args[0], args[1:]...)) + const witness = "CMPXCHG enter 5 0 0\n" + if got, cause := x86CMPXCHGFaultResult(runtime.GOOS, string(stdout), witness, err); cause != nil || got != "ill" { + t.Fatal("missing actual illegal-instruction control", got, cause) + } + if _, cause := x86CMPXCHGFaultResult(runtime.GOOS, string(stdout), witness, errors.Join(err, fmt.Errorf("owned cleanup failure"))); cause == nil { + t.Fatal("cleanup error was erased by the native exception") + } + }) +} diff --git a/amd64_cmpxchg_fault_portability_test.go b/amd64_cmpxchg_fault_portability_test.go new file mode 100644 index 000000000..f6988bd33 --- /dev/null +++ b/amd64_cmpxchg_fault_portability_test.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86CMPXCHGFaultWindowsHeaderBoundary(t *testing.T) { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang required") + } + directory := t.TempDir() + if err := os.Mkdir(filepath.Join(directory, "sys"), 0700); err != nil { + t.Fatal(err) + } + // This observes actual preprocessing of the complete driver, not a + // Windows SDK, object, link or runtime claim. Missing POSIX dependencies + // are poisoned so the host cannot accidentally supply them. + for _, name := range []string{"stdint.h", "stdio.h", "string.h", "stdlib.h", "signal.h", "windows.h"} { + if err := os.WriteFile(filepath.Join(directory, name), nil, 0600); err != nil { + t.Fatal(err) + } + } + for _, name := range []string{"unistd.h", "sys/mman.h", "sys/resource.h", "sys/wait.h"} { + if err := os.WriteFile(filepath.Join(directory, filepath.FromSlash(name)), []byte("#error unexpected_POSIX_fault_dependency\n"), 0600); err != nil { + t.Fatal(err) + } + } + input := filepath.Join(directory, "fault.c") + if err := os.WriteFile(input, []byte(x86CMPXCHGFaultReference), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, clang, "-target", "x86_64-w64-windows-gnu", "-nostdinc", + "-I", directory, "-E", "-P", input) + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil { + t.Fatalf("Windows fault driver still consumes POSIX headers: %v\n%s", err, output) + } + for _, required := range []string{"VirtualAlloc", "VirtualProtect", "VirtualFree", "kernel.original(p)", "kernel.translated(p)"} { + if !strings.Contains(string(output), required) { + t.Fatal("Windows fault driver lost its real guard or operation", required) + } + } +} diff --git a/amd64_cmpxchg_fault_runtime_test.go b/amd64_cmpxchg_fault_runtime_test.go new file mode 100644 index 000000000..b69af4f86 --- /dev/null +++ b/amd64_cmpxchg_fault_runtime_test.go @@ -0,0 +1,214 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +// Native opcode outcomes, not CPUID advertisements, establish availability and +// fault behavior. This is an instruction-level C observer tied to original Go +// opcode bytes, not execution of the stock Go ABI0 caller or a native CI claim. +func TestX86CMPXCHGOriginalOpcodeOutcomeEquivalence(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + x86CMPXCHGRunFaultCollector(t, execution, false) +} + +func x86CMPXCHGRunFaultCollector(t *testing.T, execution x86FloatCompareExecution, native bool) { + t.Helper() + var assembly, declarations, prototypes, originals, cases strings.Builder + declarations.WriteString("package ordering\n") + for index, row := range x86CMPXCHGOriginalRows { + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("cas%d_%t", index, locked) + body := x86CMPXCHGOriginalBody(index, "(SI)") + if locked { + body = "LOCK\n\t" + body + } + // Use the source register initialized by the reference: desired low + // is in CX for scalar forms, and BX/CX for pair forms. + source := strings.Replace(x86NamedLockOracleSource("amd64", body), "·locked", "·"+name, 1) + actual := x86OrderingHintGoBytes(t, "amd64", source) + opcode := []byte{0x0f, 0xb1, 0x0e} + switch row.bits { + case 8: + if !row.pair { + opcode = []byte{0x0f, 0xb0, 0x0e} + } + case 16: + opcode = append([]byte{0x66}, opcode...) + case 64: + if row.pair { + opcode = []byte{0x0f, 0xc7, 0x0e} + } else { + opcode = append([]byte{0x48}, opcode...) + } + case 128: + opcode = []byte{0x48, 0x0f, 0xc7, 0x0e} + } + if locked { + opcode = append([]byte{0xf0}, opcode...) + } + if !bytes.HasSuffix(actual, append(append([]byte(nil), opcode...), 0xc3)) { + t.Fatalf("complete original opcode/prefix binding changed: %s %x", name, actual) + } + t.Logf("actual Go %s terminal machine bytes=%x; object identity only", name, opcode) + assembly.WriteString(source) + fmt.Fprintf(&declarations, "func %s(p *[32]byte)\n", name) + fmt.Fprintf(&prototypes, "extern void %s(unsigned char *);\n", name) + fmt.Fprintf(&cases, " {%s, original_%s, %d, %d},\n", name, name, row.bits/8, index) + prefix := "" + if locked { + prefix = "lock; " + } + if row.pair { + part, mnemonic := "uint32_t", "cmpxchg8b" + if row.bits == 128 { + part, mnemonic = "uint64_t", "cmpxchg16b" + } + fmt.Fprintf(&originals, `static void original_%s(unsigned char *p) { + %s low = 2, high = 5; + __asm__ volatile("%s%s (%%4)" + : "+a"(low), "+d"(high) + : "b"((%s)3), "c"((%s)4), "r"(p) + : "cc", "memory"); +} +`, name, part, prefix, mnemonic, part, part) + } else { + typ := fmt.Sprintf("uint%d_t", row.bits) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[row.bits] + fmt.Fprintf(&originals, `static void original_%s(unsigned char *p) { + %s accumulator = 2; + __asm__ volatile("%scmpxchg%s %%1, (%%2)" + : "+a"(accumulator) + : "c"((%s)4), "r"(p) + : "cc", "memory"); +} +`, name, typ, prefix, width, typ) + } + } + } + ir, err := x86OrderingHintTranslateAPI(t, "text", assembly.String(), declarations.String(), "amd64", execution.triple) + if err != nil { + t.Fatal(err) + } + reference := fmt.Sprintf(x86CMPXCHGFaultReference, prototypes.String(), originals.String(), cases.String()) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := execution.compileLLVM(t, "cmpxchg-fault", ir, reference, level) + observations, deviations := 0, 0 + for index := 0; index < len(x86CMPXCHGOriginalRows)*2; index++ { + for address := 0; address < 4; address++ { + original, err := x86CMPXCHGObserveFault(t, execution, executable, index, address, 1) + if err != nil { + t.Fatal(err) + } + translated, err := x86CMPXCHGObserveFault(t, execution, executable, index, address, 0) + if err != nil { + t.Fatal(err) + } + if original != translated { + t.Fatalf("kernel %d address %d: original %s, translated %s", index, address, original, translated) + } + misalignedPair := x86CMPXCHGOriginalRows[index/2].bits == 128 && address == 1 + fault := address >= 2 || misalignedPair + if fault && original != "segv" && original != "bus" || !fault && original != "return" { + deviations++ + } + if native && misalignedPair && original != "segv" { + t.Fatal("native Linux misaligned CMPXCHG16B did not report #GP as SIGSEGV") + } + observations += 2 + } + } + if observations != 96 || native && deviations != 0 { + t.Fatalf("CMPXCHG opcode outcomes %d, ISA deviations %d", observations, deviations) + } + t.Logf("CMPXCHG opcode outcomes %d; ISA deviations %d; outcome failures 0", observations, deviations) + }) + } +} + +const x86CMPXCHGFaultReference = ` +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif +` + x86TestGuardPagesC + ` + +%s +%s +typedef struct { + void (*translated)(unsigned char *); + void (*original)(unsigned char *); + unsigned width, family; +} Kernel; + +static int exercise(Kernel kernel, unsigned index, unsigned address, unsigned original) { + _Alignas(16) unsigned char raw[64]; + memset(raw, 0xa7, sizeof(raw)); + unsigned char *p = raw + 16; + if (address == 1) p++; + if (address == 2) p = NULL; + if (address == 3) { + if (setup_guard()) return 82; + memset(guard, 0xa7, page); + p = guard + page - (kernel.width - 1); + } + unsigned char expected[16] = {0}; + uint64_t low = 2, high = 5, desired_low = 3, desired_high = 4; + if (kernel.family == 4) { + uint32_t pair[2] = {2, 5}, desired[2] = {3, 4}; + memcpy(expected, desired, 8); + if (address < 2) memcpy(p, pair, 8); + } else if (kernel.family == 5) { + memcpy(expected, &desired_low, 8); + memcpy(expected + 8, &desired_high, 8); + if (address < 2) { + memcpy(p, &low, 8); + memcpy(p + 8, &high, 8); + } + } else { + memcpy(expected, &desired_high, kernel.width); + if (address < 2) memcpy(p, &low, kernel.width); + } + if (printf("CMPXCHG enter %%u %%u %%u\n", index, address, original) < 0 || fflush(stdout)) return 87; + if (original) kernel.original(p); + else kernel.translated(p); + if (address == 3 && free_guard()) return 83; + if (address >= 2) return 84; + if (memcmp(p, expected, kernel.width)) return 85; + size_t begin = (size_t)(p - raw); + for (size_t i = 0; i < sizeof(raw); i++) { + if ((i < begin || i >= begin + kernel.width) && raw[i] != 0xa7) return 86; + } + if (puts("CMPXCHG complete") < 0 || fflush(stdout)) return 88; + return 0; +} + +int main(int argc, char **argv) { +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); +#else + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) return 120; +#endif + Kernel kernels[] = { +%s + }; + if (argc != 4 || strlen(argv[1]) > 2 || strlen(argv[2]) != 1 || strlen(argv[3]) != 1) return 80; + unsigned long values[3]; + for (unsigned i = 0; i < 3; i++) { + char *end; + if (argv[i+1][0] < '0' || argv[i+1][0] > '9') return 80; + values[i] = strtoul(argv[i+1], &end, 10); + if (*end) return 80; + } + if (values[0] >= sizeof(kernels)/sizeof(kernels[0]) || values[1] >= 4 || values[2] >= 2) return 80; + return exercise(kernels[values[0]], (unsigned)values[0], (unsigned)values[1], (unsigned)values[2]); +} +` diff --git a/amd64_cmpxchg_forms.go b/amd64_cmpxchg_forms.go new file mode 100644 index 000000000..886d9845c --- /dev/null +++ b/amd64_cmpxchg_forms.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86CompareExchangeSpec struct { + bits int + pair bool +} + +// Complete original yrb_mb, yrl_ml and yscond rows. Syntax acceptance is +// distinct from a physical encoder-view or native-memory permission. +var x86CompareExchangeSpecs = map[Op]x86CompareExchangeSpec{ + "CMPXCHGB": {bits: 8}, + "CMPXCHGW": {bits: 16}, + "CMPXCHGL": {bits: 32}, + "CMPXCHGQ": {bits: 64}, + "CMPXCHG8B": {bits: 64, pair: true}, + "CMPXCHG16B": {bits: 128, pair: true}, +} + +type x86CompareExchangeForm struct { + spec x86CompareExchangeSpec + source Operand + destination Operand +} + +func x86CompareExchangeProperties(op Op, ins Instr, arch string) (x86CompareExchangeForm, bool, error) { + name := strings.ToUpper(string(op)) + base := strings.SplitN(name, ".", 2)[0] + spec, recognized := x86CompareExchangeSpecs[Op(base)] + if !recognized { + return x86CompareExchangeForm{}, false, nil + } + invalid := func(reason string) (x86CompareExchangeForm, bool, error) { + return x86CompareExchangeForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + context := func(reason string) (x86CompareExchangeForm, bool, error) { + return x86CompareExchangeForm{}, true, fmt.Errorf("%w: %s %s %s: %q", ErrProbeNeedsContext, arch, op, reason, ins.Raw) + } + if name != base || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return invalid("does not accept suffixes or a contradictory source identity") + } + if arch != "amd64" && arch != "386" { + return invalid("requires a registered x86 mode") + } + if arch == "386" && (spec.bits == 128 || spec.bits == 64 && !spec.pair) { + return invalid("is illegal in 32-bit mode") + } + count := 2 + if spec.pair { + count = 1 + } + if len(ins.Args) != count { + return invalid("does not have its complete ordered operand count") + } + form := x86CompareExchangeForm{spec: spec, destination: ins.Args[count-1]} + registerAllowed := func(reg Reg) bool { + if spec.pair || spec.bits == 8 { + return isGoYmbRegisterForArch(reg, arch) + } + return isX86YrlRegisterForArch(reg, arch) + } + if !spec.pair { + form.source = ins.Args[0] + if form.source.Kind != OpReg || !registerAllowed(form.source.Reg) { + return invalid("source is outside its original Yrb/Yrl class") + } + } + destination := form.destination + if destination.Kind == OpReg { + if !registerAllowed(destination.Reg) { + return invalid("destination is outside its original Ymb/Yml class") + } + } else if !isAMD64MemoryOperand(destination) || + destination.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(destination.Sym), "$") { + return invalid("destination is not its original register/memory operand") + } else if destination.Kind == OpMem && !x86MemoryRegistersValidForArch(destination.Mem, arch) { + return invalid("memory uses an out-of-range address register") + } + if spec.pair && destination.Kind == OpReg { + return context("Go emits a non-memory pair operand with native #UD, not a numerical CAS") + } + if spec.bits == 8 { + if arch == "386" { + for _, operand := range []Operand{form.source, destination} { + if operand.Kind == OpReg && (operand.Reg == BP || operand.Reg == SI || operand.Reg == DI) { + return context("Go's byte-register expansion changes the implicit accumulator before CMPXCHG") + } + } + } else if x86CompareExchangeHasREXByteView(form) { + for _, operand := range []Operand{form.source, destination} { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == BH || operand.Reg == CH || operand.Reg == DH) { + return context("Go's REX encoding changes a high-byte spelling to an unbound low-byte physical view") + } + } + } + } + return form, true, nil +} + +func x86CompareExchangeHasREXByteView(form x86CompareExchangeForm) bool { + requiresREX := func(reg Reg) bool { + switch reg { + case SP, BP, SI, DI, BPB, SIB, DIB: + return true + } + base, _, alias := amd64ByteAlias(reg) + if alias { + reg = base + } + return strings.HasPrefix(string(reg), "R") + } + for _, operand := range []Operand{form.source, form.destination} { + if operand.Kind == OpReg && requiresREX(operand.Reg) { + return true + } + if operand.Kind == OpMem { + for _, reg := range []Reg{operand.Mem.Base, operand.Mem.Index} { + if strings.HasPrefix(string(reg), "R") { + return true + } + } + } + } + return false +} diff --git a/amd64_cmpxchg_linux_amd64_test.go b/amd64_cmpxchg_linux_amd64_test.go new file mode 100644 index 000000000..373ce1e75 --- /dev/null +++ b/amd64_cmpxchg_linux_amd64_test.go @@ -0,0 +1,44 @@ +package plan9asm + +import "testing" + +// These strict mathematical oracles require real Linux/AMD64 execution. +// The unchanged shared fixtures preserve all six flags, complete aliases, +// MXCSR and canaries. Local Go/C equivalence never satisfies this ISA axis. +func TestCrossLinuxRuntimeMatrixX86CMPXCHGOriginalGoCompleteNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86CMPXCHGOriginalNumerical(t, x86FloatCompareExecutionTools(t)) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixX86CMPXCHGIndependentCOriginalOpcodeNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86CMPXCHGOriginalCNumerical(t, x86FloatCompareExecutionTools(t)) + x87IA32RequireNativeHost(t) +} + +// This required native axis is selected by the existing CrossLinuxRuntimeMatrix +// CI prefix. Rosetta/QEMU equivalence does not satisfy its actual #GP assertion. +func TestCrossLinuxRuntimeMatrixX86CMPXCHGNativeFaultAndAccessWidth(t *testing.T) { + x87IA32RequireNativeHost(t) + x86CMPXCHGRunFaultCollector(t, x86FloatCompareExecutionTools(t), true) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixX86CMPXCHGNativeNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + execution := x86FloatCompareExecutionTools(t) + t.Run("original-Go", func(t *testing.T) { + x86CMPXCHGOriginalNumerical(t, execution) + }) + t.Run("original-C-opcode", func(t *testing.T) { + x86CMPXCHGOriginalCNumerical(t, execution) + }) + t.Run("LLVM", func(t *testing.T) { + x86CMPXCHGLLVMNumerical(t, execution) + }) + t.Run("Go-C-actual-equivalence", func(t *testing.T) { + x86CMPXCHGActualEquivalence(t, execution) + }) + x87IA32RequireNativeHost(t) +} diff --git a/amd64_cmpxchg_native.go b/amd64_cmpxchg_native.go new file mode 100644 index 000000000..0e3fd88ff --- /dev/null +++ b/amd64_cmpxchg_native.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) compareExchangeNativePrefix(memory Operand) (lock, segment string) { + if c.namedAtomicLock { + lock = "lock; " + } + if memory.Kind == OpMem && memory.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(memory.Mem.Segment)) + ":" + } + return lock, segment +} + +func (c *amd64Ctx) emitScalarCompareExchangeNative(bits int, memory Operand, pointer, pointerType, expected, desired string) (old, success string) { + lock, segment := c.compareExchangeNativePrefix(memory) + typ := amd64IntegerTypeForBits(bits) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[bits] + register := "r" + if bits == 8 { + register = "q" + } + result, oldName, successByte, successName := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + // The complete native opcode supplies the read/write cycle on both paths, + // including failure. A plain original opcode does not acquire LOCK. No LLVM + // alignment precondition is asserted, and explicit FS/GS remains attached. + fmt.Fprintf(c.b, " %%%s = call { %s, i8 } asm sideeffect \"%scmpxchg%s $3, %s$2; sete $1\", \"={ax},=q,=*m,%s,0,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %s, %s %s, %s %s, %s elementtype(%s) %s)\n", + result, typ, lock, width, segment, register, pointerType, typ, pointer, typ, desired, typ, expected, pointerType, typ, pointer) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 0\n", oldName, typ, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 1\n", successByte, typ, result) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", successName, successByte) + return "%" + oldName, "%" + successName +} + +func (c *amd64Ctx) emitPairCompareExchangeNative(bits int, memory Operand, pointer, pointerType, ax, dx, bx, cx string) (old, success string) { + lock, segment := c.compareExchangeNativePrefix(memory) + part, pair, mnemonic := I32, LLVMType("i64"), "cmpxchg8b" + if bits == 128 { + part, pair, mnemonic = I64, LLVMType("i128"), "cmpxchg16b" + } else { + ax, dx = c.truncI64(ax, I32), c.truncI64(dx, I32) + bx, cx = c.truncI64(bx, I32), c.truncI64(cx, I32) + } + result := c.newTmp() + // 8B supports unaligned memory without an invented natural alignment. + // 16B retains the actual opcode's native #GP when its address is unaligned; + // LLVM align=16 would instead give undefined behavior, not that exception. + fmt.Fprintf(c.b, " %%%s = call { %s, %s, i8 } asm sideeffect \"%s%s %s$3; sete $2\", \"={ax},={dx},=q,=*m,{bx},{cx},0,1,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %s, %s %s, %s %s, %s %s, %s %s, %s elementtype(%s) %s)\n", + result, part, part, lock, mnemonic, segment, pointerType, pair, pointer, part, bx, part, cx, part, ax, part, dx, pointerType, pair, pointer) + low, high, byteName, successName := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, %s, i8 } %%%s, 0\n", low, part, part, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, %s, i8 } %%%s, 1\n", high, part, part, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, %s, i8 } %%%s, 2\n", byteName, part, part, result) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", successName, byteName) + low64, high64 := "%"+low, "%"+high + if part == I32 { + lowWide, highWide := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", lowWide, low64) + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", highWide, high64) + low64, high64 = "%"+lowWide, "%"+highWide + } + return c.compareExchangeJoinPair(low64, high64, bits), "%" + successName +} diff --git a/amd64_cmpxchg_native_family_test.go b/amd64_cmpxchg_native_family_test.go new file mode 100644 index 000000000..29b23133a --- /dev/null +++ b/amd64_cmpxchg_native_family_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Independent complete original Go registrations, including ordered encoder +// class, prefix and bytes. Go's yscond admits a register spelling even though +// Intel's pair instructions require memory and raise #UD for ModRM.mod=3. +var x86CMPXCHGOriginalRows = []struct { + op, registration string + bits int + pair bool +}{ + {"CMPXCHGB", "{ACMPXCHGB, yrb_mb, Pb, opBytes{0x0f, 0xb0}}", 8, false}, + {"CMPXCHGW", "{ACMPXCHGW, yrl_ml, Pe, opBytes{0x0f, 0xb1}}", 16, false}, + {"CMPXCHGL", "{ACMPXCHGL, yrl_ml, Px, opBytes{0x0f, 0xb1}}", 32, false}, + {"CMPXCHGQ", "{ACMPXCHGQ, yrl_ml, Pw, opBytes{0x0f, 0xb1}}", 64, false}, + {"CMPXCHG8B", "{ACMPXCHG8B, yscond, Pm, opBytes{0xc7, 01}}", 64, true}, + {"CMPXCHG16B", "{ACMPXCHG16B, yscond, Pw, opBytes{0x0f, 0xc7, 01}}", 128, true}, +} + +func TestX86CMPXCHGByteEncoderViewsCannotInventSourceAliases(t *testing.T) { + for _, row := range []struct { + arch, body string + encoded []byte + }{ + {"amd64", "CMPXCHGB AH,R11", []byte{0x41, 0x0f, 0xb0, 0xe3}}, + {"amd64", "CMPXCHGB AH,(R12)", []byte{0x41, 0x0f, 0xb0, 0x24, 0x24}}, + {"amd64", "CMPXCHGB R11,AH", []byte{0x44, 0x0f, 0xb0, 0xdc}}, + {"386", "CMPXCHGB BP,(BX)", []byte{0x95, 0x0f, 0xb0, 0x03, 0x95}}, + {"386", "CMPXCHGB SI,DI", []byte{0x96, 0x87, 0xdf, 0x0f, 0xb0, 0xc3, 0x87, 0xdf, 0x96}}, + } { + t.Run(row.arch+"/"+row.body, func(t *testing.T) { + source := x86NamedLockOracleSource(row.arch, row.body) + if row.arch == "amd64" { + source = strings.Replace(source, "\t"+row.body, "\tMOVQ $9,R11\n\tMOVQ p+0(FP),R12\n\t"+row.body, 1) + } + actual := x86OrderingHintGoBytes(t, row.arch, source) + if !bytes.HasSuffix(actual, append(append([]byte(nil), row.encoded...), 0xc3)) { + t.Fatalf("original whole encoder effect changed: %x", actual) + } + mode := 64 + if row.arch == "386" { + mode = 32 + } + physical, err := x86asm.Decode(row.encoded, mode) + if err != nil { + t.Fatal(err) + } + t.Logf("original first physical opcode=%s; full sequence=%x", x86asm.GoSyntax(physical, 0, nil), row.encoded) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[32]byte)\n", row.arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("Go encoder view/implicit accumulator rewrite acquired a false source effect: %s/%v", api, err) + } + } + }) + } +} + +func TestX86CMPXCHGCompleteOriginalOrderedRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + table := string(data) + for _, row := range x86CMPXCHGOriginalRows { + if !strings.Contains(table, row.registration) { + t.Fatal("original opcode-specific encoding changed", row.registration) + } + } + for _, grammar := range []string{ + "var yrb_mb = []ytab{\n\t{Zr_m, 1, argList{Yrb, Ymb}},\n}", + "var yrl_ml = []ytab{\n\t{Zr_m, 1, argList{Yrl, Yml}},\n}", + "var yscond = []ytab{\n\t{Zo_m, 2, argList{Ymb}},\n}", + } { + if !strings.Contains(table, grammar) { + t.Fatal("complete ordered operand class changed", grammar) + } + } +} + +func x86CMPXCHGOriginalBody(row int, destination string) string { + op := x86CMPXCHGOriginalRows[row] + if op.pair { + return op.op + " " + destination + } + source := "CX" + if op.bits == 8 { + source = "CL" + } + return op.op + " " + source + "," + destination +} + +func TestX86CMPXCHGNativeMemoryHasNoInventedLLVMAlignment(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for index, row := range x86CMPXCHGOriginalRows { + if target.arch == "386" && (row.op == "CMPXCHGQ" || row.bits == 128) { + continue + } + for _, destination := range []string{"1(SI)", "0(FS)", "0(GS)"} { + for _, locked := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/%s/locked=%t", target.triple, row.op, destination, locked), func(t *testing.T) { + body := x86CMPXCHGOriginalBody(index, destination) + if locked { + body = "LOCK\n\t" + body + } + source := x86NamedLockOracleSource(target.arch, body) + actual := x86OrderingHintGoBytes(t, target.arch, source) + t.Logf("actual original %s bytes=%x (object only, no unsafe execution)", row.op, actual) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[32]byte)\n", target.arch, target.triple) + if err != nil { + t.Fatal("Go-accepted complete native memory operation", err) + } + if regexp.MustCompile(`(?m)^\s*%[^\n]+ = cmpxchg `).MatchString(ir) { + t.Fatal("native memory operand acquired an unproved LLVM atomic alignment") + } + want := strings.ToLower(row.op) + if locked { + want = "lock; " + want + } else if strings.Contains(ir, "lock; cmpxchg") { + t.Fatal("unprefixed original instruction acquired a LOCK effect") + } + if !strings.Contains(ir, "asm sideeffect \""+want) || !strings.Contains(ir, "=*m") || !strings.Contains(ir, "*m") { + t.Fatal("full native read/write opcode was not retained", want) + } + if destination == "0(FS)" && !strings.Contains(ir, "%fs:") || destination == "0(GS)" && !strings.Contains(ir, "%gs:") { + t.Fatal("actual original segment override disappeared") + } + compileLLVMToObject(t, findLLVM22Tool("llc"), target.triple, "cmpxchg-native.ll", "cmpxchg-native.o", ir) + }) + } + }) + } + } + } + } +} + +func TestX86CMPXCHGPairRegisterDoesNotInventSoftwareCAS(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86CMPXCHGOriginalRows[4:] { + if arch == "386" && row.bits == 128 { + continue + } + for _, register := range []string{"AX", "BX", "SI"} { + t.Run(arch+"/"+row.op+"/"+register, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, row.op+" "+register) + actual := x86OrderingHintGoBytes(t, arch, source) + t.Logf("Go-accepted pair-register #UD object bytes=%x; not a numerical oracle", actual) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[32]byte)\n", arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("pair-register #UD became a normal software CAS in %s: %v", api, err) + } + } + }) + } + } + } +} + +func TestX86CMPXCHGDispatchIdentityFailsBeforeEmission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + file, err := Parse(ArchAMD64, "TEXT locked(SB),4,$0-0\nCMPXCHGL CX,BX\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, identity := range []Op{"CMPXCHGQ", "MOVL", "CMPXCHGL.BAD"} { + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + before, temporary := out.String(), ctx.tmp + known, terminated, err := ctx.lowerCompareExchange("CMPXCHGL", ins) + if !known || terminated || err == nil || before != out.String() || temporary != ctx.tmp { + t.Fatalf("contradictory dispatched/source opcode changed IR: %s %v", identity, err) + } + } + } +} diff --git a/amd64_cmpxchg_numerical_runtime_test.go b/amd64_cmpxchg_numerical_runtime_test.go new file mode 100644 index 000000000..c3d718cde --- /dev/null +++ b/amd64_cmpxchg_numerical_runtime_test.go @@ -0,0 +1,347 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86CMPXCHGOriginalNumerical(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + declarations, source, names := x86CMPXCHGNumericalFixture() + driver := strings.Replace(x86CMPXCHGGoNumerical, "KERNEL_NAMES", names, 1) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), runtime.GOOS, "amd64", declarations, source, driver) + output := execution.run(t, original) + if string(output) != "original CMPXCHG numerical observations 576 failures 0\n" { + t.Fatalf("actual unchanged Go ABI0 numerical/status/canary oracle: %s", output) + } + t.Log(string(output)) +} + +func TestX86CMPXCHGLLVMCompleteNumerical(t *testing.T) { + x86CMPXCHGLLVMNumerical(t, x86FloatCompareExecutionTools(t)) +} + +func x86CMPXCHGLLVMNumerical(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + declarations, source, names := x86CMPXCHGNumericalFixture() + ir, err := x86OrderingHintTranslateAPI(t, "Go", source, declarations, "amd64", execution.triple) + if err != nil { + t.Fatal(err) + } + var prototypes strings.Builder + for _, name := range strings.Split(strings.TrimSpace(names), ",") { + name = strings.TrimSpace(name) + if name != "" { + fmt.Fprintf(&prototypes, "extern void %s(uint64_t *, unsigned char *, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t);\n", name) + } + } + reference := strings.Replace(x86CMPXCHGCNumerical, "KERNEL_PROTOTYPES", prototypes.String(), 1) + reference = strings.Replace(reference, "KERNEL_NAMES", names, 1) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "cmpxchg-numerical", ir, reference, level) + if string(output) != "LLVM CMPXCHG numerical observations 576 failures 0\n" { + t.Fatalf("independent full byte/status/canary oracle: %s", output) + } + t.Log(string(output)) + }) + } +} + +func x86CMPXCHGNumericalFixture() (declarations, source, names string) { + var decl, assembly, functions strings.Builder + decl.WriteString("package main\n") + // The independent family order binds the oracle's widths and pair policy. + for index, op := range []string{"CMPXCHGB", "CMPXCHGW", "CMPXCHGL", "CMPXCHGQ", "CMPXCHG8B", "CMPXCHG16B"} { + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("numeric_%d_%t", index, locked) + fmt.Fprintf(&decl, "func %s(out *[16]uint64, p *[32]byte, expectedLo, expectedHi, desiredLo, desiredHi, seed uint64)\n", name) + fmt.Fprintf(&functions, "%s,\n", name) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-56 +MOVQ out+0(FP),R10 +MOVQ p+8(FP),SI +MOVQ expectedLo+16(FP),AX +MOVQ expectedHi+24(FP),DX +MOVQ desiredLo+32(FP),BX +MOVQ desiredHi+40(FP),CX +MOVQ seed+48(FP),R8 +PUSHQ R8 +POPFQ +`, name) + if locked { + assembly.WriteString("LOCK\n") + } + switch index { + case 0: + assembly.WriteString(op + " BL,(SI)\n") + case 4, 5: + assembly.WriteString(op + " (SI)\n") + default: + assembly.WriteString(op + " BX,(SI)\n") + } + assembly.WriteString(`MOVQ AX,0(R10) +MOVQ DX,8(R10) +MOVQ BX,16(R10) +MOVQ CX,24(R10) +PUSHFQ +POPQ R9 +MOVQ R9,32(R10) +LAHF +MOVBLZX AH,BX +MOVQ BX,40(R10) +STMXCSR 48(R10) +RET +`) + } + } + decl.WriteString("func csr() uint32\n") + assembly.WriteString("TEXT ·csr(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n") + return decl.String(), assembly.String(), functions.String() +} + +const x86CMPXCHGGoNumerical = `package main + +import ( + "encoding/binary" + "fmt" + "os" + "runtime" + "unsafe" +) + +func widthMask(bits uint) uint64 { + if bits == 64 { return ^uint64(0) } + return uint64(1)<>4 + parity ^= parity>>2 + parity ^= parity>>1 + if parity&1 == 0 { flags |= 4 } + if a&15 < b&15 { flags |= 16 } + if result == 0 { flags |= 64 } + if result>>(bits-1) != 0 { flags |= 128 } + if ((a^b)&(a^result))>>(bits-1) != 0 { flags |= 2048 } + return flags +} + +func main() { + runtime.LockOSThread() + kernels := []func(*[16]uint64, *[32]byte, uint64, uint64, uint64, uint64, uint64){ + KERNEL_NAMES + } + edges := []uint64{0, 1, 15, 16, 127, 128, 255, 256, + 0x7fffffff, 0x80000000, 0x8000000000000000, ^uint64(0)} + widths := []uint{8, 16, 32, 64, 32, 64} + const canary uint64 = 0xa7a7a7a7a7a7a7a7 + observations, failures := 0, 0 + for index, kernel := range kernels { + kernelFailures, registerFailures, statusFailures := 0, 0, 0 + family := index/2 + bits := widths[family] + mask := widthMask(bits) + size := int(bits/8) + if family >= 4 { size *= 2 } + for _, edge := range edges { + for _, equal := range []bool{false, true} { + for _, seed := range []uint64{0x8c7, 0x12} { + raw := [96]byte{} + for i := range raw { raw[i] = 0xa7 } + aligned := 16 + int((-uintptr(unsafe.Pointer(&raw[0])))&15) + if family != 5 { aligned++ } + p := (*[32]byte)(unsafe.Pointer(&raw[aligned])) + low, high := edge&mask, (^edge)&mask + for byteIndex := 0; byteIndex < size; byteIndex++ { + value := low + if byteIndex >= int(bits/8) { value = high } + p[byteIndex] = byte(value >> (8*uint(byteIndex%int(bits/8)))) + } + expectedLow, expectedHigh := low, high + if !equal { expectedLow = (low+1)&mask } + expectedLow |= 0x13579bdf2468ace0 & ^mask + expectedHigh |= 0x2468ace013579bdf & ^mask + desiredLow := edge ^ 0xd3c2b1a097867564 + desiredHigh := ^edge ^ 0x789abcdef0123456 + expectedRaw := raw + if equal { + for byteIndex := 0; byteIndex < size; byteIndex++ { + value := desiredLow + if byteIndex >= int(bits/8) { value = desiredHigh } + expectedRaw[aligned+byteIndex] = byte(value >> (8*uint(byteIndex%int(bits/8)))) + } + } + out := [16]uint64{} + for i := range out { out[i] = canary } + before := csr() + kernel(&out, p, expectedLow, expectedHigh, desiredLow, desiredHigh, seed) + after := csr() + wantLow, wantHigh := expectedLow, expectedHigh + if !equal { + if bits == 32 { wantLow = low } else { wantLow = expectedLow&^mask | low } + if family >= 4 { + if bits == 32 { wantHigh = high } else { wantHigh = high } + } + } + flags := subtractFlags(expectedLow, low, bits) + if family >= 4 { + flags = seed & 0x8d5 &^ 64 + if equal { flags |= 64 } + } + csr := binary.LittleEndian.Uint32((*[4]byte)(unsafe.Pointer(&out[6]))[:]) + bad := out[0] != wantLow || out[1] != wantHigh || + out[2] != desiredLow || out[3] != desiredHigh || + out[4]&0x8d5 != flags || out[5] != (flags&0xd5 | 2) || + out[6]>>32 != canary>>32 || csr != before || after != before || raw != expectedRaw + for i := 7; i < len(out); i++ { bad = bad || out[i] != canary } + if bad { + kernelFailures++ + if out[0] != wantLow || out[1] != wantHigh || out[2] != desiredLow || out[3] != desiredHigh { + registerFailures++ + } + if out[4]&0x8d5 != flags || out[5] != (flags&0xd5 | 2) { + statusFailures++ + } + if failures < 24 { + fmt.Printf("original failure kernel=%d bits=%d edge=%x equal=%t seed=%x out=%x wantAX=%x wantDX=%x flags=%x\n", + index, bits, edge, equal, seed, out, wantLow, wantHigh, flags) + } + failures++ + } + observations++ + } + } + } + if kernelFailures != 0 { + fmt.Printf("original kernel=%d observations=48 failures=%d register=%d status=%d\n", + index, kernelFailures, registerFailures, statusFailures) + } + } + fmt.Printf("original CMPXCHG numerical observations %d failures %d\n", observations, failures) + if failures != 0 { os.Exit(1) } +} +` + +const x86CMPXCHGCNumerical = ` +#include +#include +#include + +KERNEL_PROTOTYPES +typedef void (*Kernel)(uint64_t *, unsigned char *, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); + +static uint64_t width_mask(unsigned bits) { + return bits == 64 ? UINT64_MAX : ((UINT64_C(1)<>bit)&1; + if (!(ones&1)) flags |= 4; + if ((a&15) < (b&15)) flags |= 16; + if (!difference) flags |= 64; + if (difference>>(bits-1)) flags |= 128; + if (((a^b)&(a^difference))>>(bits-1)) flags |= 2048; + return flags; +} + +int main(void) { + Kernel kernels[] = { KERNEL_NAMES }; + const uint64_t edges[] = {0, 1, 15, 16, 127, 128, 255, 256, + 0x7fffffff, 0x80000000, UINT64_C(0x8000000000000000), UINT64_MAX}; + const unsigned widths[] = {8, 16, 32, 64, 32, 64}; + const uint64_t seeds[] = {0x8c7, 0x12}; + const uint64_t canary = UINT64_C(0xa7a7a7a7a7a7a7a7); + unsigned observations = 0, failures = 0; + for (unsigned index = 0; index < 12; index++) { + unsigned kernel_failures = 0, register_failures = 0, status_failures = 0; + unsigned family = index/2, bits = widths[family]; + uint64_t mask = width_mask(bits); + unsigned size = bits/8 * (family >= 4 ? 2 : 1); + for (unsigned edge = 0; edge < 12; edge++) { + for (unsigned equal = 0; equal < 2; equal++) { + for (unsigned seed = 0; seed < 2; seed++) { + _Alignas(16) unsigned char raw[96], expected[96]; + memset(raw, 0xa7, sizeof(raw)); + unsigned start = family == 5 ? 16 : 17; + unsigned char *p = raw+start; + uint64_t low = edges[edge]&mask, high = ~edges[edge]&mask; + for (unsigned byte = 0; byte < size; byte++) { + uint64_t value = byte >= bits/8 ? high : low; + p[byte] = value>>(8*(byte%(bits/8))); + } + uint64_t expected_low = equal ? low : (low+1)&mask; + expected_low |= UINT64_C(0x13579bdf2468ace0)&~mask; + uint64_t expected_high = high | (UINT64_C(0x2468ace013579bdf)&~mask); + uint64_t desired_low = edges[edge]^UINT64_C(0xd3c2b1a097867564); + uint64_t desired_high = ~edges[edge]^UINT64_C(0x789abcdef0123456); + memcpy(expected, raw, sizeof(raw)); + if (equal) { + for (unsigned byte = 0; byte < size; byte++) { + uint64_t value = byte >= bits/8 ? desired_high : desired_low; + expected[start+byte] = value>>(8*(byte%(bits/8))); + } + } + uint64_t out[16]; + for (unsigned i = 0; i < 16; i++) out[i] = canary; + unsigned before, after; + __asm__ volatile("stmxcsr %0" : "=m"(before)); + kernels[index](out, p, expected_low, expected_high, desired_low, desired_high, seeds[seed]); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + uint64_t want_low = expected_low, want_high = expected_high; + if (!equal) { + want_low = bits == 32 ? low : (expected_low&~mask)|low; + if (family >= 4) want_high = high; + } + uint64_t flags = subtraction(expected_low, low, bits); + if (family >= 4) { + flags = (seeds[seed]&0x8d5)&~UINT64_C(64); + if (equal) flags |= 64; + } + int bad = out[0] != want_low || out[1] != want_high || + out[2] != desired_low || out[3] != desired_high || + (out[4]&0x8d5) != flags || out[5] != ((flags&0xd5)|2) || + (out[6]>>32) != (canary>>32) || (uint32_t)out[6] != before || + after != before || memcmp(raw, expected, sizeof(raw)); + for (unsigned i = 7; i < 16; i++) bad |= out[i] != canary; + if (bad) { + kernel_failures++; + register_failures += out[0] != want_low || out[1] != want_high || + out[2] != desired_low || out[3] != desired_high; + status_failures += (out[4]&0x8d5) != flags || out[5] != ((flags&0xd5)|2); + if (kernel_failures <= 2) { + fprintf(stdout, "LLVM failure kernel=%u bits=%u edge=%llx equal=%u seed=%llx AX=%llx/%llx DX=%llx/%llx flags=%llx/%llx BX=%llx CX=%llx\n", + index, bits, (unsigned long long)edges[edge], equal, + (unsigned long long)seeds[seed], (unsigned long long)out[0], + (unsigned long long)want_low, (unsigned long long)out[1], + (unsigned long long)want_high, (unsigned long long)out[4], + (unsigned long long)flags, (unsigned long long)out[2], (unsigned long long)out[3]); + } + failures++; + } + observations++; + } + } + } + if (kernel_failures) { + printf("LLVM kernel=%u observations=48 failures=%u register=%u status=%u\n", + index, kernel_failures, register_failures, status_failures); + } + } + printf("LLVM CMPXCHG numerical observations %u failures %u\n", observations, failures); + return failures ? 1 : 0; +} +` diff --git a/amd64_cmpxchg_object_test.go b/amd64_cmpxchg_object_test.go new file mode 100644 index 000000000..3015dce06 --- /dev/null +++ b/amd64_cmpxchg_object_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86CMPXCHGThreeAPIsFiveTargetsExactNativeO0O2Objects(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 compiler and disassembler are required") + } + for _, target := range x86OrderingHintTargets { + var source, declarations strings.Builder + declarations.WriteString("package ordering\n") + want := make(map[string]struct { + op string + bits int + locked bool + }) + for index, row := range x86CMPXCHGOriginalRows { + if target.arch == "386" && (row.op == "CMPXCHGQ" || row.bits == 128) { + continue + } + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("cas%d_%t", index, locked) + body := x86CMPXCHGOriginalBody(index, "1(SI)") + if locked { + body = "LOCK\n\t" + body + } + original := strings.Replace(x86NamedLockOracleSource(target.arch, body), "·locked", "·"+name, 1) + // Original objects are accepted without executing unsafe forms. + x86OrderingHintGoBytes(t, target.arch, original) + source.WriteString(original) + fmt.Fprintf(&declarations, "func %s(p *[32]byte)\n", name) + opcode := "CMPXCHG" + if row.pair { + opcode = row.op + } + want[name] = struct { + op string + bits int + locked bool + }{opcode, row.bits, locked} + } + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "native.ll"), filepath.Join(dir, "native.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", input, "-o", object)) + listing := x86FloatCompareCaptureCommand(t, exec.Command(disassembler, "--disassemble", object)) + sections := regexp.MustCompile(`(?m)^[0-9a-f]+ <_?(cas[0-9]+_(?:false|true))>:\n`).FindAllSubmatchIndex(listing, -1) + seen := make(map[string]bool) + for index, section := range sections { + name := string(listing[section[2]:section[3]]) + expected, known := want[name] + if !known || seen[name] { + t.Fatalf("unexpected or duplicate original function identity %s", name) + } + end := len(listing) + if index+1 < len(sections) { + end = sections[index+1][0] + } + var code []byte + for _, line := range strings.Split(string(listing[section[1]:end]), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasSuffix(fields[0], ":") { + continue + } + var machine []byte + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := hex.DecodeString(field) + if err != nil { + break + } + machine = append(machine, value[0]) + } + code = append(code, machine...) + } + // LLVM's disassembler displays a standalone F0 byte as a + // separate "lock" line. Decode complete contiguous bytes, + // not that presentation boundary, as the CPU does. + mode := 64 + if target.arch == "386" { + mode = 32 + } + count := 0 + for len(code) != 0 { + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len <= 0 || physical.Len > len(code) { + t.Fatalf("incomplete physical function decode %s: %x %v", name, code, err) + } + machine := code[:physical.Len] + code = code[physical.Len:] + if !strings.HasPrefix(physical.Op.String(), "CMPXCHG") { + continue + } + count++ + locked := false + for _, prefix := range physical.Prefix { + if byte(prefix) == 0xf0 { + locked = true + } + } + width := physical.DataSize + if bytes.Contains(machine, []byte{0x0f, 0xb0}) { + // x/arch DataSize retains the operand-size state for + // B0, whose opcode independently specifies byte width. + width = 8 + } + if physical.Op.String() != expected.op || locked != expected.locked || + expected.op == "CMPXCHG" && width != expected.bits { + t.Fatalf("%s/%s lost native width/prefix/opcode: %x %s\n%s", level, name, machine, + x86asm.GoSyntax(physical, 0, nil), listing) + } + } + if count != 1 { + t.Fatalf("%s/%s has %d complete native operations\n%s", level, name, count, listing) + } + seen[name] = true + } + if len(seen) != len(want) { + t.Fatalf("%s emitted %d/%d complete functions\n%s", level, len(seen), len(want), listing) + } + t.Logf("%s independently decoded %d native opcodes with exact width and LOCK identity", level, len(seen)) + } + }) + } + } +} diff --git a/amd64_cmpxchg_original_c_runtime_test.go b/amd64_cmpxchg_original_c_runtime_test.go new file mode 100644 index 000000000..422e51533 --- /dev/null +++ b/amd64_cmpxchg_original_c_runtime_test.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +func x86CMPXCHGOriginalCNumerical(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + reference := x86CMPXCHGOriginalCSource() + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := x86CMPXCHGCompileOriginalC(t, execution, reference, level) + if string(output) != "actual C opcode CMPXCHG numerical observations 576 failures 0\n" { + t.Fatalf("actual original C opcode vs independent math: %s", output) + } + t.Log(string(output)) + }) + } +} + +func x86CMPXCHGOriginalCSource() string { + _, _, names := x86CMPXCHGNumericalFixture() + var definitions strings.Builder + for index, op := range []string{"cmpxchgb %bl,", "cmpxchgw %bx,", "cmpxchgl %ebx,", "cmpxchgq %rbx,", "cmpxchg8b ", "cmpxchg16b "} { + for _, locked := range []bool{false, true} { + name := fmt.Sprintf("numeric_%d_%t", index, locked) + prefix := "" + if locked { + prefix = "lock; " + } + fmt.Fprintf(&definitions, `static void %s(uint64_t *out, unsigned char *p, + uint64_t expected_low, uint64_t expected_high, uint64_t desired_low, + uint64_t desired_high, uint64_t seed) { + __asm__ volatile( + "pushq %%[seed]; popfq; %s%s(%%%%rsi);" +`, name, prefix, strings.ReplaceAll(op, "%", "%%")) + definitions.WriteString(` "movq %%rax,0(%%rdi); movq %%rdx,8(%%rdi);" + "movq %%rbx,16(%%rdi); movq %%rcx,24(%%rdi);" + "pushfq; popq %%r9; movq %%r9,32(%%rdi);" + "lahf; movzbl %%ah,%%ebx; movq %%rbx,40(%%rdi);" + "stmxcsr 48(%%rdi);" + : "+a"(expected_low), "+d"(expected_high), "+b"(desired_low), "+c"(desired_high) + : "D"(out), "S"(p), [seed] "r"(seed) + : "r9", "cc", "memory"); +} +`) + } + } + reference := strings.Replace(x86CMPXCHGCNumerical, "KERNEL_PROTOTYPES", definitions.String(), 1) + reference = strings.Replace(reference, "KERNEL_NAMES", names, 1) + reference = strings.ReplaceAll(reference, "LLVM failure", "actual C opcode failure") + reference = strings.ReplaceAll(reference, "LLVM kernel", "actual C opcode kernel") + reference = strings.ReplaceAll(reference, "LLVM CMPXCHG numerical", "actual C opcode CMPXCHG numerical") + return reference +} + +func x86CMPXCHGCompileOriginalC(t *testing.T, execution x86FloatCompareExecution, sourceText, level string) []byte { + t.Helper() + directory := t.TempDir() + source := filepath.Join(directory, "original.c") + executable := filepath.Join(directory, gotoolchain.HostExecutable("original")) + if err := os.WriteFile(source, []byte(sourceText), 0600); err != nil { + t.Fatal(err) + } + command := append([]string(nil), execution.compiler...) + command = append(command, level, source, "-o", executable) + x86FloatCompareCaptureCommand(t, exec.Command(command[0], command[1:]...)) + return execution.run(t, executable) +} diff --git a/amd64_cmpxchg_original_catalogue_test.go b/amd64_cmpxchg_original_catalogue_test.go new file mode 100644 index 000000000..480b20152 --- /dev/null +++ b/amd64_cmpxchg_original_catalogue_test.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Keep the complete historical source bodies and all 33/24 accepted operands. +// The official include changes no instruction: it supplies their original +// NOSPLIT/NOPTR spellings from the selected actual SDK, not a invented ABI. +func x86CMPXCHGMixedOriginalCatalogue(t *testing.T, arch, source string) { + t.Helper() + directory := t.TempDir() + input, object := filepath.Join(directory, "complete.s"), filepath.Join(directory, "complete.o") + if err := os.WriteFile(input, []byte("#include \"textflag.h\"\n"+source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin", "go"), "tool", "asm", "-S", + "-I", filepath.Join(runtime.GOROOT(), "pkg", "include"), "-p", "ordering", "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH="+arch, + "GOFLAGS=", "GOENV=off", "GOWORK=off", "GOEXPERIMENT=") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, arch); failure != nil { + t.Fatalf("actual complete original syntax object: %v\n%s", failure, output) + } + t.Logf("actual original complete accepted catalogue, not unsafe execution:\n%s", output) + file, err := Parse(ArchAMD64, source) + if err != nil || len(file.Funcs) != 1 { + t.Fatalf("complete original source parse: %v", err) + } + count, physicalContext := 0, 0 + for _, instruction := range file.Funcs[0].Instrs { + if !strings.HasPrefix(string(instruction.Op), "CMPXCHG") { + continue + } + _, recognized, err := x86CompareExchangeProperties(instruction.Op, instruction, arch) + if !recognized || err != nil && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual original accepted row disappeared from grammar: %+v/%v", instruction, err) + } + count++ + if err != nil { + physicalContext++ + t.Logf("source/physical bank or native #UD Context: %s: %v", instruction.Raw, err) + } + } + want := 33 + if arch == "386" { + want = 24 + } + if count != want || physicalContext != 5 { + t.Fatalf("complete original syntax/physical-context denominator changed: %d/%d, want %d/5", count, physicalContext, want) + } +} diff --git a/amd64_comi_boundaries_test.go b/amd64_comi_boundaries_test.go new file mode 100644 index 000000000..9641cca52 --- /dev/null +++ b/amd64_comi_boundaries_test.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86COMIWholeFileNativeTargetAndDefinedFlags(t *testing.T) { + for _, name := range x86COMIOracleNames() { + for _, api := range []string{"text", "module", "Go"} { + for _, dead := range []bool{false, true} { + before := "" + if dead { + before = "RET\n" + } + for _, triple := range []string{"", "aarch64-unknown-linux-gnu", "wasm32-unknown-unknown", "i386-unknown-linux-gnu", "x86_64madeup-unknown-linux-gnu"} { + source := "TEXT ·view(SB),4,$0-0\n" + before + name + " X0,X1\nRET\n" + if _, err := x86ScalarFPBitViewAPI(t, source, "func view()\n", "amd64", triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing native target accepted", name, api, dead, triple, err) + } + malformed := strings.Replace(source, "X0,X1", "X0,Y1", 1) + if _, err := x86ScalarFPBitViewAPI(t, malformed, "func view()\n", "amd64", triple, api); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native-target context hid malformed/dead source", name, api, dead, triple, err) + } + } + } + for _, missing := range []int{-1, 0, 1} { + var setup strings.Builder + for input := 0; input < 2; input++ { + if input != missing { + fmt.Fprintf(&setup, "XORPS X%d,X%d\n", input, input) + } + } + source := "TEXT ·view(SB),4,$0-4\n" + setup.String() + "XORL AX,AX\n" + name + " X0,X1\nSETEQ AL\nMOVL AX,X2\nPEXTRD $0,X2,AX\nMOVL AX,ret+0(FP)\nRET\n" + _, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32\n", "amd64", "x86_64-unknown-linux-gnu", api) + if missing == -1 && err != nil || missing != -1 && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("undefined input acquired public result permission", name, api, missing, err) + } + } + } + } +} + +func TestX86COMIRuntimePartitionCompleteAndOriginalInventory(t *testing.T) { + remaining := make(map[string]bool) + for _, row := range x86ScalarCompareAFCases("amd64", true) { + remaining[row.name] = true + } + var counts [3]int + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareSSE, x86FloatCompareVEX, x86FloatCompareEVEX} { + cases := x86ScalarCompareAFRuntimeCases(axis) + for _, row := range cases { + if !remaining[row.name] { + t.Fatal("duplicate runtime kernel", row) + } + delete(remaining, row.name) + counts[axis-1]++ + } + goAssemblerOracleOutput(t, "amd64", x86ScalarCompareAFAssembly("amd64", cases), "", true) + } + if len(remaining) != 0 || counts != [3]int{48, 48, 48} { + t.Fatal("complete runtime partition changed", remaining, counts) + } +} diff --git a/amd64_comi_fault_runtime_test.go b/amd64_comi_fault_runtime_test.go new file mode 100644 index 000000000..60cac94c5 --- /dev/null +++ b/amd64_comi_fault_runtime_test.go @@ -0,0 +1,169 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86COMIDeadMemoryExtent(t *testing.T) { + x86COMIDeadFaults(t, false) +} + +func x86COMIDeadFaults(t *testing.T, native bool) { + t.Helper() + axis := x86FloatCompareSSE + if native { + axis = x86FloatCompareEVEX + } + execution := x86FloatCompareRuntimeTools(t, axis) + if native && (runtime.GOOS != "linux" || runtime.GOARCH != "amd64" || len(execution.prefix) != 0) { + t.Fatal("COMI exception delivery requires actual native Linux AMD64") + } + assembly, declarations, driver, _, trapChecks := x86COMIDeadFaultFixture(t, native, execution.triple) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, assembly, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, pipeline := range []struct{ name, ir, level string }{ + {"backend_O0", ir, "-O0"}, {"IR_O2/backend_O2", x86MulDivRoundOptimize(t, ir), "-O2"}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "comi-fault", pipeline.ir, driver, pipeline.level) + want := fmt.Sprintf("COMI original Go/LLVM dead reads matched; native IE/DE/SAE checks=%d\n", trapChecks) + if string(output) != want { + t.Fatal("missing complete fault receipt", string(output)) + } + }) + } + }) + } +} + +func x86COMIDeadFaultFixture(t *testing.T, native bool, triple string) (string, string, string, int, int) { + t.Helper() + var assembly, declarations, original, prototypes, pairs strings.Builder + count := 0 + for _, name := range x86COMIOracleNames() { + if strings.HasPrefix(name, "V") && !native { + continue + } + suffixes := []string{""} + if strings.HasPrefix(name, "V") { + suffixes = append(suffixes, ".SAE") + } + for _, suffix := range suffixes { + width, load := 4, "MOVSS" + if strings.HasSuffix(name, "D") { + width, load = 8, "MOVSD" + } + body := "XORPS X1,X1\nLDMXCSR (SI)\n" + name + " (DI),X1\n" + if suffix != "" { + body = "XORPS X1,X1\n" + load + " (DI),X0\nLDMXCSR (SI)\n" + name + suffix + " X0,X1\n" + } + body += "XORL AX,AX\nRET\n" // The comparison FLAGS are deliberately dead. + function, nativeFunction := fmt.Sprintf("fault%d", count), fmt.Sprintf("original%d", count) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DI\nMOVQ csr+8(FP),SI\n%s", function, body) + fmt.Fprintf(&declarations, "func %s(p *uint64,csr *uint32)\n", function) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+body) + bytes := make([]string, len(code)) + for index, value := range code { + bytes[index] = fmt.Sprintf("0x%02x", value) + } + symbol := nativeFunction + if strings.Contains(triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&original, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint64_t*,uint32_t*),%s(uint64_t*,uint32_t*);\n", function, nativeFunction) + quiet, sae := 0, 0 + if strings.Contains(name, "UCOMI") { + quiet = 1 + } + if suffix != "" { + sae = 1 + } + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d},\n", nativeFunction, function, width, quiet, sae) + count++ + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + trapChecks := 0 + if native { + trapChecks = 1 + } + driver := fmt.Sprintf("#define NATIVE_TRAPS %d\n", trapChecks) + ` +#include +#include +#include +#include +#include +#include +#include +#include +` + original.String() + prototypes.String() + ` +typedef void (*kernel)(uint64_t*,uint32_t*); +static const struct {kernel original,translated;int width,quiet,sae;} pairs[]={ +` + pairs.String() + `}; +static int outcome(kernel fn,uint8_t *p,uint32_t csr) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn((uint64_t*)p,&csr); + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) { + int signal=WTERMSIG(status); + if(signal==SIGSEGV || signal==SIGBUS) return 1; + if(signal==SIGFPE) return 2; + } + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *p=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED || mprotect(p+page,page,PROT_NONE)) return 2; + memset(p,0,page); + for(unsigned i=0;i= 16 || arch == "386" && register >= 8) { + return fail() + } + if arch == "386" && register >= 8 { + contextErr = fmt.Errorf("%w: %s lacks physical mode32 high-X identity", ErrProbeNeedsContext, ins.Op) + } + form.evex = form.evex || register >= 16 + continue + } + if index != 0 || !isAMD64MemoryOperand(operand) { + return fail() + } + if operand.Kind == OpMem { + if operand.Mem.OffRaw != "" || operand.Mem.IndexExt != "" { + return fail() + } + if err := x86SymbolIndexedMemoryError(operand.Mem, arch); err != nil { + if !errors.Is(err, ErrProbeNeedsContext) { + return form, true, err + } + contextErr = err + continue + } + if arch == "386" && strings.HasSuffix(operand.Mem.Sym, "(SB)") { + operand.Mem.Base = "" + } + } + if !x86PackedShiftMemoryValid(operand, arch) { + return fail() + } + } + return form, true, contextErr +} + +func validateX86COMIFile(file *File, options Options) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + needed := false + var contextErr error + for _, function := range file.Funcs { + for _, instruction := range function.Instrs { + _, known, err := x86COMIProperties(instruction, options.Goarch) + if known && err != nil { + if !errors.Is(err, ErrProbeNeedsContext) { + return fmt.Errorf("%s: %w", function.Sym, err) + } + if contextErr == nil { + contextErr = fmt.Errorf("%s: %w", function.Sym, err) + } + } + needed = needed || known + } + } + if contextErr != nil { + return contextErr + } + if needed && !x86ScalarFloatIntegerNativeTarget(options.Goarch, options.TargetTriple) { + return fmt.Errorf("%w: scalar flag comparison requires its native x86 MXCSR target", ErrProbeNeedsContext) + } + return nil +} + +func x86TypedCOMIByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + name, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86COMISpecs[Op(name)]; !known { + return x86TypedByteEffect{}, false, nil + } + if !strings.EqualFold(string(op), string(ins.Op)) { + return x86TypedByteEffect{}, true, fmt.Errorf("scalar flag comparison effect/source mismatch") + } + form, known, err := x86COMIProperties(ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + width := int64(form.spec.bits / 8) + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, read: true}, + }, flagsDefined: x86ByteFlagsAll}, true, nil +} + +func x86ResultCOMIStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86COMIProperties(ins, arch) + if !known || err != nil { + return false, known + } + width := int64(form.spec.bits / 8) + for _, operand := range ins.Args { + if operand.Kind == OpReg { + index, _ := amd64ParseXReg(operand.Reg) + mask := x86CallerVectorByteMask(width) + if state.vectorBytes(index)&mask != mask { + return false, true + } + } else if !x86ResultRead(operand, width, state, owned, arch) { + return false, true + } + } + state.flags = x86ByteFlagsAll + return true, true +} diff --git a/amd64_comi_grammar_test.go b/amd64_comi_grammar_test.go new file mode 100644 index 000000000..f61b1e23c --- /dev/null +++ b/amd64_comi_grammar_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func x86COMIOracleNames() []string { + return []string{"COMISS", "COMISD", "UCOMISS", "UCOMISD", "VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} +} + +func TestX86COMIIndependentCompleteEncoderRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "var yxm = []ytab{\n\t{Zm_r_xm, 1, argList{Yxm, Yxr}},\n}") { + t.Fatal("complete legacy yxm operand table changed") + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, name := range x86COMIOracleNames() { + double, vector := strings.HasSuffix(name, "D"), strings.HasPrefix(name, "V") + opcode := "0x2F" + if strings.Contains(name, "UCOMI") { + opcode = "0x2E" + } + if !vector { + prefix := "Pm" + if double { + prefix = "Pe" + } + entry := fmt.Sprintf("{A%s, yxm, %s, opBytes{%s}}", name, prefix, strings.ToLower(opcode)) + if !strings.Contains(string(legacy), entry) { + t.Fatal("legacy scalar compare opcode attributes changed", entry) + } + count++ + continue + } + vex, evex, tuple := "avxEscape | vex128 | vex0F | vexW0", "avxEscape | evex128 | evex0F | evexW0", "evexN4 | evexSaeEnabled" + if double { + vex, evex, tuple = "avxEscape | vex128 | vex66 | vex0F | vexW0", "avxEscape | evex128 | evex66 | evex0F | evexW1", "evexN8 | evexSaeEnabled" + } + opbytes := []string{vex, opcode, evex, tuple, opcode} + index := 0 + for _, row := range rules { + if row.Opcode != name { + continue + } + class, zcase, advance, offset := "Yxm", "Zvex_rm_v_r", 2, 0 + if index == 1 { + class, zcase, advance, offset = "YxmEvex", "Zevex_rm_v_r", 3, 2 + } + destination := "Yxr" + if index == 1 { + destination = "YxrEvex" + } + if row.Ytab != "_yvcomisd" || row.Prefix != "Pavx" || row.RowIndex != index || row.EncodingOffset != offset || + row.ZCase != zcase || row.ZOffset != advance || !reflect.DeepEqual(row.Classes, []string{class, destination}) || !reflect.DeepEqual(row.OpBytes, opbytes) { + t.Fatalf("%s independent complete encoder row changed: %+v", name, row) + } + index++ + count++ + } + if index != 2 { + t.Fatal("missing scalar compare encoder rows", name, index) + } + } + if count != 12 || len(x86COMISpecs) != 8 { + t.Fatal("complete eight-name/twelve-row family changed", count, len(x86COMISpecs)) + } +} + +func TestX86COMITypedEveryByteAndEffects(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86COMIOracleNames() { + width := 4 + if strings.HasSuffix(name, "D") { + width = 8 + } + instruction := x86ScalarLaneInstruction(t, name+" X1,X2") + effect, known, err := x86TypedInstructionByteEffect(instruction.Op, instruction, arch) + want := x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: instruction.Args[0], width: int64(width), read: true}, + {index: 1, operand: instruction.Args[1], width: int64(width), read: true}, + }, flagsDefined: x86ByteFlagsAll} + if !known || err != nil || !reflect.DeepEqual(effect, want) { + t.Fatal("comparison gained a write or lost exact input bytes/FLAGS", arch, name, effect, err) + } + for _, input := range []int{1, 2} { + for missing := -1; missing < 64; missing++ { + state := &x86ResultState{} + state.writeVector(1, 64, ^uint64(0), false) + state.writeVector(2, 64, ^uint64(0), false) + bytes := ^uint64(0) + if missing >= 0 { + bytes &^= 1 << uint(missing) + } + state.writeVector(input, 64, bytes, false) + accepted := x86ResultTransfer(instruction, state, nil, 0, arch) + want := missing < 0 || missing >= width + if accepted != want || state.vectorBytes(input) != bytes || accepted && state.flags != x86ByteFlagsAll || !accepted && state.flags != 0 { + t.Fatal("comparison consumed undefined bytes or changed inputs/FLAGS on failure", arch, name, input, missing) + } + } + } + for _, kind := range []string{"FP", "AUTO", "SB", "pointer"} { + for missing := -1; missing < width; missing++ { + owned := make([]bool, 16) + state := &x86ResultState{fp: make([]bool, 16), auto: make([]bool, 16)} + state.writeVector(2, 64, ^uint64(0), false) + for i := range owned { + owned[i], state.fp[i], state.auto[i] = i != missing, i != missing, i != missing + } + operand := "value+0(FP)" + switch kind { + case "AUTO": + operand = "local-16(SP)" + case "SB": + operand = "external(SB)" + case "pointer": + operand = "(BX)" + state.pointerGP[3] = true + } + accepted := x86ResultTransfer(x86ScalarLaneInstruction(t, name+" "+operand+",X2"), state, owned, 0, arch) + want := (kind == "FP" || kind == "AUTO") && missing == -1 + if accepted != want { + t.Fatal("comparison acquired unowned/undefined memory", arch, name, kind, missing) + } + } + } + } + } +} + +func TestX86COMIHelperCannotBorrowForeignDispatch(t *testing.T) { + for _, operation := range []Op{"ADDQ", "MOVQ", "UNKNOWN.COMISS", "COMISS"} { + var ir strings.Builder + ctx := amd64Ctx{b: &ir, goarch: "amd64", targetTriple: "x86_64-unknown-linux-gnu"} + instruction := x86ScalarLaneInstruction(t, "MOVQ AX,BX") + known, terminated, err := ctx.lowerVectorScalarFlagCompare(operation, instruction) + _, effectKnown, effectErr := x86TypedCOMIByteEffect(operation, instruction, "amd64") + want := operation == "COMISS" + if known != want || effectKnown != want || (err != nil) != want || (effectErr != nil) != want || terminated || ir.Len() != 0 { + t.Fatal("COMI helper borrowed foreign source/dispatch identity", operation, known, effectKnown, err, effectErr) + } + } +} diff --git a/amd64_comi_review_test.go b/amd64_comi_review_test.go new file mode 100644 index 000000000..49dbb987f --- /dev/null +++ b/amd64_comi_review_test.go @@ -0,0 +1,103 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86COMIModeContextCannotHideMalformedSource(t *testing.T) { + for _, name := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, body := range []string{ + name + " X8,Y1\n", + name + " slot+0(FP)(AX*1),Y1\n", + name + " X8,X1\nRET\n" + name + " X0,Y1\n", + name + " X8,X1\nRET\nTEXT ·later(SB),4,$0-0\n" + name + " X0,Y1\n", + } { + for _, api := range []string{"text", "module", "Go"} { + t.Run(name+"/"+api+"/"+body, func(t *testing.T) { + _, err := x86ScalarFPBitViewAPI(t, "TEXT ·view(SB),4,$0-0\n"+body+"RET\n", + "func view()\nfunc later()\n", "386", "i386-unknown-linux-gnu", api) + if err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mode32 context hid malformed original source: %v", err) + } + }) + } + } + } +} + +func TestX86COMIDirectLowererRequiresNativeTarget(t *testing.T) { + for _, triple := range []string{"", "aarch64-unknown-linux-gnu", "i386-unknown-linux-gnu"} { + var output strings.Builder + ctx := amd64Ctx{b: &output, goarch: "amd64", targetTriple: triple} + instruction := x86ScalarLaneInstruction(t, "COMISS X0,X1") + known, terminated, err := ctx.lowerVectorScalarFlagCompare(instruction.Op, instruction) + if !known || terminated || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatal("direct helper emitted without native target", triple, known, terminated, err, output.String()) + } + } +} + +func TestX86COMIRawVEXLengthIsNotPortableIgnoredBit(t *testing.T) { + for _, name := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+name+" X0,X1\nRET\n") + if len(code) != 5 || code[0] != 0xc5 || code[1]&4 != 0 { + t.Fatalf("actual Go must retain its VEX.L0 encoding: %s %x", name, code) + } + code[1] |= 4 + for _, mode := range []int{32, 64} { + if _, _, matched, err := decodedX86ScalarFlagCompareInstruction(code[:4], mode); !matched || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("raw VEX.L1 normalized to L0 without physical contract: %s mode%d %x %v", name, mode, code, err) + } + var source strings.Builder + source.WriteString("TEXT ·view(SB),4,$0-0\n") + for _, value := range code[:4] { + fmt.Fprintf(&source, "BYTE $%#x\n", value) + } + source.WriteString("RET\n") + arch, triple := "amd64", "x86_64-unknown-linux-gnu" + if mode == 32 { + arch, triple = "386", "i386-unknown-linux-gnu" + } + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source.String(), "func view()\n", arch, triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("raw L1 lost context through public API", name, arch, api, err) + } + } + } + } +} + +func TestX86COMIRawEVEXGenerationSurvivesLowRegisters(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT ·view(SB),4,$0-0\n") + for _, opcode := range []int{0x2e, 0x2f} { + for _, double := range []bool{false, true} { + code := []byte{0x62, 0xf1, 0x7c, 0x08, byte(opcode), 0xc8} + if double { + code[2] |= 0x81 + } + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#x\n", value) + } + } + } + source.WriteString("RET\n") + for _, target := range x86RawReturnTargets { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), "func view()\n", target.arch, target.triple, api) + if err != nil || strings.Count(ir, "{evex} v") != 4 || !strings.Contains(ir, "+avx512f") { + t.Fatal("low-register raw EVEX generation/feature lost", err, ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-comi.ll", "raw-evex-comi.o", ir) + }) + } + } +} diff --git a/amd64_comi_runtime_linux_amd64_test.go b/amd64_comi_runtime_linux_amd64_test.go new file mode 100644 index 000000000..2bacd937f --- /dev/null +++ b/amd64_comi_runtime_linux_amd64_test.go @@ -0,0 +1,28 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86COMIRawNativeCompare(t *testing.T) { + x86COMIRawNativeCompare(t) +} + +func TestCrossLinuxRuntimeMatrixX86COMINativeDeadExceptions(t *testing.T) { + x86COMIDeadFaults(t, true) +} + +func TestCrossLinuxRuntimeMatrixX86COMINativeVectorFlags(t *testing.T) { + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareVEX, x86FloatCompareEVEX} { + t.Run(map[x86FloatCompareRuntimeISA]string{x86FloatCompareVEX: "VEX", x86FloatCompareEVEX: "EVEX"}[axis], func(t *testing.T) { + x86ScalarCompareAFOriginalGo(t, axis) + x86ScalarCompareAFLLVM(t, axis) + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86COMINativeVectorMXCSR(t *testing.T) { + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareVEX, x86FloatCompareEVEX} { + t.Run(map[x86FloatCompareRuntimeISA]string{x86FloatCompareVEX: "VEX", x86FloatCompareEVEX: "EVEX"}[axis], func(t *testing.T) { + x86COMIOriginalConstantsAndMXCSR(t, axis) + }) + } +} diff --git a/amd64_comi_runtime_owner_test.go b/amd64_comi_runtime_owner_test.go new file mode 100644 index 000000000..0ce95443d --- /dev/null +++ b/amd64_comi_runtime_owner_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "reflect" + "testing" +) + +func TestX86COMIRequiredNativeRuntimeOwners(t *testing.T) { + const path = "amd64_comi_runtime_linux_amd64_test.go" + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "386"}, {"linux", "arm64"}, {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + ctx := build.Default + ctx.GOOS, ctx.GOARCH = target.os, target.arch + matched, err := ctx.MatchFile(".", path) + if err != nil || matched != (target.os == "linux" && target.arch == "amd64") { + t.Fatal("native COMI runtime owner selection changed", target, matched, err) + } + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil { + t.Fatal(err) + } + selector, err := x86FloatCompareLinuxCISelector(string(workflow)) + if err != nil { + t.Fatal(err) + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Fatal(err) + } + want := map[string]map[string]string{ + "TestCrossLinuxRuntimeMatrixX86COMIRawNativeCompare": {"x86COMIRawNativeCompare": ""}, + "TestCrossLinuxRuntimeMatrixX86COMINativeDeadExceptions": {"x86COMIDeadFaults": "true"}, + "TestCrossLinuxRuntimeMatrixX86COMINativeVectorFlags": { + "x86ScalarCompareAFOriginalGo": "axis", "x86ScalarCompareAFLLVM": "axis", + }, + "TestCrossLinuxRuntimeMatrixX86COMINativeVectorMXCSR": {"x86COMIOriginalConstantsAndMXCSR": "axis"}, + } + for _, declaration := range file.Decls { + fn, ok := declaration.(*ast.FuncDecl) + if !ok { + continue + } + expected, known := want[fn.Name.Name] + if !known || !selector.MatchString(fn.Name.Name) { + t.Fatal("unexpected or unselected native owner", fn.Name.Name) + } + delete(want, fn.Name.Name) + actual, ranges := make(map[string]string), 0 + ast.Inspect(fn.Body, func(node ast.Node) bool { + switch node := node.(type) { + case *ast.IfStmt, *ast.ReturnStmt, *ast.ForStmt, *ast.BranchStmt: + t.Fatal("native owner cannot conditionally skip or truncate its fixture", fn.Name.Name) + case *ast.RangeStmt: + ranges++ + values, ok := node.X.(*ast.CompositeLit) + if !ok || len(values.Elts) != 2 { + t.Fatal("native vector owner lost exhaustive VEX/EVEX axes") + } + for index, name := range []string{"x86FloatCompareVEX", "x86FloatCompareEVEX"} { + value, ok := values.Elts[index].(*ast.Ident) + if !ok || value.Name != name { + t.Fatal("native vector owner changed ISA selection", fn.Name.Name) + } + } + case *ast.CallExpr: + if method, ok := node.Fun.(*ast.SelectorExpr); ok && method.Sel.Name != "Run" { + t.Fatal("native owner must not replace execution with skip/log helpers", fn.Name.Name) + } + callee, ok := node.Fun.(*ast.Ident) + if !ok { + return true + } + argument, known := expected[callee.Name] + if !known || len(node.Args) != 1 && len(node.Args) != 2 { + t.Fatal("unknown native fixture call", callee.Name) + } + got := "" + if len(node.Args) == 2 { + value, ok := node.Args[1].(*ast.Ident) + if !ok { + t.Fatal("native fixture lost explicit native/ISA argument") + } + got = value.Name + } + if _, duplicate := actual[callee.Name]; duplicate || got != argument { + t.Fatal("duplicated or weakened native fixture", callee.Name, got) + } + actual[callee.Name] = got + } + return true + }) + vector := false + for _, argument := range expected { + vector = vector || argument == "axis" + } + if !reflect.DeepEqual(actual, expected) || (ranges == 1) != vector || ranges > 1 { + t.Fatal("incomplete native fixture dispatch", fn.Name.Name, actual, ranges) + } + } + if len(want) != 0 { + t.Fatal("native COMI execution owners missing", want) + } +} diff --git a/amd64_comi_status_runtime_test.go b/amd64_comi_status_runtime_test.go new file mode 100644 index 000000000..e94f633bf --- /dev/null +++ b/amd64_comi_status_runtime_test.go @@ -0,0 +1,145 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86COMIConstantOracle(axis x86FloatCompareRuntimeISA) (assembly, declarations, original, driver string, functions int) { + var asm, decl, goCalls, cDecl, cCalls strings.Builder + index := 0 + for _, name := range []string{"COMISS", "UCOMISS", "COMISD", "UCOMISD"} { + if axis != x86FloatCompareSSE { + name = "V" + name + } + values, one := []uint64{0x7fc12345, 0x7f812345, 1}, uint64(0x3f800000) + if strings.HasSuffix(name, "D") { + values, one = []uint64{0x7ff8123456789abc, 0x7ff0123456789abc, 1}, 0x3ff0000000000000 + } + forms := []struct{ suffix, source, destination string }{{"", "X0", "X1"}} + if axis == x86FloatCompareEVEX { + forms = []struct{ suffix, source, destination string }{ + {"", "X16", "X17"}, {"", "X30", "X31"}, + {".SAE", "X0", "X1"}, {".SAE", "X16", "X17"}, + } + } + for _, form := range forms { + for _, bits := range values { + for _, dead := range []bool{false, true} { + function := fmt.Sprintf("comi%d", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$8-12\nSTMXCSR saved-4(SP)\nMOVQ out+0(FP),DI\n", function) + fmt.Fprintf(&asm, "MOVQ $%#x,AX\nMOVQ AX,X0\nMOVQ $%#x,AX\nMOVQ AX,X1\n", bits, one) + if form.source != "X0" { + fmt.Fprintf(&asm, "VMOVAPS X0,%s\nVMOVAPS X1,%s\n", form.source, form.destination) + } + fmt.Fprintf(&asm, "LDMXCSR csr+8(FP)\n%s%s %s,%s\n", name, form.suffix, form.source, form.destination) + if dead { + asm.WriteString("MOVQ $0x77,BX\n") + } else { + asm.WriteString("PUSHFQ\nPOPQ BX\nANDQ $0x8d5,BX\n") + } + asm.WriteString("MOVQ BX,(DI)\nSTMXCSR 8(DI)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&decl, "func %s(out *[4]uint64, csr uint32)\n", function) + fmt.Fprintf(&goCalls, "check(%d,%s)\n", index, function) + fmt.Fprintf(&cDecl, "extern void %s(uint64_t*,uint32_t);\n", function) + fmt.Fprintf(&cCalls, "check(%d,%s);\n", index, function) + index++ + } + } + } + } + original = `package main +import "fmt" +func check(index int, fn func(*[4]uint64,uint32)) { + for mode:=0; mode<8; mode++ { + out:=[4]uint64{0x1111111111111111,0x2222222222222222,0x3333333333333333,0x4444444444444444} + csr:=uint32(0x1f80)|uint32(mode&1)<<6|uint32(mode>>1&1)<<15|uint32(mode>>2)<<5 + fn(&out,csr) + fmt.Printf("%d %d %016x %016x %016x %016x\n",index,mode,out[0],out[1],out[2],out[3]) + } +} +` + decl.String() + "func main(){\n" + goCalls.String() + "}\n" + driver = `#include +#include +static void check(int index,void (*fn)(uint64_t*,uint32_t)) { + for(unsigned mode=0; mode<8; mode++) { + uint64_t out[]={UINT64_C(0x1111111111111111),UINT64_C(0x2222222222222222),UINT64_C(0x3333333333333333),UINT64_C(0x4444444444444444)}; + uint32_t csr=0x1f80|((mode&1)<<6)|((mode>>1&1)<<15)|((mode>>2)<<5); + fn(out,csr); + printf("%d %u %016llx %016llx %016llx %016llx\n",index,mode,(unsigned long long)out[0],(unsigned long long)out[1],(unsigned long long)out[2],(unsigned long long)out[3]); + } +} +` + cDecl.String() + "int main(void){\n" + cCalls.String() + "return 0;}\n" + return asm.String(), decl.String(), original, driver, index +} + +func TestCrossLinuxRuntimeMatrixX86COMIOriginalConstantsAndMXCSR(t *testing.T) { + x86COMIOriginalConstantsAndMXCSR(t, x86FloatCompareSSE) +} + +func TestX86COMICompleteStatusOriginalInventory(t *testing.T) { + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareSSE, x86FloatCompareVEX, x86FloatCompareEVEX} { + assembly, _, _, _, functions := x86COMIConstantOracle(axis) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + want := 24 + if axis == x86FloatCompareEVEX { + want = 96 + } + if functions != want { + t.Fatal("changed constant status denominator", axis, functions) + } + } +} + +func x86COMIOriginalConstantsAndMXCSR(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + assembly, declarations, main, driver, count := x86COMIConstantOracle(axis) + original := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, main) + want := strings.Split(strings.TrimSpace(string(original)), "\n") + expected := 24 + if axis == x86FloatCompareEVEX { + expected = 96 + } + if count != expected || len(want) != expected*8 { + t.Fatalf("original Go produced %d functions/%d rows, want %d functions x 8 MXCSR controls", count, len(want), expected) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, assembly, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR_O2=%v/backend%s", optimized, backend), func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "comi-status", input, driver, backend) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM rows=%d original=%d", len(got), len(want)) + } + mismatches := 0 + for index := range want { + if got[index] != want[index] { + if mismatches == 0 { + t.Logf("first mismatch: original %s; LLVM %s", want[index], got[index]) + } + mismatches++ + } + } + if mismatches != 0 { + t.Errorf("original Go FLAGS/MXCSR/canary mismatches=%d/%d", mismatches, len(want)) + } + }) + } + } + }) + } +} diff --git a/amd64_conditional_set_effects_test.go b/amd64_conditional_set_effects_test.go new file mode 100644 index 000000000..37a55626e --- /dev/null +++ b/amd64_conditional_set_effects_test.go @@ -0,0 +1,271 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86ConditionalSetTestNames = strings.Fields("SETCC SETCS SETEQ SETGE SETGT SETHI SETLE SETLS SETLT SETMI SETNE SETOC SETOS SETPC SETPL SETPS") + +func TestX86ConditionalSetActualGoTableAndOrderedForms(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A(SET[A-Z]+), yscond, Pb, opBytes\{0x0f, (0x[0-9a-f]+), 00\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 16 || !strings.Contains(string(data), "var yscond = []ytab{\n\t{Zo_m, 2, argList{Ymb}},\n}") { + t.Fatalf("actual SDK SETcc row/class inventory changed: %d", len(rows)) + } + for _, row := range rows { + if _, registered := x86ConditionalSetConditions[Op(row[1])]; !registered { + t.Fatalf("actual Go row has no shared conditional byte form: %v", row) + } + } + for _, arch := range []string{"amd64", "386"} { + registers := "AX BX CX DX AH AL BH BL CH CL DH DL BP SI DI" + if arch == "amd64" { + registers += " SP BPB SIB DIB R8 R9 R10 R11 R12 R13 R14 R15 R8B R9B R10B R11B R12B R13B R14B R15B" + } + var source strings.Builder + source.WriteString("TEXT ·Value(SB),4,$0-1\n") + for _, op := range x86ConditionalSetTestNames { + for _, destination := range append(strings.Fields(registers), "(AX)", "7(AX)(CX*4)", "state(SB)", "ret0+0(FP)") { + line := op + " " + destination + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-1\n"+line+"\nRET") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); !recognized || err != nil { + t.Fatalf("%s %s shared row: %v", arch, line, err) + } + if !x86SourceStackInstructionKnown(ins, "MOVQ", "Q", arch) { + t.Fatalf("valid lowerer row is missing source-stack registration: %s", line) + } + if isAMD64MemoryOperand(ins.Args[0]) { + width, write, known := x86InstructionMemoryShape(ins, 0, arch) + if width != 1 || !write || !known || x86ScalarMoveMemoryBytes(ins, 0, arch) != 1 { + t.Fatalf("conditional byte memory shape: %s %d/%t/%t", line, width, write, known) + } + if ins.Args[0].Kind == OpFP { + if width, write, known := X86FPAccessBytes(ins, 0, arch); width != 1 || !write || !known { + t.Fatal("FP observation disagrees with the ordered form") + } + } + } + if fmt.Sprintf("%#v", ins) != before { + t.Fatal("typed form changed source identity") + } + source.WriteString(line + "\n") + } + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, source.String(), "", true) + } +} + +func TestX86ConditionalSetRejectsMalformedAndConflictingIdentity(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{"SETNE", "SETNE $1", "SETNE X0", "SETNE M0", "SETNE K1", "SETNE AX,BX", "SETNE.Z AX", "SETNE (AX)(SP*1)"} { + source := "TEXT ·Value(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); !recognized || err == nil { + t.Fatalf("malformed source obtained a conditional byte effect: %s/%s", arch, row) + } + } + } + for _, identity := range []Op{"", "SETNE", "setne", "SETEQ", "SETNE.BAD"} { + var output strings.Builder + ctx := amd64Ctx{b: &output, goarch: "amd64"} + ins := Instr{Op: identity, Args: []Operand{{Kind: OpReg, Reg: AX}}} + // Success is tested at the grammar seam without relying on default + // LLVM flags. Contradictions must fail before even loading flags. + _, recognized, err := x86ConditionalSetProperties("SETNE", ins, "amd64") + valid := identity == "" || strings.EqualFold(string(identity), "SETNE") + if !recognized || (err == nil) != valid { + t.Fatalf("dispatch identity %q: %v", identity, err) + } + if !valid { + if _, _, err := ctx.lowerConditionalSet("SETNE", ins); err == nil || output.Len() != 0 { + t.Fatal("contradictory dispatch emitted IR") + } + } + } +} + +func TestX86ConditionalSetCallerConsumesOnlyDefinedFlagsAndExactByte(t *testing.T) { + for _, op := range x86ConditionalSetTestNames { + for _, destination := range []string{"AX", "AH", "(DX)"} { + body := "CMPL AX,$0\n" + op + " " + destination + if destination != "(DX)" { + body += "\nMOVB " + destination + ",(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + source := strings.NewReplacer("var res float32", "var res uint8", ") float32 {", ") uint8 {").Replace(x86CallerFixtureSource) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s byte admission: %v", body, err) + } + } + } + for name, body := range map[string]string{ + "unknown flags": "SETNE AX\nMOVB AX,(DX)", + "pointer flags": "CMPQ DX,$0\nSETNE AX\nMOVB AX,(DX)", + "partial pointer": "CMPL AX,$0\nSETNE DX\nMOVB AX,(DX)", + "input write": "CMPL AX,$0\nSETNE (DI)", + "unknown address": "CMPL AX,$0\nSETNE (BX)", + "wide result": "CMPL AX,$0\nSETNE 4(DX)", + "undefined upper": "CMPL AX,$0\nSETNE BX\nMOVL BX,(DX)", + "join flags": "CMPL AX,$0\nJE other\nADDQ $0,DX\nother:\nSETNE AX\nMOVB AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved flags/byte/address acquired continuation: %v", err) + } + }) + } + // A private-region pointer is never washed into a number by an eight-bit + // output. The exact AH bit mask preserves unrelated known bytes. + state := x86SourceStackState{numericFlags: true, known: map[Reg]uint8{AX: 1}, callers: map[Reg]x86CallerPointerValue{DX: {argument: 0}}} + ins := Instr{Op: "SETNE", Args: []Operand{{Kind: OpReg, Reg: AH}}} + if _, valid := x86CallerConditionalSetStep(&ins, &state, nil); !valid || state.known[AX] != 3 || !state.numericFlags { + t.Fatal("SETcc does not preserve exact byte knowledge and flags") + } + ins.Args[0].Reg = DX + if _, valid := x86CallerConditionalSetStep(&ins, &state, nil); valid || len(state.callers) != 1 { + t.Fatal("SETcc washed surviving pointer bytes") + } +} + +func TestX86ConditionalSetThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + word, move := 8, "MOVQ" + if target.arch == "386" { + word, move = 4, "MOVL" + } + var body strings.Builder + fmt.Fprintf(&body, "%s x+0(FP),SI\nMOVL $7,AX\nCMPL AX,$3\n", move) + for _, op := range x86ConditionalSetTestNames { + fmt.Fprintf(&body, "%s AX\n%s AH\n%s (SI)\n%s ret0+%d(FP)\n", op, op, op, op, word) + } + body.WriteString("RET") + pkg, file, sig, source := x86ResultInitializationFixture(t, target.arch, "func Value(x *uint8) uint8", body.String()) + goAssemblerOracleOutput(t, target.arch, source, "", true) + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Value": sig}, ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }} + text, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, text, "text") + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, module.String(), "module") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: options.ResolveSym}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, translated.Module.String(), "typed-go") + }) + } +} + +func TestX86ABI0OccupiedPointerAndSetRetainUnavailableInputs(t *testing.T) { + for name, body := range map[string]string{ + "pointer bytes": "MOVQ x+0(FP),AX\nMOVL AX,ret0+24(FP)\nRET", + "partial pointer bytes": "MOVB x+0(FP),AX\nMOVL AX,ret0+24(FP)\nRET", + "pointer dereference": "MOVQ x+0(FP),AX\nMOVL (AX),BX\nMOVL BX,ret0+24(FP)\nRET", + "unknown flags": "SETNE AX\nMOVB AX,ret0+24(FP)\nRET", + "unknown upper": "MOVL $1,BX\nCMPL BX,$0\nSETNE AX\nMOVL AX,ret0+24(FP)\nRET", + "missing result bytes": "MOVL $1,AX\nCMPL AX,$0\nSETNE ret0+24(FP)\nRET", + "flags join": "MOVL $1,AX\nCMPL AX,$0\nJE other\nSETNE ret0+24(FP)\nother:\nRET", + "call": "MOVL $1,ret0+24(FP)\nCALL ·other(SB)\nRET", + } { + t.Run(name, func(t *testing.T) { + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value(x []uint32) uint32", body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + before := fmt.Sprintf("%#v %#v", file, sig) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("occupied pointer or byte output manufactured input/result: %v", err) + } + if !reflect.DeepEqual(before, fmt.Sprintf("%#v %#v", file, sig)) { + t.Fatal("observer mutated source/layout") + } + }) + } + // Result initialization observes reachable execution, not unused source + // bytes. A complete result followed by an unconditional jump does not read + // the dead pointer input. Transporting an unused opaque pointer is also + // legal; consuming its bytes as a numeric result must still fail. + const dead = "MOVL $1,ret0+24(FP)\nJMP end\nMOVQ x+0(FP),AX\nend:\nRET" + for _, fixture := range []struct { + name, body string + context bool + }{ + {"dead pointer bytes", dead, false}, + {"reachable opaque pointer unused", strings.Replace(dead, "JMP end", "JMP input\ninput:", 1), false}, + {"reachable pointer result", strings.Replace(strings.Replace(dead, "JMP end", "JMP input\ninput:", 1), + "MOVQ x+0(FP),AX", "MOVQ x+0(FP),AX\nMOVL AX,ret0+24(FP)", 1), true}, + } { + t.Run(fixture.name, func(t *testing.T) { + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value(x []uint32) uint32", fixture.body) + goAssemblerOracleOutput(t, "amd64", source, "", true) + before := fmt.Sprintf("%#v %#v", file, sig) + err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64") + if fixture.context && !errors.Is(err, ErrProbeNeedsContext) || !fixture.context && err != nil { + t.Fatalf("reachable result/input boundary changed: %v", err) + } + if before != fmt.Sprintf("%#v %#v", file, sig) { + t.Fatal("result observer changed source or manufactured pointer layout") + } + }) + } + // A derived pointer layout still cannot become a pointer result contract. + pkg := mustGoPackage(t, "example.invalid/pointer", "package pointer\nfunc Value(x *uint32) *uint32\n") + sig, err := DeriveGoFuncABI0Signature("Value", pkg.Types.Scope().Lookup("Value").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$0-16\nMOVQ x+0(FP),AX\nMOVQ AX,ret0+8(FP)\nRET") + if err != nil { + t.Fatal(err) + } + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("pointer result gained numeric result authority") + } +} diff --git a/amd64_conditional_set_forms.go b/amd64_conditional_set_forms.go new file mode 100644 index 000000000..7be3efa52 --- /dev/null +++ b/amd64_conditional_set_forms.go @@ -0,0 +1,76 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Both reviewed Go encoders use yscond: one Ymb destination, one byte +// written, no input data operand and no flags output. This grammar establishes +// neither initialized flags nor any authority for the destination address. +var x86ConditionalSetConditions = map[Op]string{ + "SETCC": "CC", "SETCS": "CS", "SETEQ": "EQ", "SETGE": "GE", + "SETGT": "GT", "SETHI": "HI", "SETLE": "LE", "SETLS": "LS", + "SETLT": "LT", "SETMI": "MI", "SETNE": "NE", "SETOC": "OC", + "SETOS": "OS", "SETPC": "PC", "SETPL": "PL", "SETPS": "PS", +} + +type x86ConditionalSetForm struct { + condition string + destination Operand +} + +func x86ConditionalSetProperties(op Op, ins Instr, arch string) (x86ConditionalSetForm, bool, error) { + var form x86ConditionalSetForm + name := strings.ToUpper(string(op)) + base := strings.SplitN(name, ".", 2)[0] + condition, recognized := x86ConditionalSetConditions[Op(base)] + if !recognized { + return form, false, nil + } + fail := func(reason string) (x86ConditionalSetForm, bool, error) { + return form, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if arch != "amd64" && arch != "386" || name != base || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail("has an unbound mode, suffix or dispatch identity") + } + if len(ins.Args) != 1 { + return fail("expects one Ymb destination") + } + dst := ins.Args[0] + if dst.Kind == OpReg { + if !isGoYmbRegisterForArch(dst.Reg, arch) { + return fail("register is outside the Go Ymb class") + } + } else if !isAMD64MemoryOperand(dst) { + return fail("expects a Ymb register or memory destination") + } + if dst.Kind == OpMem { + memory := dst.Mem + if err := x86SymbolIndexedMemoryError(memory, arch); err != nil { + return form, true, err + } + if arch == "386" && strings.HasSuffix(memory.Sym, "(SB)") { + memory.Base = "" // The named-memory encoder discards this annotation. + } + if !x86MemoryRegistersValidForArch(memory, arch) || memory.Index == SP || + memory.Index != "" && memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 || + memory.Off != int64(int32(memory.Off)) && !(arch == "386" && memory.Off == int64(uint32(memory.Off))) { + return fail("has an unencodable memory address") + } + } + form.condition, form.destination = condition, dst + return form, true, nil +} + +func x86CallerConditionalSetStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (bool, bool) { + form, recognized, err := x86ConditionalSetProperties(ins.Op, *ins, "amd64") + if !recognized || err != nil || !x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(form.condition)) { + return recognized, false + } + if form.destination.Kind == OpReg { + return true, x86CallerNumericWrite(form.destination.Reg, 1, state) + } + return true, x86CallerOrdinaryMemoryStep(ins, 0, state, proof, 1, true) +} diff --git a/amd64_conditional_set_runtime_test.go b/amd64_conditional_set_runtime_test.go new file mode 100644 index 000000000..3a9c4ce6e --- /dev/null +++ b/amd64_conditional_set_runtime_test.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// This independent boolean reference uses architectural flags, not the +// translator's condition builder or the form map's condition strings. +const x86ConditionalSetIndependentC = ` +#include +#include +#include +static unsigned truth(unsigned condition,uint64_t flags) { + unsigned c=flags&1,p=flags>>2&1,z=flags>>6&1,s=flags>>7&1,o=flags>>11&1; + switch(condition) { + case 0:return !c;case 1:return c;case 2:return z;case 3:return s==o; + case 4:return !z&&s==o;case 5:return !c&&!z;case 6:return z||s!=o; + case 7:return c||z;case 8:return s!=o;case 9:return s;case 10:return !z; + case 11:return !o;case 12:return o;case 13:return !p;case 14:return !s;case 15:return p; + } return 2; +} +` + +func TestCrossLinuxRuntimeMatrixX86ConditionalSetAllFlagsAliasesAndCanaries(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, tests, c strings.Builder + declarations.WriteString("package conditional\n") + tests.WriteString(`package conditional +import("testing";"encoding/binary") +func TestOriginal(t *testing.T) { + for sample:=0;sample<64;sample++ { + flags:=uint64(2) + for bit,index:=range []uint{0,2,4,6,7,11} {if sample&(1<> (i / form.lane)) & 1 != 0 { + want = src[i] + } else if form.zero { + want = 0 + } + if got := out[form.offset+i]; got != want { + t.Fatalf("original move offset=%d byte=%d: got=%02x want=%02x", form.offset, i, got, want) + } + } + } + for i, value := range storage { + if (i < offset || i >= offset+len(out)) && value != 0xa7 { + t.Fatalf("original move output canary changed at %d", i) + } + } +} + +func TestOriginalSameWidthPackedConversion(t *testing.T) { + var ints [16]int32 + floats := [16]float32{ + 1.5,-1.5,2.75,-2.75,0.5,-0.5,3.25,-3.25, + 4.5,-4.5,5.75,-5.75,6.5,-6.5,7.25,-7.25, + } + var squares32 [16]float32 + var squares64 [8]float64 + for i:=0;i<16;i++ { + ints[i]=int32(i-8) + squares32[i]=float32((i+1)*(i+1)) + } + for i:=0;i<8;i++ { + squares64[i]=float64((i+1)*(i+1)) + } + nearest := [16]int32{2,-2,3,-3,0,0,3,-3,4,-4,6,-6,6,-6,7,-7} + truncated := [16]int32{1,-1,2,-2,0,0,3,-3,4,-4,5,-5,6,-6,7,-7} + down := [16]int32{1,-2,2,-3,0,-1,3,-4,4,-5,5,-6,6,-7,7,-8} + up := [16]int32{2,-1,3,-2,1,0,4,-3,5,-4,6,-5,7,-6,8,-7} + var guarded [578]byte + guarded[0],guarded[577]=0xa7,0xb9 + output:=guarded[1:577] + sameWidthPackedConversionSemantics((*[576]byte)(output),&ints,&floats,&squares32,&squares64) + if guarded[0]!=0xa7 || guarded[577]!=0xb9 { + t.Fatal("original wide conversion output canary changed") + } + single := func(offset int) float32 { + return math.Float32frombits(binary.LittleEndian.Uint32(output[offset:])) + } + integer := func(offset int) int32 { + return int32(binary.LittleEndian.Uint32(output[offset:])) + } + for i:=0;i<16;i++ { + if single(4*i)!=float32(ints[i]) || integer(64+4*i)!=nearest[i] || + integer(128+4*i)!=truncated[i] || single(192+4*i)!=float32(i+1) || + integer(320+4*i)!=nearest[i] || integer(384+4*i)!=down[i] || + integer(448+4*i)!=up[i] || integer(512+4*i)!=truncated[i] { + t.Fatalf("original wide conversion lane %d output=%x",i,output) + } + } + for i:=0;i<8;i++ { + value:=math.Float64frombits(binary.LittleEndian.Uint64(output[256+8*i:])) + if value!=float64(i+1) { + t.Fatalf("original wide sqrt lane %d value=%v",i,value) + } + } +} +` diff --git a/amd64_conformance_test.go b/amd64_conformance_test.go index f481eb5d6..13cad5a4c 100644 --- a/amd64_conformance_test.go +++ b/amd64_conformance_test.go @@ -1,8 +1,7 @@ -//go:build !llgo - package plan9asm import ( + "fmt" "os" "os/exec" "path/filepath" @@ -15,9 +14,9 @@ func TestAMD64ConformanceNativeGo(t *testing.T) { if runtime.GOARCH != "amd64" && !(runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable()) { t.Skip("native Go assembler oracle requires an amd64 host") } - cmd := exec.Command("go", "test", "./testdata/conformance/amd64") + cmd := exec.Command("go", "test", "-count=1", "./testdata/conformance/amd64") if runtime.GOARCH != "amd64" { - cmd.Env = append(os.Environ(), "GOARCH=amd64") + cmd.Env = append(os.Environ(), "GOARCH=amd64", "PLAN9ASM_ROSETTA=1") } if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("native Go conformance failed: %v\n%s", err, out) @@ -25,14 +24,11 @@ func TestAMD64ConformanceNativeGo(t *testing.T) { } func TestAMD64ConformanceLLVMRuntime(t *testing.T) { - crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() - if runtime.GOARCH != "amd64" && !crossRosetta { - t.Skip("runtime execution test only runs on amd64 host") - } - llc, clang, ok := findLlcAndClang(t) - if !ok { - t.Skip("llc/clang not found") - } + amd64ConformanceLLVMRuntime(t, x86FloatCompareRuntimeTools(t, x86FloatCompareSSE), false) +} + +func amd64ConformanceLLVMRuntime(t *testing.T, execution x86FloatCompareExecution, nativeWide bool) { + t.Helper() src, err := os.ReadFile(filepath.Join("testdata", "conformance", "amd64", "conformance_amd64.s")) if err != nil { @@ -47,12 +43,7 @@ func TestAMD64ConformanceLLVMRuntime(t *testing.T) { {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, }} - triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) - var runPrefix []string - if crossRosetta { - triple = "x86_64-apple-macosx" - runPrefix = []string{"/usr/bin/arch", "-x86_64"} - } + triple := execution.triple ll, err := Translate(file, Options{ TargetTriple: triple, ResolveSym: resolve, @@ -133,6 +124,238 @@ func TestAMD64ConformanceLLVMRuntime(t *testing.T) { {Offset: 24, Type: I64, Index: 3, Field: -1}, }}, }, + "scalarShiftRotateSemantics": { + Name: "scalarShiftRotateSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "incDecSemantics": { + Name: "incDecSemantics", + Args: []LLVMType{Ptr, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }}, + }, + "scalarAddSubSemantics": { + Name: "scalarAddSubSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "compareExchangeScalarSemantics": { + Name: "compareExchangeScalarSemantics", + Args: []LLVMType{Ptr, Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }, + "compareExchangePairSemantics": { + Name: "compareExchangePairSemantics", + Args: []LLVMType{Ptr, Ptr, I64, I64, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }}, + }, + "parallelBitSemantics": { + Name: "parallelBitSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "bitTestRegisterSemantics": { + Name: "bitTestRegisterSemantics", + Args: []LLVMType{Ptr, I64, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "bitTestMemorySemantics": { + Name: "bitTestMemorySemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "packedWordMultiplyAddSemantics": { + Name: "packedWordMultiplyAddSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedUnsignedSignedByteMultiplyAddSemantics": { + Name: "packedUnsignedSignedByteMultiplyAddSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedDotProductSemantics": { + Name: "packedDotProductSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + }}, + }, + "packedIntegerMinMaxSemantics": { + Name: "packedIntegerMinMaxSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedWordMultiplySemantics": { + Name: "packedWordMultiplySemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedFloatShuffleSemantics": { + Name: "packedFloatShuffleSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "inLaneFloatingPermuteSemantics": { + Name: "inLaneFloatingPermuteSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "immediatePackedBlendSemantics": { + Name: "immediatePackedBlendSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedUnpackSemantics": { + Name: "packedUnpackSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "qwordPermuteSemantics": { + Name: "qwordPermuteSemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "qwordVariablePermuteSemantics": { + Name: "qwordVariablePermuteSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedFloatMoveVEXSemantics": { + Name: "packedFloatMoveVEXSemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "packedFloatMoveMaskSemantics": { + Name: "packedFloatMoveMaskSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "packedSignExtendMoveVEXSemantics": { + Name: "packedSignExtendMoveVEXSemantics", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "packedSignExtendMoveMaskSemantics": { + Name: "packedSignExtendMoveMaskSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, "goHexVectorOps": { Name: "goHexVectorOps", Args: []LLVMType{Ptr, Ptr, Ptr}, @@ -161,12 +384,141 @@ func TestAMD64ConformanceLLVMRuntime(t *testing.T) { Ret: Void, Frame: ptrParams, }, + "packedSubtractSemantics": { + Name: "packedSubtractSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "expandedEcosystemVectors": { + Name: "expandedEcosystemVectors", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + "parityBranches": { + Name: "parityBranches", + Args: []LLVMType{I8}, + Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }, + }, + "unorderedBranch": { + Name: "unorderedBranch", + Args: []LLVMType{LLVMType("double")}, + Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: LLVMType("double"), Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }, + }, + "packedFloatToDwordModes": { + Name: "packedFloatToDwordModes", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "sameWidthPackedConversionSemantics": { + Name: "sameWidthPackedConversionSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }, + "packedSingleToDouble": { + Name: "packedSingleToDouble", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "conditionalMoveCodes": { + Name: "conditionalMoveCodes", + Args: []LLVMType{Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + "fma3Semantics": { + Name: "fma3Semantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + }}, + }, + "binaryFloatingSemantics": { + Name: "binaryFloatingSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }, + "horizontalFloatingSemantics": { + Name: "horizontalFloatingSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }, }, }) if err != nil { t.Fatal(err) } - mainC := ` + // Numeric fixtures must return normally. Go-accepted EVEX zeroing memory + // stores raise #UD and belong in the separate exhaustive fault oracle, + // not in a fixture whose C driver expects an arithmetic result. + if strings.Contains(ll, `asm sideeffect "ud2"`) { + t.Fatal("numeric conformance fixture contains an architectural trap; preserve it in the separate expected-fault oracle") + } + // Every source function reaches the LLVM object on both execution axes. + // Only native aligned moves and wide conversions require AVX512 execution; + // their complete assertions also run against the original Go object there. + execution.compileAndRunLLVM(t, "amd64_conformance", ll, amd64ConformanceRuntimeC(nativeWide), "-O2") + t.Logf("complete original assembly object; native-wide runtime=%v, actual transport=%v", nativeWide, execution.prefix) +} + +func amd64ConformanceRuntimeC(nativeWide bool) string { + return fmt.Sprintf("#define PLAN9ASM_CONFORMANCE_NATIVE_WIDE %d\n", x86FloatAddSubBoolInt(nativeWide)) + ` #include extern void byteMemory(uint8_t *p, uint8_t value); extern void byteFlags(uint8_t *p, uint8_t *flags); @@ -176,9 +528,61 @@ extern uint32_t shiftLegacyThreeOperand(uint32_t src, uint32_t dst, uint32_t amo extern uint64_t clearTopBit(uint64_t value); extern void doubleShift32(uint32_t *out, uint32_t src, uint32_t dst, uint32_t amount); extern void doubleShift64(uint64_t *out, uint64_t src, uint64_t dst, uint64_t amount); +extern void scalarShiftRotateSemantics(uint64_t *out, uint64_t value, uint64_t count); +extern void incDecSemantics(uint64_t *out, uint64_t value); +extern void scalarAddSubSemantics(uint64_t *out, uint64_t value, uint64_t source); +extern void compareExchangeScalarSemantics(uint64_t *out, uint64_t *memory, uint64_t expected, uint64_t desired); +extern void compareExchangePairSemantics(uint64_t *out, uint64_t *memory, uint64_t expected_low, uint64_t expected_high, uint64_t desired_low, uint64_t desired_high); +extern void parallelBitSemantics(uint64_t *out, uint64_t mask, uint64_t source); +extern void bitTestRegisterSemantics(uint64_t *out, uint64_t value, uint64_t index); +extern void bitTestMemorySemantics(uint8_t *out, uint64_t *data); +extern void packedWordMultiplyAddSemantics(int32_t *out, int16_t *a, int16_t *b); +extern void packedUnsignedSignedByteMultiplyAddSemantics(int16_t *out, int8_t *signed_input, uint8_t *unsigned_input); +extern void packedDotProductSemantics(int32_t *out, int8_t *signed_bytes, uint8_t *unsigned_bytes, int16_t *words_a, int16_t *words_b, int32_t *accumulator); +extern void packedIntegerMinMaxSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void packedWordMultiplySemantics(uint16_t *out, int16_t *a, int16_t *b); +extern void packedFloatShuffleSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void inLaneFloatingPermuteSemantics(uint8_t *out, uint8_t *data, uint8_t *control); +extern void immediatePackedBlendSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void packedUnpackSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void qwordPermuteSemantics(uint64_t *out, uint64_t *data); +extern void qwordVariablePermuteSemantics(uint64_t *out, uint64_t *data, uint64_t *control); +extern void packedFloatMoveVEXSemantics(uint8_t *out, uint8_t *src); +extern void packedFloatMoveMaskSemantics(uint8_t *out, uint8_t *src, uint8_t *old); +extern void packedSignExtendMoveVEXSemantics(uint8_t *out, uint8_t *src); +extern void packedSignExtendMoveMaskSemantics(uint8_t *out, uint8_t *src, uint8_t *old); extern void goHexVectorOps(uint8_t *out, uint8_t *a, uint8_t *b); extern uint64_t goHexWordOps(uint64_t value, uint64_t count); extern void packedArithmeticShift32(int32_t *out, int32_t *src); +extern void packedSubtractSemantics(uint8_t *out, uint8_t *a, uint8_t *b); +extern void expandedEcosystemVectors(uint8_t *out, uint8_t *a, uint8_t *b); +extern uint64_t parityBranches(uint8_t value); +extern uint64_t unorderedBranch(double value); +extern void packedFloatToDwordModes(int32_t *out, float *src); +extern void sameWidthPackedConversionSemantics(uint8_t *out, int32_t *ints, float *floats, float *squares32, double *squares64); +extern void packedSingleToDouble(double *out, float *src); +extern void conditionalMoveCodes(uint64_t *out, uint64_t *src); +extern void fma3Semantics(uint8_t *out, double *a64, double *b64, double *c64, float *a32, float *b32, float *c32); +extern void binaryFloatingSemantics(uint8_t *out, double *a64, double *b64, float *a32, float *b32); +extern void horizontalFloatingSemantics(uint8_t *out, float *a32, float *b32, double *a64, double *b64); + +static uint16_t load16(const uint8_t *p) { + return (uint16_t)p[0] | (uint16_t)p[1] << 8; +} +static uint32_t load32(const uint8_t *p) { + return (uint32_t)load16(p) | (uint32_t)load16(p + 2) << 16; +} +static uint64_t load64(const uint8_t *p) { + return (uint64_t)load32(p) | (uint64_t)load32(p + 4) << 32; +} +static float loadf32(const uint8_t *p) { + union { uint32_t bits; float value; } converted = {load32(p)}; + return converted.value; +} +static double loadf64(const uint8_t *p) { + union { uint64_t bits; double value; } converted = {load64(p)}; + return converted.value; +} static uint32_t shld32(uint32_t src, uint32_t dst, uint32_t count) { count &= 31; @@ -196,6 +600,77 @@ static uint64_t shrd64(uint64_t src, uint64_t dst, uint64_t count) { count &= 63; return count == 0 ? dst : (dst >> count) | (src << (64 - count)); } +static uint64_t rol(uint64_t value, unsigned bits, uint64_t count) { + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + count %= bits; + value &= mask; + return count == 0 ? value : ((value << count) | (value >> (bits - count))) & mask; +} +static uint64_t ror(uint64_t value, unsigned bits, uint64_t count) { + return rol(value, bits, bits - count % bits); +} +static uint64_t replace_low(uint64_t value, uint64_t low, unsigned bits) { + uint64_t mask = (UINT64_C(1) << bits) - 1; + return (value & ~mask) | (low & mask); +} + +static uint64_t reference_scalar_add_sub(int add, unsigned bits, uint64_t value, uint64_t source, + int register_destination, uint64_t *flags) { + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + uint64_t destination_low = value & mask, source_low = source & mask; + uint64_t result_low = add ? (destination_low + source_low) & mask : (destination_low - source_low) & mask; + uint64_t carry = add ? result_low < destination_low : destination_low < source_low; + uint64_t sign_bit = UINT64_C(1) << (bits - 1); + uint64_t overflow = add + ? ((~(destination_low ^ source_low) & (destination_low ^ result_low) & sign_bit) != 0) + : (((destination_low ^ source_low) & (destination_low ^ result_low) & sign_bit) != 0); + uint8_t low_byte = (uint8_t)result_low; + uint64_t parity = 1; + while (low_byte != 0) { + parity ^= 1; + low_byte &= low_byte - 1; + } + *flags = carry | overflow << 8 | (result_low == 0) << 16 | + ((result_low & sign_bit) != 0) << 24 | parity << 32; + if (bits == 64 || (register_destination && bits == 32)) return result_low; + return (value & ~mask) | result_low; +} + +static uint64_t reference_pdep(uint64_t mask, uint64_t source) { + uint64_t result = 0; + while (mask != 0) { + uint64_t lowest = mask & (0 - mask); + if (source & 1) result |= lowest; + source >>= 1; + mask &= mask - 1; + } + return result; +} + +static uint64_t reference_pext(uint64_t mask, uint64_t source) { + uint64_t result = 0, output_bit = 1; + while (mask != 0) { + uint64_t lowest = mask & (0 - mask); + if (source & lowest) result |= output_bit; + output_bit <<= 1; + mask &= mask - 1; + } + return result; +} + +static uint64_t reference_bit_test_register(int operation, unsigned bits, uint64_t value, uint64_t index, uint64_t *carry) { + uint64_t bit = index & (bits - 1); + *carry = (value >> bit) & 1; + if (operation == 0) return value; + uint64_t mask = UINT64_C(1) << bit; + uint64_t low = value; + if (operation == 1) low ^= mask; + if (operation == 2) low &= ~mask; + if (operation == 3) low |= mask; + if (bits == 16) return (value & ~UINT64_C(0xffff)) | (low & UINT64_C(0xffff)); + if (bits == 32) return (uint32_t)low; + return low; +} int main(void) { uint8_t bytes[4] = {0x10, 0xf0, 0x5a, 0xa0}; @@ -265,6 +740,538 @@ int main(void) { return 50 + i; } } + const uint64_t scalar_value = UINT64_C(0x8123456789abcdef); + const uint64_t word_dst = UINT64_C(0xfedcba9876543210); + for (uint64_t count = 0; count < 16; count++) { + uint64_t out[14] = {0}, want[14] = {0}; + scalarShiftRotateSemantics(out, scalar_value, count); + uint64_t hardware_count = count & 31; + uint64_t byte_logical = hardware_count < 8 ? (uint8_t)scalar_value >> hardware_count : 0; + uint64_t byte_arithmetic = hardware_count < 8 ? (uint8_t)((int8_t)scalar_value >> hardware_count) : UINT8_MAX; + uint64_t word_count = count & 15; + uint16_t shldw = (uint16_t)word_dst, shrdw = (uint16_t)word_dst; + if (word_count != 0) { + shldw = (uint16_t)(((uint16_t)word_dst << word_count) | ((uint16_t)scalar_value >> (16 - word_count))); + shrdw = (uint16_t)(((uint16_t)word_dst >> word_count) | ((uint16_t)scalar_value << (16 - word_count))); + } + want[0] = replace_low(scalar_value, rol(scalar_value, 8, count), 8); + want[1] = replace_low(scalar_value, ror(scalar_value, 8, count), 8); + want[2] = replace_low(scalar_value, byte_arithmetic, 8); + want[3] = replace_low(scalar_value, byte_logical, 8); + want[4] = replace_low(scalar_value, rol(scalar_value, 16, count), 16); + want[5] = replace_low(scalar_value, (uint16_t)((int16_t)scalar_value >> hardware_count), 16); + want[6] = (uint32_t)scalar_value << hardware_count; + want[7] = ror(scalar_value, 64, count); + want[8] = replace_low(scalar_value, byte_logical, 8); + want[9] = replace_low(scalar_value, rol(scalar_value, 16, count), 16); + want[10] = replace_low(word_dst, shldw, 16); + want[11] = replace_low(word_dst, shrdw, 16); + want[12] = replace_low(scalar_value, (uint8_t)scalar_value << 3, 8); + want[13] = (uint32_t)((int32_t)scalar_value >> 3); + for (int i = 0; i < 14; i++) + if (out[i] != want[i]) return 196 + i; + } + const uint64_t inc_values[] = {0, 1, 0x7f, 0xff, 0xffff, 0xffffffff, UINT64_C(0x8123456789abcdef)}; + for (unsigned v = 0; v < sizeof(inc_values) / sizeof(inc_values[0]); v++) { + uint64_t value = inc_values[v], out[12] = {0}; + incDecSemantics(out, value); + uint64_t want[12] = { + replace_low(value, (uint8_t)value + 1, 8), + replace_low(value, (uint8_t)value - 1, 8), + replace_low(value, (uint16_t)value + 1, 16), + replace_low(value, (uint16_t)value - 1, 16), + (uint32_t)value + 1, (uint32_t)value - 1, value + 1, value - 1, + replace_low(value, (uint8_t)value + 1, 8), + replace_low(value, (uint16_t)value - 1, 16), + UINT64_C(0x0000000000010101), UINT64_C(0x0000000101000001), + }; + for (int i = 0; i < 12; i++) + if (out[i] != want[i]) return 220 + i; + } + const uint64_t scalar_add_sub_cases[][2] = { + {0, 0}, {UINT64_MAX, 1}, {0x7f, 1}, {0x7fff, 1}, + {0x7fffffff, 1}, {UINT64_C(0x7fffffffffffffff), 1}, + {UINT64_C(0x8123456789abcdef), UINT64_C(0xfedcba9876543210)}, + }; + const unsigned scalar_add_sub_widths[] = {8, 16, 32, 64}; + for (unsigned v = 0; v < sizeof(scalar_add_sub_cases) / sizeof(scalar_add_sub_cases[0]); v++) { + uint64_t value = scalar_add_sub_cases[v][0], source = scalar_add_sub_cases[v][1]; + uint64_t out[16] = {0}, want[16] = {0}; + scalarAddSubSemantics(out, value, source); + for (unsigned width_index = 0; width_index < 4; width_index++) { + unsigned bits = scalar_add_sub_widths[width_index]; + want[width_index * 4] = reference_scalar_add_sub(1, bits, value, source, 1, &want[width_index * 4 + 1]); + want[width_index * 4 + 2] = reference_scalar_add_sub(0, bits, value, UINT64_MAX, 0, &want[width_index * 4 + 3]); + } + for (int i = 0; i < 16; i++) + if (out[i] != want[i]) return 232 + i; + } + const uint64_t cmpxchg_expected_values[] = { + 0, UINT64_MAX, 0x7f, 0x7fff, 0x7fffffff, + UINT64_C(0x7fffffffffffffff), UINT64_C(0x8123456789abcdef), + }; + const uint64_t cmpxchg_desired = UINT64_C(0xfedcba9876543210); + for (unsigned v = 0; v < sizeof(cmpxchg_expected_values) / sizeof(cmpxchg_expected_values[0]); v++) { + uint64_t expected = cmpxchg_expected_values[v], memory[8] = {0}, initial[8] = {0}; + uint64_t out[24] = {0}, want[24] = {0}; + for (unsigned width_index = 0; width_index < 4; width_index++) { + unsigned bits = scalar_add_sub_widths[width_index]; + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + uint64_t base = UINT64_C(0xa5a5a5a5a5a5a5a5) & ~mask; + memory[width_index * 2] = base | (expected & mask); + memory[width_index * 2 + 1] = base | ((expected ^ (UINT64_C(1) << (bits - 1))) & mask); + initial[width_index * 2] = memory[width_index * 2]; + initial[width_index * 2 + 1] = memory[width_index * 2 + 1]; + } + compareExchangeScalarSemantics(out, memory, expected, cmpxchg_desired); + for (unsigned width_index = 0; width_index < 4; width_index++) { + unsigned bits = scalar_add_sub_widths[width_index]; + uint64_t mask = bits == 64 ? UINT64_MAX : (UINT64_C(1) << bits) - 1; + uint64_t success_initial = initial[width_index * 2]; + uint64_t failure_initial = initial[width_index * 2 + 1]; + unsigned output_index = width_index * 6; + want[output_index] = (success_initial & ~mask) | (cmpxchg_desired & mask); + want[output_index + 1] = expected; + reference_scalar_add_sub(0, bits, success_initial, expected, 1, &want[output_index + 2]); + want[output_index + 3] = failure_initial; + if (bits == 8 || bits == 16) want[output_index + 4] = (expected & ~mask) | (failure_initial & mask); + if (bits == 32) want[output_index + 4] = (uint32_t)failure_initial; + if (bits == 64) want[output_index + 4] = failure_initial; + reference_scalar_add_sub(0, bits, expected, failure_initial, 1, &want[output_index + 5]); + } + for (int i = 0; i < 24; i++) + if (out[i] != want[i]) return 248 + i; + } + const uint64_t cmpxchg_expected_low = UINT64_C(0xaaaaaaaa11223344); + const uint64_t cmpxchg_expected_high = UINT64_C(0xbbbbbbbb55667788); + const uint64_t cmpxchg_desired_low = UINT64_C(0x0123456789abcdef); + const uint64_t cmpxchg_desired_high = UINT64_C(0xfedcba9876543210); + for (int success = 0; success <= 1; success++) { + _Alignas(16) uint64_t memory[4] = {0}, out[9] = {0}, want[9] = {0}; + if (success) { + memory[0] = cmpxchg_expected_low; + memory[1] = cmpxchg_expected_high; + memory[2] = (cmpxchg_expected_low & UINT32_MAX) | (cmpxchg_expected_high & UINT32_MAX) << 32; + } else { + memory[0] = UINT64_C(0x1020304050607080); + memory[1] = UINT64_C(0x90a0b0c0d0e0f001); + memory[2] = UINT64_C(0x8877665544332211); + } + uint64_t old16_low = memory[0], old16_high = memory[1], old8 = memory[2]; + compareExchangePairSemantics(out, memory, cmpxchg_expected_low, cmpxchg_expected_high, + cmpxchg_desired_low, cmpxchg_desired_high); + if (success) { + want[0] = (cmpxchg_desired_low & UINT32_MAX) | (cmpxchg_desired_high & UINT32_MAX) << 32; + want[1] = cmpxchg_expected_low; + want[2] = cmpxchg_expected_high; + want[3] = 1; + want[4] = cmpxchg_desired_low; + want[5] = cmpxchg_desired_high; + want[6] = cmpxchg_expected_low; + want[7] = cmpxchg_expected_high; + want[8] = 1; + } else { + want[0] = old8; + want[1] = (uint32_t)old8; + want[2] = (uint32_t)(old8 >> 32); + want[4] = old16_low; + want[5] = old16_high; + want[6] = old16_low; + want[7] = old16_high; + } + for (int i = 0; i < 9; i++) + if (out[i] != want[i]) return 272 + i; + } + const uint64_t parallel_cases[][2] = { + {0, UINT64_C(0x0123456789abcdef)}, + {UINT64_MAX, UINT64_C(0x0123456789abcdef)}, + {UINT64_C(0xaaaaaaaaaaaaaaaa), UINT64_C(0x0123456789abcdef)}, + {UINT64_C(0x8040201008040201), UINT64_C(0xfedcba9876543210)}, + {UINT64_C(0x8000000100000081), UINT64_MAX}, + }; + for (unsigned v = 0; v < sizeof(parallel_cases) / sizeof(parallel_cases[0]); v++) { + uint64_t mask = parallel_cases[v][0], source = parallel_cases[v][1], out[8] = {0}; + parallelBitSemantics(out, mask, source); + uint64_t mask32 = (uint32_t)mask, source32 = (uint32_t)source; + uint64_t want[8] = { + reference_pdep(mask, source), reference_pdep(mask, source), + reference_pext(mask, source), reference_pext(mask, source), + reference_pdep(mask32, source32), reference_pdep(mask32, source32), + reference_pext(mask32, source32), reference_pext(mask32, source32), + }; + for (int i = 0; i < 8; i++) + if (out[i] != want[i]) return 124; + } + const uint64_t bit_test_value = UINT64_C(0xfedcba9876543210); + const uint64_t bit_test_index = 21; + uint64_t bit_test_out[24] = {0}; + bitTestRegisterSemantics(bit_test_out, bit_test_value, bit_test_index); + const unsigned bit_test_widths[3] = {16, 32, 64}; + for (int width_index = 0; width_index < 3; width_index++) { + unsigned bits = bit_test_widths[width_index]; + for (int operation = 0; operation < 4; operation++) { + uint64_t operand_index = bit_test_index; + if (operation == 0) operand_index = bits - 1; + if (operation == 2) operand_index = 4; + uint64_t carry = 0; + uint64_t result = reference_bit_test_register(operation, bits, bit_test_value, operand_index, &carry); + int output_index = 2 * (width_index * 4 + operation); + if (bit_test_out[output_index] != result || bit_test_out[output_index + 1] != carry) return 125; + } + } + uint64_t bit_test_data[9] = { + UINT64_MAX, UINT64_C(1) << 17, 0, + UINT64_MAX, UINT64_C(1) << 33, 0, + UINT64_MAX, 0, UINT64_C(1) << 1, + }; + uint8_t bit_test_carry[12] = {0}; + bitTestMemorySemantics(bit_test_carry, bit_test_data); + const uint8_t bit_test_want_carry[12] = {1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0}; + const uint64_t bit_test_want_data[9] = { + UINT64_C(0x7fffffffffffffff), 1, 0, + UINT64_C(0x7fffffffffffffff), 1, 0, + UINT64_C(0x7fffffffffffffff), 1, 0, + }; + for (int i = 0; i < 12; i++) + if (bit_test_carry[i] != bit_test_want_carry[i]) return 126; + for (int i = 0; i < 9; i++) + if (bit_test_data[i] != bit_test_want_data[i]) return 127; + int16_t word_madd_a[16] = { + INT16_MIN, INT16_MIN, -30000, -1, 0, 1, 12345, INT16_MAX, + 7, -11, 1000, -2000, 22222, -12345, INT16_MAX, INT16_MAX, + }; + int16_t word_madd_b[16] = { + INT16_MIN, INT16_MIN, -23456, INT16_MAX, -1, INT16_MAX, -2345, INT16_MAX, + -9, 13, -3000, 4000, -11111, 23456, INT16_MAX, INT16_MAX, + }; + int32_t word_madd_out[18] = {0}, word_madd_reference[8] = {0}, word_madd_want[18] = {0}; + for (int i = 0; i < 8; i++) { + int64_t sum = (int64_t)word_madd_a[2*i] * word_madd_b[2*i] + + (int64_t)word_madd_a[2*i+1] * word_madd_b[2*i+1]; + word_madd_reference[i] = (int32_t)(uint32_t)sum; + } + for (int i = 0; i < 2; i++) word_madd_want[i] = word_madd_reference[i]; + for (int i = 0; i < 4; i++) { + word_madd_want[2+i] = word_madd_reference[i]; + word_madd_want[6+i] = word_madd_reference[i]; + } + for (int i = 0; i < 8; i++) word_madd_want[10+i] = word_madd_reference[i]; + packedWordMultiplyAddSemantics(word_madd_out, word_madd_a, word_madd_b); + for (int i = 0; i < 18; i++) + if (word_madd_out[i] != word_madd_want[i]) return 128; + int8_t byte_madd_signed[32] = { + 127, 127, -128, -128, 100, -100, 1, -1, + 64, -64, 11, 12, -13, 14, 15, -16, + 31, -32, 33, -34, 35, -36, 37, -38, + 39, -40, 41, -42, 43, -44, 45, -46, + }; + uint8_t byte_madd_unsigned[32] = { + 255, 255, 255, 255, 200, 3, 255, 128, + 250, 249, 17, 19, 23, 29, 31, 37, + 41, 43, 47, 53, 59, 61, 67, 71, + 73, 79, 83, 89, 97, 101, 103, 107, + }; + int16_t byte_madd_out[32] = {0}, byte_madd_reference[16] = {0}, byte_madd_want[32] = {0}; + for (int i = 0; i < 16; i++) { + int32_t sum = (int32_t)byte_madd_signed[2*i] * byte_madd_unsigned[2*i] + + (int32_t)byte_madd_signed[2*i+1] * byte_madd_unsigned[2*i+1]; + if (sum > INT16_MAX) sum = INT16_MAX; + if (sum < INT16_MIN) sum = INT16_MIN; + byte_madd_reference[i] = (int16_t)sum; + } + for (int i = 0; i < 8; i++) { + byte_madd_want[i] = byte_madd_reference[i]; + byte_madd_want[8+i] = byte_madd_reference[i]; + } + for (int i = 0; i < 16; i++) byte_madd_want[16+i] = byte_madd_reference[i]; + packedUnsignedSignedByteMultiplyAddSemantics(byte_madd_out, byte_madd_signed, byte_madd_unsigned); + for (int i = 0; i < 32; i++) + if (byte_madd_out[i] != byte_madd_want[i]) return 129; + int8_t dot_signed_bytes[64]; + uint8_t dot_unsigned_bytes[64]; + int16_t dot_words_a[32], dot_words_b[32]; + int32_t dot_accumulator[16], dot_out[64] = {0}; + for (int i = 0; i < 64; i++) { + dot_signed_bytes[i] = (i / 4) & 1 ? -128 : 127; + dot_unsigned_bytes[i] = (uint8_t)(255 - (i & 3)); + } + for (int i = 0; i < 32; i++) { + dot_words_a[i] = (i / 2) & 1 ? INT16_MIN : INT16_MAX; + dot_words_b[i] = (i & 1) ? INT16_MAX : INT16_MIN; + } + for (int i = 0; i < 16; i++) { + switch (i & 3) { + case 0: dot_accumulator[i] = INT32_MAX - 10; break; + case 1: dot_accumulator[i] = INT32_MIN + 10; break; + case 2: dot_accumulator[i] = 123456789; break; + default: dot_accumulator[i] = -987654321; break; + } + } + packedDotProductSemantics(dot_out, dot_signed_bytes, dot_unsigned_bytes, dot_words_a, dot_words_b, dot_accumulator); + for (int lane = 0; lane < 16; lane++) { + int64_t byte_sum = dot_accumulator[lane]; + for (int item = 0; item < 4; item++) byte_sum += (int64_t)dot_signed_bytes[4*lane+item] * dot_unsigned_bytes[4*lane+item]; + int64_t byte_sat = byte_sum; + if (byte_sat > INT32_MAX) byte_sat = INT32_MAX; + if (byte_sat < INT32_MIN) byte_sat = INT32_MIN; + int64_t word_sum = dot_accumulator[lane]; + for (int item = 0; item < 2; item++) word_sum += (int64_t)dot_words_a[2*lane+item] * dot_words_b[2*lane+item]; + int64_t word_sat = word_sum; + if (word_sat > INT32_MAX) word_sat = INT32_MAX; + if (word_sat < INT32_MIN) word_sat = INT32_MIN; + if ((uint32_t)dot_out[lane] != (uint32_t)byte_sum) return 930 + lane; + if (dot_out[16+lane] != (int32_t)byte_sat) return 950 + lane; + if ((uint32_t)dot_out[32+lane] != (uint32_t)word_sum) return 970 + lane; + if (dot_out[48+lane] != (int32_t)word_sat) return 990 + lane; + } + uint8_t minmax_a[64], minmax_b[64], minmax_out[1024] = {0}; + for (int i = 0; i < 64; i++) { + minmax_a[i] = (uint8_t)(0x80 + 73*i); + minmax_b[i] = (uint8_t)(0xff - 29*i); + } + packedIntegerMinMaxSemantics(minmax_out, minmax_a, minmax_b); + for (int lane = 0; lane < 64; lane++) { + int8_t a = (int8_t)minmax_a[lane], b = (int8_t)minmax_b[lane]; + if ((int8_t)minmax_out[lane] != (a < b ? a : b)) return 1010 + lane; + if (minmax_out[64+lane] != (minmax_a[lane] < minmax_b[lane] ? minmax_a[lane] : minmax_b[lane])) return 1080 + lane; + if ((int8_t)minmax_out[128+lane] != (a > b ? a : b)) return 1150 + lane; + if (minmax_out[192+lane] != (minmax_a[lane] > minmax_b[lane] ? minmax_a[lane] : minmax_b[lane])) return 1220 + lane; + } + for (int lane = 0; lane < 32; lane++) { + uint16_t au = load16(minmax_a + 2*lane), bu = load16(minmax_b + 2*lane); + int16_t as = (int16_t)au, bs = (int16_t)bu; + if ((int16_t)load16(minmax_out + 256 + 2*lane) != (as < bs ? as : bs)) return 1290 + lane; + if (load16(minmax_out + 320 + 2*lane) != (au < bu ? au : bu)) return 1330 + lane; + if ((int16_t)load16(minmax_out + 384 + 2*lane) != (as > bs ? as : bs)) return 1370 + lane; + if (load16(minmax_out + 448 + 2*lane) != (au > bu ? au : bu)) return 1410 + lane; + } + for (int lane = 0; lane < 16; lane++) { + uint32_t au = load32(minmax_a + 4*lane), bu = load32(minmax_b + 4*lane); + int32_t as = (int32_t)au, bs = (int32_t)bu; + if ((int32_t)load32(minmax_out + 512 + 4*lane) != (as < bs ? as : bs)) return 1450 + lane; + if (load32(minmax_out + 576 + 4*lane) != (au < bu ? au : bu)) return 1470 + lane; + if ((int32_t)load32(minmax_out + 640 + 4*lane) != (as > bs ? as : bs)) return 1490 + lane; + if (load32(minmax_out + 704 + 4*lane) != (au > bu ? au : bu)) return 1510 + lane; + } + for (int lane = 0; lane < 8; lane++) { + uint64_t au = load64(minmax_a + 8*lane), bu = load64(minmax_b + 8*lane); + int64_t as = (int64_t)au, bs = (int64_t)bu; + if ((int64_t)load64(minmax_out + 768 + 8*lane) != (as < bs ? as : bs)) return 1530 + lane; + if (load64(minmax_out + 832 + 8*lane) != (au < bu ? au : bu)) return 1540 + lane; + if ((int64_t)load64(minmax_out + 896 + 8*lane) != (as > bs ? as : bs)) return 1550 + lane; + if (load64(minmax_out + 960 + 8*lane) != (au > bu ? au : bu)) return 1560 + lane; + } + int16_t word_a[8] = {INT16_MIN, INT16_MIN, -30000, -1, 0, 1, 12345, INT16_MAX}; + int16_t word_b[8] = {INT16_MIN, INT16_MAX, -23456, INT16_MAX, -1, INT16_MAX, -2345, INT16_MAX}; + uint16_t word_mul_out[64] = {0}, word_mul_want[64] = {0}; + packedWordMultiplySemantics(word_mul_out, word_a, word_b); + for (int i = 0; i < 8; i++) { + int32_t product = (int32_t)word_a[i] * (int32_t)word_b[i]; + uint32_t unsigned_product = (uint32_t)(uint16_t)word_a[i] * (uint32_t)(uint16_t)word_b[i]; + int32_t rounded = (product + 0x4000) >> 15; + uint16_t values[4] = {(uint16_t)product, (uint16_t)(product >> 16), (uint16_t)(unsigned_product >> 16), (uint16_t)rounded}; + for (int mode = 0; mode < 4; mode++) { + word_mul_want[mode*8+i] = values[mode]; + word_mul_want[(mode+4)*8+i] = values[mode]; + } + } + for (int i = 0; i < 64; i++) + if (word_mul_out[i] != word_mul_want[i]) return 240 + i; + uint32_t shuffle_a[8], shuffle_b[8]; + uint8_t shuffle_out[96] = {0}; + for (int i = 0; i < 8; i++) { + shuffle_a[i] = 0xa0 + i; + shuffle_b[i] = 0xb0 + i; + } + packedFloatShuffleSemantics(shuffle_out, (uint8_t *)shuffle_a, (uint8_t *)shuffle_b); + const uint32_t shuffle_ps[12] = { + 0xb3, 0xb2, 0xa1, 0xa0, + 0xb3, 0xb2, 0xa1, 0xa0, + 0xb7, 0xb6, 0xa5, 0xa4, + }; + for (int i = 0; i < 4; i++) + if (load32(shuffle_out + 4*i) != shuffle_ps[i]) return 310 + i; + if (load64(shuffle_out + 16) != UINT64_C(0x000000b1000000b0) || + load64(shuffle_out + 24) != UINT64_C(0x000000a3000000a2)) return 314; + for (int i = 0; i < 8; i++) + if (load32(shuffle_out + 32 + 4*i) != shuffle_ps[4+i]) return 315 + i; + const uint64_t shuffle_pd[4] = { + UINT64_C(0x000000b1000000b0), UINT64_C(0x000000a3000000a2), + UINT64_C(0x000000b5000000b4), UINT64_C(0x000000a7000000a6), + }; + for (int i = 0; i < 4; i++) + if (load64(shuffle_out + 64 + 8*i) != shuffle_pd[i]) return 323 + i; + uint32_t in_lane_data[8] = {0x100, 0x101, 0x102, 0x103, 0x104, 0x105, 0x106, 0x107}; + uint32_t in_lane_control[8] = {3, 2, 1, 0, 4, 5, 6, 7}; + uint8_t in_lane_out[144] = {0}; + inLaneFloatingPermuteSemantics(in_lane_out, (uint8_t *)in_lane_data, (uint8_t *)in_lane_control); + for (int lane = 0; lane < 4; lane++) + if (load32(in_lane_out + 4*lane) != in_lane_data[3-lane]) return 324; + for (int lane = 0; lane < 8; lane++) { + uint32_t immediate_value = in_lane_data[(lane/4)*4 + 3-(lane%4)]; + uint32_t variable_value = in_lane_data[(lane/4)*4 + (in_lane_control[lane]&3)]; + if (load32(in_lane_out + 16 + 4*lane) != immediate_value) return 325; + if (load32(in_lane_out + 48 + 4*lane) != variable_value) return 326; + } + for (int lane = 0; lane < 4; lane++) { + uint64_t data_q[4], control_q[4]; + for (int i = 0; i < 4; i++) { + data_q[i] = (uint64_t)in_lane_data[2*i] | (uint64_t)in_lane_data[2*i+1] << 32; + control_q[i] = (uint64_t)in_lane_control[2*i] | (uint64_t)in_lane_control[2*i+1] << 32; + } + uint64_t immediate_value = data_q[(lane/2)*2 + ((0x05 >> lane)&1)]; + uint64_t variable_value = data_q[(lane/2)*2 + ((control_q[lane] >> 1)&1)]; + if (load64(in_lane_out + 80 + 8*lane) != immediate_value) return 327; + if (load64(in_lane_out + 112 + 8*lane) != variable_value) return 328; + } + uint8_t blend_a[32], blend_b[32], blend_out[240] = {0}; + for (int i = 0; i < 32; i++) { + blend_a[i] = (uint8_t)(0x20 + i); + blend_b[i] = (uint8_t)(0xc0 + i); + } + immediatePackedBlendSemantics(blend_out, blend_a, blend_b); + const int blend_offsets[11] = {0, 16, 32, 48, 64, 96, 112, 144, 160, 192, 208}; + const int blend_widths[11] = {16, 16, 16, 16, 32, 16, 32, 16, 32, 16, 32}; + const int blend_lane_bytes[11] = {2, 4, 8, 2, 2, 4, 4, 4, 4, 8, 8}; + for (int block = 0; block < 11; block++) { + for (int i = 0; i < blend_widths[block]; i++) { + int bit = (i / blend_lane_bytes[block]) % 8; + uint8_t want = (0xa5 & (1 << bit)) ? blend_a[i] : blend_b[i]; + if (blend_out[blend_offsets[block] + i] != want) return 327 + block; + } + } + uint8_t unpack_out[384] = {0}; + packedUnpackSemantics(unpack_out, blend_a, blend_b); + const int unpack_offsets[16] = {0,16,32,48,64,80,96,112,128,160,192,224,256,288,320,352}; + const int unpack_widths[16] = {16,16,16,16,16,16,16,16,32,32,32,32,32,32,32,32}; + const int unpack_lane_bytes[16] = {1,1,2,2,4,4,8,8,1,1,2,2,4,4,8,8}; + for (int block = 0; block < 16; block++) { + int output = unpack_offsets[block]; + int high = block & 1; + for (int group = 0; group < unpack_widths[block]; group += 16) { + int lanes_per_group = 16 / unpack_lane_bytes[block]; + int start = high ? lanes_per_group / 2 : 0; + for (int lane = start; lane < start + lanes_per_group / 2; lane++) { + int input = group + lane * unpack_lane_bytes[block]; + for (int byte = 0; byte < unpack_lane_bytes[block]; byte++) { + if (unpack_out[output++] != blend_b[input + byte]) return 338 + block; + } + for (int byte = 0; byte < unpack_lane_bytes[block]; byte++) { + if (unpack_out[output++] != blend_a[input + byte]) return 354 + block; + } + } + } + } + uint64_t permute_data[4] = {0x10, 0x21, 0x32, 0x43}; + uint64_t permute_control[4] = {3, 0, 5, 2}; + uint64_t permute_out[8] = {0}, variable_permute_out[8] = {0}; + const uint64_t permute_immediate[4] = {0x10, 0x32, 0x21, 0x43}; + const uint64_t permute_variable[4] = {0x43, 0x10, 0x21, 0x32}; + qwordPermuteSemantics(permute_out, permute_data); + qwordVariablePermuteSemantics(variable_permute_out, permute_data, permute_control); + for (int block = 0; block < 2; block++) { + for (int lane = 0; lane < 4; lane++) + if (permute_out[4*block+lane] != permute_immediate[lane]) return 370 + block; + } + for (int block = 0; block < 2; block++) { + for (int lane = 0; lane < 4; lane++) + if (variable_permute_out[4*block+lane] != permute_variable[lane]) return 372 + block; + } + _Alignas(64) uint8_t move_src[64], move_old[64]; + uint8_t move_vex_out[128] = {0}; + for (int i = 0; i < 64; i++) { + move_src[i] = (uint8_t)(3 + 17*i); + move_old[i] = (uint8_t)(0xf0 - 3*i); + } + packedFloatMoveVEXSemantics(move_vex_out, move_src); + for (int block = 0; block < 4; block++) + for (int i = 0; i < 32; i++) + if (move_vex_out[32*block+i] != move_src[i]) return 374 + block; +#if PLAN9ASM_CONFORMANCE_NATIVE_WIDE + // X stores need 16-byte alignment. Z stores at offsets 432 and 624 + // need a base congruent to 16 modulo 64, not a 64-aligned base itself. + _Alignas(64) uint8_t move_mask_storage[768]; + for (int i = 0; i < 768; i++) move_mask_storage[i] = 0xa7; + uint8_t *move_mask_out = move_mask_storage + 16; + packedFloatMoveMaskSemantics(move_mask_out, move_src, move_old); + for (int i = 0; i < 16; i++) + if (move_mask_storage[i] != 0xa7) return 950; + for (int i = 704; i < 768; i++) + if (move_mask_storage[i] != 0xa7) return 951; + const int move_offsets[14] = {0,16,32,48,64,80,96,112,144,176,208,240,272,304}; + const int move_widths[14] = {16,16,16,16,16,16,16,32,32,32,32,32,32,64}; + const int move_lane_bytes[14] = {4,4,4,4,8,8,8,4,4,4,8,8,8,4}; + const int move_zero[14] = {0,1,0,0,0,1,0,0,1,0,0,1,0,0}; + for (int block = 0; block < 14; block++) { + for (int i = 0; i < move_widths[block]; i++) { + int active = (0xa55a >> (i / move_lane_bytes[block])) & 1; + uint8_t want = active ? move_src[i] : (move_zero[block] ? 0 : move_old[i]); + if (move_mask_out[move_offsets[block]+i] != want) return 378 + block; + } + } + const int move_z_offsets[5] = {368,432,496,560,624}; + const int move_z_lane_bytes[5] = {4,4,8,8,8}; + const int move_z_zero[5] = {1,0,0,1,0}; + for (int block = 0; block < 5; block++) { + for (int i = 0; i < 64; i++) { + int active = (0xa55a >> (i / move_z_lane_bytes[block])) & 1; + uint8_t want = active ? move_src[i] : (move_z_zero[block] ? 0 : move_old[i]); + if (move_mask_out[move_z_offsets[block]+i] != want) return 392 + block; + } + } +#endif + uint8_t extend_src[32] = { + 0x00,0x01,0x7f,0x80,0xff,0x55,0xaa,0x40, + 0xc0,0x11,0xee,0x33,0xcc,0x66,0x99,0x22, + 0xdd,0x44,0xbb,0x77,0x88,0x10,0xf0,0x20, + 0xe0,0x30,0xd0,0x50,0xb0,0x60,0xa0,0x70, + }; + uint8_t extend_vex_out[288] = {0}; + const int extend_input_bits[6] = {8,8,8,16,16,32}; + const int extend_output_bits[6] = {16,32,64,32,64,64}; + packedSignExtendMoveVEXSemantics(extend_vex_out, extend_src); + for (int op = 0; op < 6; op++) { + for (int form = 0; form < 2; form++) { + int width = form ? 32 : 16; + int offset = form ? 96 + 32*op : 16*op; + int lanes = width * 8 / extend_output_bits[op]; + for (int lane = 0; lane < lanes; lane++) { + const uint8_t *p = extend_src + lane * extend_input_bits[op] / 8; + int64_t expected = extend_input_bits[op] == 8 ? (int64_t)(int8_t)p[0] : + extend_input_bits[op] == 16 ? (int64_t)(int16_t)load16(p) : (int64_t)(int32_t)load32(p); + const uint8_t *actual = extend_vex_out + offset + lane * extend_output_bits[op] / 8; + uint64_t actual_bits = extend_output_bits[op] == 16 ? load16(actual) : + extend_output_bits[op] == 32 ? load32(actual) : load64(actual); + uint64_t mask = extend_output_bits[op] == 16 ? UINT64_C(0xffff) : + extend_output_bits[op] == 32 ? UINT64_C(0xffffffff) : UINT64_MAX; + if (actual_bits != ((uint64_t)expected & mask)) return 397 + op; + } + } + } + uint8_t extend_old[64], extend_mask_out[768] = {0}; + for (int i = 0; i < 64; i++) extend_old[i] = (uint8_t)(0xe7 - 5*i); + packedSignExtendMoveMaskSemantics(extend_mask_out, extend_src, extend_old); + for (int op = 0; op < 6; op++) { + int lanes = 512 / extend_output_bits[op]; + for (int zero = 0; zero < 2; zero++) { + int offset = 128*op + 64*zero; + for (int lane = 0; lane < lanes; lane++) { + const uint8_t *input = extend_src + lane * extend_input_bits[op] / 8; + int64_t extended = extend_input_bits[op] == 8 ? (int64_t)(int8_t)input[0] : + extend_input_bits[op] == 16 ? (int64_t)(int16_t)load16(input) : (int64_t)(int32_t)load32(input); + const uint8_t *actual = extend_mask_out + offset + lane * extend_output_bits[op] / 8; + uint64_t actual_bits = extend_output_bits[op] == 16 ? load16(actual) : + extend_output_bits[op] == 32 ? load32(actual) : load64(actual); + const uint8_t *old = extend_old + lane * extend_output_bits[op] / 8; + uint64_t old_bits = extend_output_bits[op] == 16 ? load16(old) : + extend_output_bits[op] == 32 ? load32(old) : load64(old); + uint64_t width_mask = extend_output_bits[op] == 16 ? UINT64_C(0xffff) : + extend_output_bits[op] == 32 ? UINT64_C(0xffffffff) : UINT64_MAX; + uint64_t expected = (UINT64_C(0xa55aa55a) >> lane) & 1 ? (uint64_t)extended & width_mask : (zero ? 0 : old_bits); + if (actual_bits != expected) return 403 + op; + } + } + } uint8_t a[16], b[16], vector_out[160] = {0}; const uint8_t or_mask[16] = { 0x80, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, @@ -310,10 +1317,321 @@ int main(void) { packedArithmeticShift32(packed_out, packed_src); for (int i = 0; i < 8; i++) if (packed_out[i] != packed_want[i]) return 80 + i; + + uint8_t subtract_a[16] = {0x7f, 0x80, 0x64, 0x9c, 0x32, 0xce, 0x00, 0x01, 0xff, 0x00, 0x34, 0x12, 0x00, 0x80, 0xff, 0x7f}; + uint8_t subtract_b[16] = {0xff, 0x01, 0x9c, 0x64, 0xce, 0x32, 0x01, 0x00, 0x01, 0xff, 0x78, 0x56, 0xff, 0x7f, 0x01, 0x80}; + uint8_t subtract_out[80] = {0}; + packedSubtractSemantics(subtract_out, subtract_a, subtract_b); + for (int i = 0; i < 16; i++) { + int signed_value = (int)(int8_t)subtract_a[i] - (int)(int8_t)subtract_b[i]; + if (signed_value < -128) signed_value = -128; + if (signed_value > 127) signed_value = 127; + if (subtract_out[i] != (uint8_t)(int8_t)signed_value) return 190; + int unsigned_value = (int)subtract_a[i] - (int)subtract_b[i]; + if (unsigned_value < 0) unsigned_value = 0; + if (subtract_out[32+i] != (uint8_t)unsigned_value) return 191; + } + for (int i = 0; i < 8; i++) { + int signed_value = (int)(int16_t)load16(subtract_a + 2*i) - (int)(int16_t)load16(subtract_b + 2*i); + if (signed_value < -32768) signed_value = -32768; + if (signed_value > 32767) signed_value = 32767; + if (load16(subtract_out + 16 + 2*i) != (uint16_t)(int16_t)signed_value) return 192; + int unsigned_value = (int)load16(subtract_a + 2*i) - (int)load16(subtract_b + 2*i); + if (unsigned_value < 0) unsigned_value = 0; + if (load16(subtract_out + 48 + 2*i) != (uint16_t)unsigned_value) return 193; + } + for (int i = 0; i < 2; i++) + if (load64(subtract_out + 64 + 8*i) != load64(subtract_a + 8*i) - load64(subtract_b + 8*i)) return 194; + + uint8_t ecosystem_a[16] = { + 0x00, 0x7f, 0x80, 0xff, 0x34, 0x12, 0xfe, 0xff, + 0x78, 0x56, 0x34, 0x12, 0x00, 0x00, 0x00, 0x80, + }; + uint8_t ecosystem_b[16] = { + 0xff, 0x80, 0x7f, 0x00, 0x78, 0x56, 0x02, 0x00, + 0xef, 0xcd, 0xab, 0x90, 0xff, 0xff, 0xff, 0x7f, + }; + uint8_t ecosystem_out[816] = {0}; + expandedEcosystemVectors(ecosystem_out, ecosystem_a, ecosystem_b); + for (int i = 0; i < 16; i++) { + uint8_t min_u = ecosystem_a[i] < ecosystem_b[i] ? ecosystem_a[i] : ecosystem_b[i]; + uint8_t max_u = ecosystem_a[i] > ecosystem_b[i] ? ecosystem_a[i] : ecosystem_b[i]; + int8_t as = (int8_t)ecosystem_a[i], bs = (int8_t)ecosystem_b[i]; + if (ecosystem_out[i] != min_u || ecosystem_out[16+i] != (uint8_t)(as < bs ? as : bs)) return 100; + if (ecosystem_out[96+i] != max_u || ecosystem_out[112+i] != (uint8_t)(as > bs ? as : bs)) return 101; + } + for (int i = 0; i < 8; i++) { + uint16_t au = load16(ecosystem_a + 2*i), bu = load16(ecosystem_b + 2*i); + int16_t as = (int16_t)au, bs = (int16_t)bu; + if (load16(ecosystem_out + 32 + 2*i) != (au < bu ? au : bu)) return 102; + if ((int16_t)load16(ecosystem_out + 48 + 2*i) != (as < bs ? as : bs)) return 103; + if (load16(ecosystem_out + 128 + 2*i) != (au > bu ? au : bu)) return 104; + if ((int16_t)load16(ecosystem_out + 144 + 2*i) != (as > bs ? as : bs)) return 105; + } + for (int i = 0; i < 4; i++) { + uint32_t au = load32(ecosystem_a + 4*i), bu = load32(ecosystem_b + 4*i); + int32_t as = (int32_t)au, bs = (int32_t)bu; + if (load32(ecosystem_out + 64 + 4*i) != (au < bu ? au : bu)) return 106; + if ((int32_t)load32(ecosystem_out + 80 + 4*i) != (as < bs ? as : bs)) return 107; + if (load32(ecosystem_out + 160 + 4*i) != (au > bu ? au : bu)) return 108; + if ((int32_t)load32(ecosystem_out + 176 + 4*i) != (as > bs ? as : bs)) return 109; + } + for (int i = 0; i < 2; i++) + if (load64(ecosystem_out + 192 + 8*i) != (load64(ecosystem_a + 8*i) << 13)) return 110; + for (int i = 208; i < 224; i++) + if (ecosystem_out[i] != 0) return 111; + if (load64(ecosystem_out + 224) != 0x0123456789abcdefULL) return 112; + if (load64(ecosystem_out + 232) != 0x0000000089abcdefULL) return 113; + if (load32(ecosystem_out + 240) != 0x89abcdefU) return 119; + for (int i = 244; i < 256; i++) if (ecosystem_out[i] != 0) return 121; + if (load64(ecosystem_out + 256) != load64(ecosystem_a)) return 122; + if (load64(ecosystem_out + 264) != 0) return 123; + for (int i = 0; i < 16; i++) + if (ecosystem_out[272+i] != (uint8_t)(((uint16_t)ecosystem_a[i] + ecosystem_b[i] + 1) >> 1)) return 124; + for (int i = 0; i < 8; i++) + if (load16(ecosystem_out + 288 + 2*i) != (uint16_t)(((uint32_t)load16(ecosystem_a + 2*i) + load16(ecosystem_b + 2*i) + 1) >> 1)) return 125; + for (int i = 0; i < 16; i++) + if (ecosystem_out[304+i] != ((int8_t)ecosystem_a[i] > (int8_t)ecosystem_b[i] ? 0xff : 0)) return 126; + for (int i = 0; i < 8; i++) + if (load16(ecosystem_out + 320 + 2*i) != ((int16_t)load16(ecosystem_a + 2*i) > (int16_t)load16(ecosystem_b + 2*i) ? 0xffff : 0)) return 127; + for (int i = 0; i < 4; i++) + if (load32(ecosystem_out + 336 + 4*i) != ((int32_t)load32(ecosystem_a + 4*i) > (int32_t)load32(ecosystem_b + 4*i) ? 0xffffffffU : 0)) return 128; + for (int i = 0; i < 2; i++) + if (load64(ecosystem_out + 352 + 8*i) != ((int64_t)load64(ecosystem_a + 8*i) > (int64_t)load64(ecosystem_b + 8*i) ? ~0ULL : 0)) return 129; + for (int i = 0; i < 8; i++) { + int16_t value = (int16_t)load16(ecosystem_a + 2*i); + if ((int16_t)load16(ecosystem_out + 368 + 2*i) != (int16_t)(value >> 3)) return 150; + if ((int16_t)load16(ecosystem_out + 384 + 2*i) != (int16_t)(value >> 15)) return 151; + } + for (int i = 0; i < 2; i++) { + if (load32(ecosystem_out + 400 + 8*i) != load32(ecosystem_a + 4*i)) return 152; + if (load32(ecosystem_out + 404 + 8*i) != load32(ecosystem_b + 4*i)) return 153; + if (load32(ecosystem_out + 416 + 8*i) != load32(ecosystem_a + 8 + 4*i)) return 154; + if (load32(ecosystem_out + 420 + 8*i) != load32(ecosystem_b + 8 + 4*i)) return 155; + } + if (load64(ecosystem_out + 432) != load64(ecosystem_a) || load64(ecosystem_out + 440) != load64(ecosystem_b)) return 156; + if (load64(ecosystem_out + 448) != load64(ecosystem_a + 8) || load64(ecosystem_out + 456) != load64(ecosystem_b + 8)) return 157; + for (int i = 0; i < 8; i++) if (load16(ecosystem_out + 464 + 2*i) != ecosystem_a[i]) return 158; + for (int i = 0; i < 4; i++) if (load32(ecosystem_out + 480 + 4*i) != ecosystem_a[i]) return 159; + for (int i = 0; i < 2; i++) if (load64(ecosystem_out + 496 + 8*i) != ecosystem_a[i]) return 160; + for (int i = 0; i < 4; i++) if (load32(ecosystem_out + 512 + 4*i) != load16(ecosystem_a + 2*i)) return 161; + for (int i = 0; i < 2; i++) if (load64(ecosystem_out + 528 + 8*i) != load16(ecosystem_a + 2*i)) return 162; + for (int i = 0; i < 2; i++) if (load64(ecosystem_out + 544 + 8*i) != load32(ecosystem_a + 4*i)) return 163; + for (int i = 0; i < 2; i++) { + if (load32(ecosystem_out + 560 + 4*i) != load32(ecosystem_a + 8*i) + load32(ecosystem_a + 8*i + 4)) return 164; + if (load32(ecosystem_out + 568 + 4*i) != load32(ecosystem_b + 8*i) + load32(ecosystem_b + 8*i + 4)) return 165; + if (load32(ecosystem_out + 608 + 4*i) != load32(ecosystem_a + 8*i) - load32(ecosystem_a + 8*i + 4)) return 166; + if (load32(ecosystem_out + 616 + 4*i) != load32(ecosystem_b + 8*i) - load32(ecosystem_b + 8*i + 4)) return 167; + } + for (int i = 0; i < 4; i++) { + int32_t aa = (int16_t)load16(ecosystem_a + 4*i) + (int16_t)load16(ecosystem_a + 4*i + 2); + int32_t ba = (int16_t)load16(ecosystem_b + 4*i) + (int16_t)load16(ecosystem_b + 4*i + 2); + int32_t as = (int16_t)load16(ecosystem_a + 4*i) - (int16_t)load16(ecosystem_a + 4*i + 2); + int32_t bs = (int16_t)load16(ecosystem_b + 4*i) - (int16_t)load16(ecosystem_b + 4*i + 2); + if (aa < -32768) aa = -32768; if (aa > 32767) aa = 32767; + if (ba < -32768) ba = -32768; if (ba > 32767) ba = 32767; + if (as < -32768) as = -32768; if (as > 32767) as = 32767; + if (bs < -32768) bs = -32768; if (bs > 32767) bs = 32767; + if ((int16_t)load16(ecosystem_out + 576 + 2*i) != (int16_t)aa) return 168; + if ((int16_t)load16(ecosystem_out + 584 + 2*i) != (int16_t)ba) return 169; + if (load16(ecosystem_out + 592 + 2*i) != (uint16_t)(load16(ecosystem_a + 4*i) + load16(ecosystem_a + 4*i + 2))) return 170; + if (load16(ecosystem_out + 600 + 2*i) != (uint16_t)(load16(ecosystem_b + 4*i) + load16(ecosystem_b + 4*i + 2))) return 171; + if ((int16_t)load16(ecosystem_out + 624 + 2*i) != (int16_t)as) return 172; + if ((int16_t)load16(ecosystem_out + 632 + 2*i) != (int16_t)bs) return 173; + if (load16(ecosystem_out + 640 + 2*i) != (uint16_t)(load16(ecosystem_a + 4*i) - load16(ecosystem_a + 4*i + 2))) return 174; + if (load16(ecosystem_out + 648 + 2*i) != (uint16_t)(load16(ecosystem_b + 4*i) - load16(ecosystem_b + 4*i + 2))) return 175; + } + uint16_t minimum = 0xffff, minimum_index = 0; + for (int i = 0; i < 8; i++) { + uint16_t value = load16(ecosystem_a + 2*i); + if (value < minimum) { minimum = value; minimum_index = (uint16_t)i; } + } + if (load16(ecosystem_out + 656) != minimum || load16(ecosystem_out + 658) != minimum_index) return 176; + for (int i = 660; i < 672; i++) if (ecosystem_out[i] != 0) return 177; + for (int block = 0; block < 2; block++) + for (int i = 0; i < 16; i++) { + uint8_t want_masked = (int8_t)ecosystem_b[i] < 0 ? ecosystem_a[i] : ecosystem_b[i]; + if (ecosystem_out[672 + 16*block + i] != want_masked) return 178 + block; + } + for (int i = 0; i < 8; i++) { + if (load16(ecosystem_out + 704 + 2*i) != (uint16_t)(load16(ecosystem_a + 2*i) << 3)) return 180; + if (load16(ecosystem_out + 752 + 2*i) != (uint16_t)(load16(ecosystem_a + 2*i) >> 3)) return 181; + } + for (int i = 0; i < 4; i++) { + if (load32(ecosystem_out + 720 + 4*i) != load32(ecosystem_a + 4*i) << 3) return 182; + if (load32(ecosystem_out + 768 + 4*i) != load32(ecosystem_a + 4*i) >> 3) return 183; + } + for (int i = 0; i < 2; i++) { + if (load64(ecosystem_out + 736 + 8*i) != load64(ecosystem_a + 8*i) << 3) return 184; + if (load64(ecosystem_out + 784 + 8*i) != load64(ecosystem_a + 8*i) >> 3) return 185; + } + if (load64(ecosystem_out + 800) != 3 || load64(ecosystem_out + 808) != 0) return 186; + uint64_t parity0 = parityBranches(0x00); + if (parity0 != 1) return 120 + (int)parity0; + if (parityBranches(0x03) != 1) return 114; + if (parityBranches(0x01) != 2) return 115; + if (parityBranches(0x07) != 2) return 116; + volatile double zero = 0.0; + if (unorderedBranch(1.25) != 0) return 117; + if (unorderedBranch(zero / zero) != 1) return 118; + float conversion_src[4] = {1.5f, -1.5f, 2.75f, -2.75f}; + int32_t conversion_out[20] = {0}; + int32_t conversion_want[20] = { + 2, -2, 3, -3, + 1, -2, 2, -3, + 2, -1, 3, -2, + 1, -1, 2, -2, + 1, -1, 2, -2, + }; + packedFloatToDwordModes(conversion_out, conversion_src); + for (int i = 0; i < 20; i++) + if (conversion_out[i] != conversion_want[i]) return 130 + i; +#if PLAN9ASM_CONFORMANCE_NATIVE_WIDE + int32_t same_width_ints[16]; + float same_width_floats[16] = { + 1.5f, -1.5f, 2.75f, -2.75f, 0.5f, -0.5f, 3.25f, -3.25f, + 4.5f, -4.5f, 5.75f, -5.75f, 6.5f, -6.5f, 7.25f, -7.25f, + }; + float same_width_squares32[16]; + double same_width_squares64[8]; + for (int i = 0; i < 16; i++) { + same_width_ints[i] = i - 8; + same_width_squares32[i] = (float)((i + 1) * (i + 1)); + } + for (int i = 0; i < 8; i++) same_width_squares64[i] = (double)((i + 1) * (i + 1)); + int32_t same_width_nearest[16] = {2,-2,3,-3,0,0,3,-3,4,-4,6,-6,6,-6,7,-7}; + int32_t same_width_trunc[16] = {1,-1,2,-2,0,0,3,-3,4,-4,5,-5,6,-6,7,-7}; + int32_t same_width_down[16] = {1,-2,2,-3,0,-1,3,-4,4,-5,5,-6,6,-7,7,-8}; + int32_t same_width_up[16] = {2,-1,3,-2,1,0,4,-3,5,-4,6,-5,7,-6,8,-7}; + uint8_t same_width_out[576] = {0}; + sameWidthPackedConversionSemantics(same_width_out, same_width_ints, same_width_floats, same_width_squares32, same_width_squares64); + for (int i = 0; i < 16; i++) { + if (loadf32(same_width_out + 4*i) != (float)same_width_ints[i]) return 760 + i; + if ((int32_t)load32(same_width_out + 64 + 4*i) != same_width_nearest[i]) return 780 + i; + if ((int32_t)load32(same_width_out + 128 + 4*i) != same_width_trunc[i]) return 800 + i; + if (loadf32(same_width_out + 192 + 4*i) != (float)(i + 1)) return 820 + i; + if ((int32_t)load32(same_width_out + 320 + 4*i) != same_width_nearest[i]) return 840 + i; + if ((int32_t)load32(same_width_out + 384 + 4*i) != same_width_down[i]) return 860 + i; + if ((int32_t)load32(same_width_out + 448 + 4*i) != same_width_up[i]) return 880 + i; + if ((int32_t)load32(same_width_out + 512 + 4*i) != same_width_trunc[i]) return 900 + i; + } + for (int i = 0; i < 8; i++) + if (loadf64(same_width_out + 256 + 8*i) != (double)(i + 1)) return 920 + i; +#endif + double widening_out[4] = {0}; + double widening_want[4] = {1.5, -1.5, 2.75, -2.75}; + packedSingleToDouble(widening_out, conversion_src); + for (int i = 0; i < 4; i++) + if (widening_out[i] != widening_want[i]) return 180 + i; + uint64_t cmov_src[16]; + uint64_t cmov_out[16] = {0}; + for (int i = 0; i < 16; i++) cmov_src[i] = 101 + (uint64_t)i; + conditionalMoveCodes(cmov_out, cmov_src); + int cmov_taken[16] = {1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1}; + for (int i = 0; i < 16; i++) { + uint64_t want = cmov_taken[i] ? cmov_src[i] : UINT64_MAX; + if (cmov_out[i] != want) return 160 + i; + } + double fma_a64[2] = {2, -3}, fma_b64[2] = {5, 7}, fma_c64[2] = {11, -13}; + float fma_a32[4] = {2, -3, 4, -5}; + float fma_b32[4] = {5, 7, -11, -13}; + float fma_c32[4] = {17, -19, 23, -29}; + uint8_t fma_out[288] = {0}; + fma3Semantics(fma_out, fma_a64, fma_b64, fma_c64, fma_a32, fma_b32, fma_c32); + for (int lane = 0; lane < 2; lane++) { + double a = fma_a64[lane], b = fma_b64[lane], c = fma_c64[lane]; + double want[8] = { + c*a+b, b*c+a, b*a+c, c*a-b, -(b*c)+a, -(b*a)-c, + lane % 2 == 0 ? c*a-b : c*a+b, + lane % 2 == 0 ? b*a+c : b*a-c, + }; + for (int block = 0; block < 8; block++) + if (loadf64(fma_out + 16*block + 8*lane) != want[block]) return 500 + 2*block + lane; + } + for (int lane = 0; lane < 4; lane++) { + float a = fma_a32[lane], b = fma_b32[lane], c = fma_c32[lane]; + float want[6] = { + c*a+b, b*c-a, -(b*a)+c, -(c*a)-b, + lane % 2 == 0 ? b*c-a : b*c+a, + lane % 2 == 0 ? b*a+c : b*a-c, + }; + for (int block = 0; block < 6; block++) + if (loadf32(fma_out + 128 + 16*block + 4*lane) != want[block]) return 520 + 4*block + lane; + } + if (loadf64(fma_out + 224) != fma_c64[0]*fma_a64[0]+fma_b64[0] || + loadf64(fma_out + 232) != fma_c64[1]) return 550; + if (loadf64(fma_out + 240) != fma_b64[0]*fma_c64[0]-fma_a64[0] || + loadf64(fma_out + 248) != fma_c64[1]) return 551; + if (loadf32(fma_out + 256) != -(fma_b32[0]*fma_a32[0])+fma_c32[0] || + loadf32(fma_out + 260) != fma_c32[1] || + loadf32(fma_out + 264) != fma_c32[2] || + loadf32(fma_out + 268) != fma_c32[3]) return 552; + if (loadf32(fma_out + 272) != -(fma_c32[0]*fma_a32[0])-fma_b32[0] || + loadf32(fma_out + 276) != fma_c32[1] || + loadf32(fma_out + 280) != fma_c32[2] || + loadf32(fma_out + 284) != fma_c32[3]) return 553; + double binary_a64[2] = {2, -3}, binary_b64[2] = {20, 30}; + float binary_a32[4] = {2, -3, 4, -5}, binary_b32[4] = {20, 30, -40, -50}; + double binary_want64[6][2] = { + {22, 27}, {18, 33}, {40, -90}, {10, -10}, {20, 30}, {2, -3}, + }; + float binary_want32[6][4] = { + {22, 27, -36, -55}, + {18, 33, -44, -45}, + {40, -90, -160, 250}, + {10, -10, -10, 10}, + {20, 30, 4, -5}, + {2, -3, -40, -50}, + }; + uint8_t binary_out[384] = {0}; + binaryFloatingSemantics(binary_out, binary_a64, binary_b64, binary_a32, binary_b32); + for (int operation = 0; operation < 6; operation++) { + for (int lane = 0; lane < 2; lane++) + if (loadf64(binary_out + 16*operation + 8*lane) != binary_want64[operation][lane]) return 600 + 2*operation + lane; + for (int lane = 0; lane < 4; lane++) + if (loadf32(binary_out + 96 + 16*operation + 4*lane) != binary_want32[operation][lane]) return 620 + 4*operation + lane; + for (int lane = 0; lane < 2; lane++) { + double want = lane == 0 ? binary_want64[operation][0] : binary_b64[lane]; + if (loadf64(binary_out + 192 + 16*operation + 8*lane) != want) return 650 + 2*operation + lane; + } + for (int lane = 0; lane < 4; lane++) { + float want = lane == 0 ? binary_want32[operation][0] : binary_b32[lane]; + if (loadf32(binary_out + 288 + 16*operation + 4*lane) != want) return 670 + 4*operation + lane; + } + } + float horizontal_a32[8] = {1, 2, 3, 4, 5, 6, 7, 8}; + float horizontal_b32[8] = {10, 20, 30, 40, 50, 60, 70, 80}; + double horizontal_a64[4] = {1, 2, 3, 4}; + double horizontal_b64[4] = {10, 20, 30, 40}; + float horizontal_want32[4][8] = { + {30, 70, 3, 7, 110, 150, 11, 15}, + {-10, -10, -1, -1, -10, -10, -1, -1}, + {30, 70, 3, 7, 0, 0, 0, 0}, + {-10, -10, -1, -1, 0, 0, 0, 0}, + }; + double horizontal_want64[4][4] = { + {30, 3, 70, 7}, + {-10, -1, -10, -1}, + {30, 3, 0, 0}, + {-10, -1, 0, 0}, + }; + uint8_t horizontal_out[192] = {0}; + horizontalFloatingSemantics(horizontal_out, horizontal_a32, horizontal_b32, horizontal_a64, horizontal_b64); + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 8; lane++) + if (loadf32(horizontal_out + 32*block + 4*lane) != horizontal_want32[block][lane]) return 700 + 8*block + lane; + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 4; lane++) + if (loadf64(horizontal_out + 64 + 32*block + 8*lane) != horizontal_want64[block][lane]) return 720 + 4*block + lane; + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 4; lane++) + if (loadf32(horizontal_out + 128 + 16*block + 4*lane) != horizontal_want32[2+block][lane]) return 740 + 4*block + lane; + for (int block = 0; block < 2; block++) + for (int lane = 0; lane < 2; lane++) + if (loadf64(horizontal_out + 160 + 16*block + 8*lane) != horizontal_want64[2+block][lane]) return 750 + 2*block + lane; return 0; } ` - compileAndRunRuntimeTestForTarget(t, llc, clang, "amd64_conformance", triple, ll, mainC, runPrefix) } func rosettaAvailable() bool { diff --git a/amd64_continuation.go b/amd64_continuation.go new file mode 100644 index 000000000..56ef3c50c --- /dev/null +++ b/amd64_continuation.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// X86TailGroup names one ordinary entry and its private continuation TEXTs. +// Names use source spelling, before Options.ResolveSym. See X86TailGroups. +type X86TailGroup struct { + Root string + Helpers []string +} + +type x86Continuation struct { + root string + label string +} + +func (f *File) isX86Continuation(name string, resolve func(string) string) bool { + for symbol := range f.x86Continuations { + if resolve(symbol) == name { + return true + } + } + return false +} + +func coalesceX86Continuations(file *File, opt Options) (*File, error) { + if file == nil { + return nil, fmt.Errorf("nil file") + } + if len(opt.X86TailGroups) == 0 || len(file.x86Continuations) != 0 { + return file, nil + } + if file.Arch != ArchAMD64 || (opt.Goarch != "amd64" && opt.Goarch != "386") { + return nil, fmt.Errorf("x86 continuation groups require an x86 target") + } + resolve := opt.ResolveSym + if resolve == nil { + resolve = func(s string) string { return s } + } + functions := make(map[string]Func, len(file.Funcs)) + for _, fn := range file.Funcs { + if _, duplicate := functions[fn.Sym]; duplicate { + return nil, fmt.Errorf("duplicate continuation TEXT %q", fn.Sym) + } + functions[fn.Sym] = fn + } + addresses := map[string]x86Continuation{} + owners := map[string]string{} + for gi, group := range opt.X86TailGroups { + if len(group.Helpers) == 0 { + return nil, fmt.Errorf("empty continuation group %q", group.Root) + } + members := append([]string{group.Root}, group.Helpers...) + for mi, name := range members { + fn, exists := functions[name] + if !exists || owners[name] != "" { + return nil, fmt.Errorf("missing or repeated continuation TEXT %q", name) + } + owners[name] = group.Root + if fn.X86RawText != nil || fn.FrameSize != 0 { + return nil, fmt.Errorf("continuation TEXT %q requires a zero frame and no native layout dependency", name) + } + if len(fn.Instrs) == 0 { + return nil, fmt.Errorf("empty continuation TEXT %q", name) + } + last := fn.Instrs[len(fn.Instrs)-1] + if (last.Op != "RET" || len(last.Args) != 0) && last.Op != "JMP" { + return nil, fmt.Errorf("continuation TEXT %q has no explicit terminating transfer", name) + } + if mi == 0 { + continue + } + sig, ok := opt.Sigs[resolve(name)] + if !ok || sig.Ret != Void || len(sig.Args) != 0 || fn.ArgSize != 0 || + len(sig.Frame.Params)+len(sig.Frame.Results) != 0 { + return nil, fmt.Errorf("continuation %q must be a zero-frame void helper", name) + } + addresses[name] = x86Continuation{ + root: group.Root, label: fmt.Sprintf("p9_cont_%d_%d", gi, mi), + } + } + } + + // These addresses are local control-flow labels, not callable functions or + // byte-addressable code. Refuse a group before emitting any changed ABI. + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + for ai, arg := range ins.Args { + if arg.Kind != OpSym { + continue + } + base, off, ok := parseSBRef(arg.Sym) + target, continuation := addresses[strings.TrimPrefix(base, "$")] + if !ok || !continuation { + continue + } + if off != 0 || ins.Op == "CALL" || (ins.Op == "JMP" && owners[fn.Sym] != target.root) { + return nil, fmt.Errorf("invalid continuation reference in %q: %s", fn.Sym, ins.Raw) + } + if ins.Op == "JMP" && len(ins.Args) == 1 { + continue + } + address := ins.Op == "LEAQ" || ins.Op == "LEAL" || + ((ins.Op == "MOVQ" || ins.Op == "MOVL") && strings.HasPrefix(arg.Sym, "$")) + if !address || ai != 0 || len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return nil, fmt.Errorf("continuation %q is not readable or callable code: %s", base, ins.Raw) + } + } + } + } + for _, d := range file.Data { + if d.Addr != "" { + base, _, _ := parseSBRef(d.Addr) + if _, ok := addresses[strings.TrimPrefix(base, "$")]; ok { + return nil, fmt.Errorf("continuation DATA relocations require an explicit initializer") + } + } + } + + fused := map[string]Func{} + for gi, group := range opt.X86TailGroups { + root := functions[group.Root] + root.Instrs = nil + root.x86IndirectLabels = nil + for mi, name := range append([]string{group.Root}, group.Helpers...) { + fn := functions[name] + prefix := fmt.Sprintf("p9_body_%d_%d_", gi, mi) + if mi != 0 { + label := addresses[name].label + root.Instrs = append(root.Instrs, Instr{Op: OpLABEL, Args: []Operand{{Kind: OpLabel, Sym: label}}}) + root.x86IndirectLabels = append(root.x86IndirectLabels, label) + } + labels := map[string]string{} + for i, ins := range fn.Instrs { + if ins.Op == OpLABEL && len(ins.Args) == 1 { + labels[ins.Args[0].Sym] = fmt.Sprintf("%s%d", prefix, i) + } + } + for _, ins := range fn.Instrs { + ins.Args = append([]Operand(nil), ins.Args...) + for ai, arg := range ins.Args { + switch arg.Kind { + case OpLabel: + ins.Args[ai].Sym = labels[arg.Sym] + case OpIdent: + if label, ok := labels[arg.Ident]; ok { + ins.Args[ai].Ident = label + } + case OpReg: + if label, ok := labels[string(arg.Reg)]; ok && (ins.Op == "JMP" || isAMD64ConditionalBranch(ins.Op)) { + ins.Args[ai] = Operand{Kind: OpIdent, Ident: label} + } + case OpSym: + base, off, ok := parseSBRef(arg.Sym) + if target, exists := addresses[base]; ok && exists && off == 0 && ins.Op == "JMP" { + ins.Args[ai] = Operand{Kind: OpIdent, Ident: target.label} + } + } + } + root.Instrs = append(root.Instrs, ins) + } + } + fused[group.Root] = root + } + result := *file + result.Funcs = nil + result.x86Continuations = addresses + for _, fn := range file.Funcs { + if _, helper := addresses[fn.Sym]; helper { + continue + } + if root, ok := fused[fn.Sym]; ok { + fn = root + } + fn.x86ContinuationAddresses = addresses + result.Funcs = append(result.Funcs, fn) + } + return &result, nil +} diff --git a/amd64_continuation_test.go b/amd64_continuation_test.go new file mode 100644 index 000000000..25d66051b --- /dev/null +++ b/amd64_continuation_test.go @@ -0,0 +1,274 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// The empty helper declarations describe Go entries, not the register/frame +// contract of a JMP from another TEXT. This fixture borrows the root frame. +const x86ContinuationSource = ` +TEXT ·initTable(SB),$0-0 + LEAQ ·step(SB),AX + MOVQ AX,table<>(SB) + RET +GLOBL table<>(SB),16,$8 +TEXT ·run(SB),$0-24 + MOVQ out+0(FP),DI + MOVQ n+8(FP),BX + MOVQ $7,R15 + MOVQ R15,X13 + LEAQ table<>(SB),R14 + XORQ R15,R15 + TESTQ BX,BX + JEQ done +dispatch: + MOVQ (R14),AX + JMP AX +done: + JMP ·exit(SB) +TEXT ·step(SB),$0-0 + MOVQ X13,DX + ADDQ DX,R15 + SUBQ $1,BX + JMP ·check(SB) +TEXT ·check(SB),$0-0 + JEQ done + MOVQ (R14),AX + JMP AX +done: + JMP ·exit(SB) +TEXT ·exit(SB),$0-0 + MOVQ R15,(DI) + MOVQ R15,result+16(FP) + RET +` + +func x86ContinuationOptions(triple string) Options { + return Options{ + Goarch: "amd64", TargetTriple: triple, + X86TailGroups: []X86TailGroup{{Root: "·run", Helpers: []string{"·step", "·check", "·exit"}}}, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }, + Sigs: map[string]FuncSig{ + "initTable": {Ret: Void}, + "run": { + Args: []LLVMType{Ptr, I64}, Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }, + Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0, Field: -1}}, + }, + }, + "step": {Ret: Void}, "check": {Ret: Void}, "exit": {Ret: Void}, + }, + } +} + +func TestX86ContinuationBorrowedFrame(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", x86ContinuationSource, true) + ir := x86ContinuationIR(t, "x86_64-unknown-linux-gnu", x86ContinuationSource) + if !strings.Contains(ir, "indirectbr") || !strings.Contains(ir, "blockaddress") { + t.Fatal("continuations must branch within the root CFG") + } + if strings.Contains(ir, "define void @step") || strings.Contains(ir, "call i64 %") { + t.Fatal("a continuation must not be emitted as a fresh function invocation") + } +} + +func x86ContinuationIR(t *testing.T, triple, source string) string { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86ContinuationOptions(triple)) + if err != nil { + t.Fatal(err) + } + return ir +} + +func TestX86ContinuationLLVM(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH == "amd64" || crossRosetta { + runX86ContinuationGo(t) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + for _, memoryJump := range []bool{false, true} { + source := x86ContinuationSource + if memoryJump { + source = strings.ReplaceAll(source, " MOVQ (R14),AX\n JMP AX", " JMP (R14)") + } + ir := x86ContinuationIR(t, triple, source) + compileLLVMToObject(t, llc, triple, "continuation.ll", "continuation.o", ir) + if (runtime.GOARCH == "amd64" && runtime.GOOS == "linux" && triple == "x86_64-unknown-linux-gnu") || + ((runtime.GOARCH == "amd64" || crossRosetta) && runtime.GOOS == "darwin" && triple == "x86_64-apple-darwin") { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang not found") + } + runX86Continuation(t, llc, clang, triple, ir) + } + } + }) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir := x86Continuation386IR(t, triple) + compileLLVMToObject(t, llc, triple, "continuation386.ll", "continuation386.o", ir) + }) + } +} + +func x86Continuation386IR(t *testing.T, triple string) string { + t.Helper() + source := strings.NewReplacer( + " MOVQ R15,X13\n", "", " MOVQ X13,DX\n ADDQ DX,R15", " ADDL $7,DX", + "R15", "DX", "R14", "SI", "MOVQ", "MOVL", "LEAQ", "LEAL", + "XORQ", "XORL", "TESTQ", "TESTL", "SUBQ", "SUBL", + "n+8(FP)", "n+4(FP)", "result+16(FP)", "result+8(FP)", + "$0-24", "$0-12", "GLOBL table<>(SB),16,$8", "GLOBL table<>(SB),16,$4", + ).Replace(x86ContinuationSource) + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86ContinuationOptions(triple) + opt.Goarch = "386" + sig := opt.Sigs["run"] + sig.Args[1], sig.Ret = I32, I32 + sig.Frame.Params[1].Offset, sig.Frame.Params[1].Type = 4, I32 + sig.Frame.Results[0].Offset, sig.Frame.Results[0].Type = 8, I32 + opt.Sigs["run"] = sig + ir, err := Translate(file, opt) + if err != nil { + t.Fatal(err) + } + return ir +} + +func runX86Continuation(t *testing.T, llc, clang, triple, ir string) { + t.Helper() + if dir := os.Getenv("PLAN9ASM_RUNTIME_ARTIFACT_DIR"); dir != "" { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + for name, data := range map[string]string{"continuation.ll": ir, "main.c": x86ContinuationMain} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0644); err != nil { + t.Fatal(err) + } + } + } + compiler := []string{clang} + var runner []string + if runtime.GOARCH != "amd64" && strings.Contains(triple, "darwin") { + compiler = append(compiler, "-target", triple) + runner = []string{"/usr/bin/arch", "-x86_64"} + } + if strings.Contains(triple, "linux") { + // The shared harness invokes llc with its default static relocation + // model. Match that object when linking a native executable. + compiler = append(compiler, "-no-pie") + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "continuation", triple, ir, x86ContinuationMain, runner) +} + +func runX86ContinuationGo(t *testing.T) { + t.Helper() + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module example.com/continuation\n\ngo 1.20\n", + "continuation_amd64.s": strings.ReplaceAll(x86ContinuationSource, "(SB),$0-", "(SB),4,$0-"), + "continuation.go": "package p\nfunc initTable()\nfunc run(out *uint64, n uint64) (result uint64)\nfunc step()\nfunc check()\nfunc exit()\n", + "continuation_test.go": `package p +import "testing" +func TestState(t *testing.T) { + initTable() + for _, n := range []uint64{0, 1, 2, 19, 100000} { + out := [3]uint64{0x1234, ^uint64(0), 0x5678} + result := run(&out[1], n) + if result != n*7 || out[1] != result || out[0] != 0x1234 || out[2] != 0x5678 { + t.Fatalf("n=%d result=%d out=%v", n, result, out) + } + } +} +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + // asmdecl describes callable entries, not the helpers' borrowed frame. + // This oracle tests Go's actual assembler/linker/runtime semantics. + cmd := exec.Command("go", "test", "-vet=off", "-count=1", ".") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("native Go continuation oracle: %v\n%s", err, out) + } +} + +func TestX86ContinuationRejectsInvalidGroups(t *testing.T) { + for _, test := range []struct { + name string + change func(*File, *Options) + }{ + {"other_arch", func(f *File, o *Options) { f.Arch = ArchWASM; o.Goarch = "wasm" }}, + {"missing_root", func(f *File, o *Options) { o.X86TailGroups[0].Root = "absent" }}, + {"missing_helper", func(f *File, o *Options) { o.X86TailGroups[0].Helpers[0] = "absent" }}, + {"duplicate", func(f *File, o *Options) { o.X86TailGroups = append(o.X86TailGroups, o.X86TailGroups[0]) }}, + {"empty", func(f *File, o *Options) { o.X86TailGroups[0].Helpers = nil }}, + {"local_frame", func(f *File, o *Options) { f.Funcs[2].FrameSize = 8 }}, + {"root_frame", func(f *File, o *Options) { f.Funcs[1].FrameSize = 8 }}, + {"fallthrough", func(f *File, o *Options) { f.Funcs[2].Instrs = append(f.Funcs[2].Instrs, Instr{Op: "NOP"}) }}, + {"code_read", func(f *File, o *Options) { f.Funcs[0].Instrs[1].Op = "MOVQ" }}, + {"arg_frame", func(f *File, o *Options) { f.Funcs[2].ArgSize = 8 }}, + {"typed_entry", func(f *File, o *Options) { o.Sigs["step"] = FuncSig{Args: []LLVMType{I64}, Ret: Void} }}, + {"callable", func(f *File, o *Options) { + f.Funcs[0].Instrs[0] = Instr{Op: "CALL", Args: []Operand{{Kind: OpSym, Sym: "·step(SB)"}}} + }}, + {"offset", func(f *File, o *Options) { f.Funcs[0].Instrs[1].Args[0].Sym = "·step+1(SB)" }}, + {"data", func(f *File, o *Options) { f.Data = []DataStmt{{Sym: "table<>", Width: 8, Addr: "·step(SB)"}} }}, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, x86ContinuationSource) + if err != nil { + t.Fatal(err) + } + opt := x86ContinuationOptions("x86_64-unknown-linux-gnu") + test.change(file, &opt) + if _, err := coalesceX86Continuations(file, opt); err == nil { + t.Fatal("invalid continuation proof accepted") + } + }) + } +} + +const x86ContinuationMain = ` +#include +extern void initTable(void); +extern uint64_t run(uint64_t *, uint64_t); +int main(void) { + initTable(); + const uint64_t counts[] = {0, 1, 2, 19, 100000}; + for (unsigned i = 0; i < sizeof(counts) / sizeof(counts[0]); i++) { + uint64_t out[3] = {0x1234, UINT64_MAX, 0x5678}; + uint64_t result = run(out + 1, counts[i]); + if (result != counts[i] * 7 || out[1] != result || + out[0] != 0x1234 || out[2] != 0x5678) return 1; + } + return 0; +} +` diff --git a/amd64_cpu_query_consumers_test.go b/amd64_cpu_query_consumers_test.go new file mode 100644 index 000000000..73308627d --- /dev/null +++ b/amd64_cpu_query_consumers_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86CPUQueryTypedGrammarAndDispatchBoundaries(t *testing.T) { + if len(x86CPUQuerySpecs) != 2 { + t.Fatal("complete official CPU-query family changed") + } + for _, op := range []Op{OpCPUID, OpXGETBV} { + spec := x86CPUQuerySpecs[op] + inputs, outputs, encoding := []Reg{AX, CX}, []Reg{AX, BX, CX, DX}, []byte{0xa2} + if op == OpXGETBV { + inputs, outputs, encoding = []Reg{CX}, []Reg{AX, DX}, []byte{0x01, 0xd0} + } + if spec.native != strings.ToLower(string(op)) || !reflect.DeepEqual(spec.inputs, inputs) || + !reflect.DeepEqual(spec.outputs, outputs) || !bytes.Equal(spec.opBytes, encoding) { + t.Fatal("CPU-query typed syntax or implicit effect changed", op) + } + for _, arch := range []string{"386", "amd64", "", "arm", "arm64", "wasm"} { + for _, suffix := range []string{"", ".Z", ".SAE", ".BCST"} { + for _, identity := range []Op{"", op, Op(strings.ToLower(string(op))), "MOVQ", Op(string(op) + ".BAD")} { + t.Run(arch+"/"+string(op)+suffix+"/identity="+string(identity), func(t *testing.T) { + instruction := Instr{Op: identity, Raw: string(op) + suffix} + before := fmt.Sprintf("%#v", instruction) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + ir, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerX86CPUQuery(Op(string(op)+suffix), instruction) + invalid := arch != "386" && arch != "amd64" || suffix != "" || + identity != "" && !strings.EqualFold(string(identity), string(op)) + if !known || terminated || (err != nil) != invalid || + invalid && (ir != output.String() || temporary != ctx.tmp) || before != fmt.Sprintf("%#v", instruction) { + t.Fatal("CPU-query dispatch mutated input or emitted before rejection", err) + } + }) + } + } + } + } + if _, known, err := x86CPUQueryProperties("RDTSC", Instr{}, "amd64"); known || err != nil { + t.Fatal("timestamps must not acquire the CPU-query grammar") + } +} + +func TestX86CPUQueryConstructedFileConsumersRetainParserGrammar(t *testing.T) { + pkg := mustGoPackage(t, "cpuquery", "package cpuquery\nfunc query()\n") + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + for _, arch := range []string{"386", "amd64"} { + for _, op := range []Op{OpCPUID, OpXGETBV} { + for _, operand := range []string{"AX", "(AX)", "$1", "AX,DX"} { + t.Run(arch+"/"+string(op)+"/"+operand, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·query(SB),4,$0-0\n"+string(op)+"\nRET\n") + if err != nil { + t.Fatal(err) + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + file.Funcs[0].Instrs[1].Args, err = parseOperandsCSV(ArchAMD64, op, operand) + if err != nil { + t.Fatal(err) + } + before := fmt.Sprintf("%#v", file.Funcs[0]) + options := Options{Goarch: arch, ResolveSym: resolve, Sigs: signatures} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("constructed text consumer bypassed Go's zero-operand grammar", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + t.Fatal("constructed module consumer returned an invalid query module", err) + } + if err == nil || before != fmt.Sprintf("%#v", file.Funcs[0]) { + t.Fatal("constructed module grammar changed its source or silently accepted it", err) + } + }) + } + } + } +} + +func TestX86CPUQueryDescriptorCannotSupplyCallerOrStackPermission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + move, suffix := OpMOVQ, "Q" + if arch == "386" { + move, suffix = OpMOVL, "L" + } + for _, op := range []Op{OpCPUID, OpXGETBV} { + instruction := Instr{Op: op, Raw: string(op)} + if _, known, err := x86CPUQueryProperties(op, instruction, arch); !known || err != nil { + t.Fatal("valid CPU-query grammar is not bound", op, err) + } + if x86SourceStackInstructionKnown(instruction, move, suffix, arch) { + t.Fatal("implicit-register descriptor granted a native caller/stack contract", op) + } + } + } +} + +func x86CPUQueryIR(t *testing.T, source, arch, triple, api string) string { + t.Helper() + return x86CPUQueryIRFromDeclaration(t, source, "func query()\n", arch, triple, api) +} + +func x86CPUQueryIRFromDeclaration(t *testing.T, source, declarations, arch, triple, api string) string { + t.Helper() + pkg := mustGoPackage(t, "cpuquery", "package cpuquery\n"+declarations) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "Go" { + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: resolve}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil || translated == nil || translated.Module.IsNil() { + t.Fatalf("Go module CPU-query consumer: %v", err) + } + return translated.Module.String() + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, ResolveSym: resolve, Sigs: signatures} + if api == "text" { + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + return ir + } + if api != "module" { + t.Fatal("unknown CPU-query consumer", api) + } + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil || module.IsNil() { + t.Fatalf("LLVM module CPU-query consumer: %v", err) + } + return module.String() +} + +func TestX86CPUQueryNamedRawThreeAPIsFiveTargetsProduceActualObjects(t *testing.T) { + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM 22 llc and llvm-objdump required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + for _, op := range []Op{OpCPUID, OpXGETBV} { + for _, raw := range []bool{false, true} { + body := string(op) + "\n" + if raw { + body = "BYTE $0x0f\n" + encoding := []byte{0xa2} + if op == OpXGETBV { + encoding = []byte{0x01, 0xd0} + } + for _, value := range encoding { + body += fmt.Sprintf("BYTE $%#x\n", value) + } + } + source := "TEXT ·query(SB),4,$0-0\n" + body + "RET\n" + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+string(op)+fmt.Sprint("/raw=", raw)+"/"+api, func(t *testing.T) { + ir := x86CPUQueryIR(t, source, target.arch, target.triple, api) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprint("O2=", optimized), func(t *testing.T) { + input := ir + if optimized { + input = x86UndefinedTrapOptimizedIR(t, input) + } + if strings.Count(input, "asm sideeffect \""+strings.ToLower(string(op))+"\"") != 1 { + t.Fatal("whole CPU query was removed or duplicated") + } + dir := t.TempDir() + ll, object := filepath.Join(dir, "query.ll"), filepath.Join(dir, "query.o") + if err := os.WriteFile(ll, []byte(input), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, "-filetype=obj", "-mtriple="+target.triple, ll, "-o", object)) + actual := x86FloatCompareCaptureCommand(t, exec.Command(objdump, "--disassemble", object)) + physical := regexp.MustCompile(`(?m)\b` + strings.ToLower(string(op)) + `\b`) + if len(physical.FindAll(actual, -1)) != 1 { + t.Fatalf("actual LLVM object lost native CPU-query identity:\n%s", actual) + } + }) + } + }) + } + } + } + } +} diff --git a/amd64_cpu_query_family_test.go b/amd64_cpu_query_family_test.go new file mode 100644 index 000000000..9ed2764a1 --- /dev/null +++ b/amd64_cpu_query_family_test.go @@ -0,0 +1,127 @@ +package plan9asm + +import ( + "bytes" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86CPUQueryCompleteOfficialRowsAndImplicitRegisters(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(table), "var ynone = []ytab{\n\t{Zlit, 1, argList{}},\n}") { + t.Fatal("complete Go zero-operand table changed") + } + for _, form := range []struct { + op Op + row string + encoding []byte + registers []Reg + }{ + {OpCPUID, "{ACPUID, ynone, Pm, opBytes{0xa2}}", []byte{0x0f, 0xa2, 0xc3}, []Reg{AX, BX, CX, DX}}, + {OpXGETBV, "{AXGETBV, ynone, Pm, opBytes{01, 0xd0}}", []byte{0x0f, 0x01, 0xd0, 0xc3}, []Reg{AX, CX, DX}}, + } { + if !strings.Contains(string(table), form.row) { + t.Fatal("complete Go CPU-query encoding changed", form.op) + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch+"/"+string(form.op), func(t *testing.T) { + source := "TEXT query(SB),4,$0-0\n" + string(form.op) + "\nRET\n" + if actual := assembleX87ControlBytes(t, arch, source); !bytes.Equal(actual, form.encoding) { + t.Fatalf("original Go CPU-query identity: %x", actual) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, _ := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + for _, register := range form.registers { + if ctx.regSlot[register] == "" { + t.Errorf("implicit CPU-query register %s has no allocated slot", register) + } + } + }) + } + } +} + +func TestX86CPUQueryBareAcceptedSourceAcrossThreeAPIs(t *testing.T) { + pkg := mustGoPackage(t, "cpuquery", "package cpuquery\nfunc query()\n") + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + for _, arch := range []string{"386", "amd64"} { + for _, op := range []Op{OpCPUID, OpXGETBV} { + source := "TEXT ·query(SB),4,$0-0\n" + string(op) + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(arch+"/"+string(op)+"/"+api, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, ResolveSym: resolve, Sigs: signatures} + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + module, translateErr := TranslateModuleInContext(ctx, file, options) + err = translateErr + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "Go": + translated, translateErr := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, ResolveSym: resolve}) + err = translateErr + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + ir = translated.Module.String() + } + } + if err != nil || !strings.Contains(ir, "asm sideeffect \""+strings.ToLower(string(op))+"\"") { + t.Fatalf("accepted zero-operand query lost by %s consumer: %v\n%s", api, err, ir) + } + }) + } + } + } +} + +func TestX86CPUQueryConstructedInvalidOperandsRejectBeforeIR(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []Op{OpCPUID, OpXGETBV} { + for _, operand := range []string{"AX", "(AX)", "$1", "AX,DX"} { + t.Run(arch+"/"+string(op)+"/"+operand, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT query(SB),4,$0-0\n"+string(op)+" "+operand+"\nRET\n", "", false) + // Parse already rejects these source forms. Constructed File/helper + // callers must not bypass that grammar and silently emit a query. + args, err := parseOperandsCSV(ArchAMD64, op, operand) + if err != nil { + t.Fatal(err) + } + fn := Func{Instrs: []Instr{{Op: OpMOVL, Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}}}, {Op: OpMOVL, Args: []Operand{{Kind: OpReg, Reg: DX}, {Kind: OpReg, Reg: AX}}}}} + ctx, output := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerArith(op, Instr{Op: op, Args: args, Raw: string(op) + " " + operand}) + if !known || terminated || err == nil || before != output.String() || temporary != ctx.tmp { + t.Fatalf("invalid CPU-query form emitted IR: %t/%t/%v", known, terminated, err) + } + }) + } + } + } +} diff --git a/amd64_cpu_query_forms.go b/amd64_cpu_query_forms.go new file mode 100644 index 000000000..7fd54551e --- /dev/null +++ b/amd64_cpu_query_forms.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86CPUQuerySpec struct { + native string + inputs []Reg + outputs []Reg + opBytes []byte // Complete Go ynone/Pm opcode row, after its 0x0f prefix. +} + +var x86CPUQuerySpecs = map[Op]x86CPUQuerySpec{ + OpCPUID: {"cpuid", []Reg{AX, CX}, []Reg{AX, BX, CX, DX}, []byte{0xa2}}, + OpXGETBV: {"xgetbv", []Reg{CX}, []Reg{AX, DX}, []byte{0x01, 0xd0}}, +} + +func x86CPUQueryProperties(op Op, ins Instr, arch string) (x86CPUQuerySpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86CPUQuerySpecs[Op(name)] + if !known { + return x86CPUQuerySpec{}, false, nil + } + if len(ins.Args) != 0 || dotted || arch != "amd64" && arch != "386" || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s requires the exact zero-operand Go CPU-query form: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +func (spec x86CPUQuerySpec) resultType() LLVMType { + return LLVMType("{ " + strings.TrimSuffix(strings.Repeat("i32, ", len(spec.outputs)), ", ") + " }") +} + +func (spec x86CPUQuerySpec) constraints() string { + var fields []string + for _, register := range spec.outputs { + fields = append(fields, "={"+strings.ToLower(string(register))+"}") + } + for _, register := range spec.inputs { + fields = append(fields, "{"+strings.ToLower(string(register))+"}") + } + // Preserve source ordering at the compiler boundary. These conservative + // clobbers do not assert that XGETBV serializes hardware or changes flags. + fields = append(fields, "~{dirflag}", "~{fpsr}", "~{flags}", "~{memory}") + return strings.Join(fields, ",") +} + +func emitX86CPUQuery(b *strings.Builder, spec x86CPUQuerySpec, inputs []string, newTmp func() string) []string { + arguments := make([]string, len(inputs)) + for index, input := range inputs { + arguments[index] = "i32 " + input + } + call := newTmp() + fmt.Fprintf(b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", + call, spec.resultType(), spec.native, spec.constraints(), strings.Join(arguments, ", ")) + outputs := make([]string, len(spec.outputs)) + for index := range outputs { + part := newTmp() + fmt.Fprintf(b, " %%%s = extractvalue %s %%%s, %d\n", part, spec.resultType(), call, index) + outputs[index] = "%" + part + } + return outputs +} + +func (c *amd64Ctx) lowerX86CPUQuery(op Op, ins Instr) (bool, bool, error) { + spec, known, err := x86CPUQueryProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + inputs := make([]string, len(spec.inputs)) + for index, register := range spec.inputs { + value, err := c.loadReg(register) + if err != nil { + return true, false, err + } + inputs[index] = c.truncI64(value, I32) + } + outputs := emitX86CPUQuery(c.b, spec, inputs, c.newTmp) + for index, register := range spec.outputs { + if err := c.storeRegSized(register, I32, outputs[index]); err != nil { + return true, false, err + } + } + return true, false, nil +} diff --git a/amd64_cpu_query_linux_amd64_test.go b/amd64_cpu_query_linux_amd64_test.go new file mode 100644 index 000000000..447a75f5e --- /dev/null +++ b/amd64_cpu_query_linux_amd64_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CPUQuery386IndependentC() string { + return strings.NewReplacer( + "uint64_t", "uint32_t", "UINT64_C(0xabcdef0123456789)", "UINT32_C(0x23456789)", + "UINT64_C(0x13579bdf2468ace0)", "UINT32_C(0x2468ace0)", + "UINT64_C(0x7e6d5c4b3a291807)", "UINT32_C(0x3a291807)", + "UINT64_C(0xa1b2c3d400000000)", "UINT32_C(0)", + ).Replace(x86CPUQueryIndependentC) +} + +// Real Linux IA32 execution is required separately from Rosetta, emulators +// and portable object compilation. Missing LLVM 22/cross tools fail this root. +func TestCrossLinuxRuntimeMatrixX86CPUQueryNative386(t *testing.T) { + x87IA32RequireNativeHost(t) + llc := findLLVM22Tool("llc") + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if llc == "" || err != nil { + t.Fatalf("required LLVM 22/native IA32 compiler: %v", err) + } + dir := t.TempDir() + referenceSource, referenceExecutable := filepath.Join(dir, "reference.c"), filepath.Join(dir, "reference") + if err := os.WriteFile(referenceSource, []byte("#define REFERENCE_ONLY 1\n"+x86CPUQuery386IndependentC()), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", referenceSource, "-O2", "-o", referenceExecutable)) + referenceProgram, err := x87IA32StaticExecutable(referenceExecutable) + if err != nil { + t.Fatal(err) + } + reference := string(x87IA32RunDirect(t, referenceProgram, nil, os.Environ())) + lines := strings.Split(strings.TrimSpace(reference), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "cpu query ") || !strings.HasPrefix(lines[1], "xgetbv ") { + t.Fatalf("actual native IA32 query reference: %q", reference) + } + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + original := x86CPUQueryBuildOriginal386(t, raw) + for _, suffix := range []string{"", "_af"} { + if err := original.verify(); err != nil { + t.Fatal(err) + } + output, err := x87IA32Capture(t, original.path, []string{suffix}, "", append(os.Environ(), "GOTRACEBACK=system")) + if guard := original.verify(); guard != nil { + t.Fatal(guard) + } + if lines[1] == "xgetbv SIGILL" { + if !x86UndefinedTrapOriginalSignal(output, err) || !strings.HasPrefix(string(output), lines[0]+"\n") { + t.Fatalf("original native IA32 query fault: %v\n%s", err, output) + } + } else if err != nil || string(output) != reference { + t.Fatalf("original native IA32 query state: %v\n%s", err, output) + } + } + declarations, assembly := x86CPUQueryRuntimeFixtures(raw, "386") + pkg := mustGoPackage(t, "cpuquery386", "package cpuquery386\n"+declarations) + ctx := llvm.NewContext() + defer ctx.Dispose() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + GOARCH: "386", TargetTriple: "i386-unknown-linux-gnu", + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if translated != nil && !translated.Module.IsNil() { + if err != nil { + translated.Module.Dispose() + } + } + if err != nil || translated == nil || translated.Module.IsNil() { + t.Fatalf("native IA32 query module: %v", err) + } + ir := translated.Module.String() + verification := llvm.VerifyModule(translated.Module, llvm.ReturnStatusAction) + translated.Module.Dispose() + if verification != nil { + t.Fatal(verification) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + dir := t.TempDir() + ll, object := filepath.Join(dir, "query.ll"), filepath.Join(dir, "query.o") + main, executable := filepath.Join(dir, "main.c"), filepath.Join(dir, "query") + for name, contents := range map[string]string{ll: input, main: x86CPUQuery386IndependentC()} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple=i386-unknown-linux-gnu", "-filetype=obj", ll, "-o", object)) + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", object, main, "-O2", "-o", executable)) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != reference { + t.Fatalf("actual native IA32 LLVM query state: %q / %q", output, reference) + } + }) + } + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_cpu_query_native_reference_test.go b/amd64_cpu_query_native_reference_test.go new file mode 100644 index 000000000..9ff660213 --- /dev/null +++ b/amd64_cpu_query_native_reference_test.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func x86CPUQueryBuildOriginal386(t *testing.T, raw bool) x87IA32Executable { + t.Helper() + declarations, assembly := x86CPUQueryRuntimeFixtures(raw, "386") + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module cpuquery386\n\ngo 1.20\n", "main.go": x86CPUQuery386OriginalDriver(), + "decl.go": "package main\n" + declarations, "query_386.s": assembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-p=1", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS=linux", "GOARCH=386", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil || sealed.verify() != nil { + t.Fatalf("unchanged IA32 query reference did not compile as stable ELF32: %v", err) + } + return sealed +} + +// This portable axis compiles unchanged original Go objects, not a native +// execution pass. The required Linux counterpart runs these same programs. +func TestX86CPUQuery386OriginalNativeReferenceCompiles(t *testing.T) { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + _ = x86CPUQueryBuildOriginal386(t, raw) + }) + } +} + +func TestX86CPUQuery386RuntimeSourceObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, raw := range []bool{false, true} { + declarations, source := x86CPUQueryRuntimeFixtures(raw, "386") + goAssemblerOracleOutput(t, "386", source, "", true) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + for _, api := range []string{"text", "module", "Go"} { + t.Run(fmt.Sprintf("%s/raw=%t/%s", triple, raw, api), func(t *testing.T) { + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "386", triple, api) + for _, op := range []string{"cpuid", "xgetbv"} { + if strings.Count(ir, "asm sideeffect \""+op+"\"") != 2 { + t.Fatal("native IA32 query reference lost a complete opcode", op) + } + } + compileLLVMToObject(t, llc, triple, "query.ll", "query.o", ir) + compileLLVMToObject(t, llc, triple, "query-opt.ll", "query-opt.o", x86UndefinedTrapOptimizedIR(t, ir)) + }) + } + } + } +} diff --git a/amd64_cpu_query_runtime_fixtures_test.go b/amd64_cpu_query_runtime_fixtures_test.go new file mode 100644 index 000000000..70c4508bc --- /dev/null +++ b/amd64_cpu_query_runtime_fixtures_test.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86CPUQueryRuntimeFixtures(raw bool, arch string) (declarations, assembly string) { + for _, op := range []Op{OpCPUID, OpXGETBV} { + name := "query" + if op == OpXGETBV { + name = "fault" + } + body := string(op) + "\n" + if raw { + body = "BYTE $0x0f\nBYTE $0xa2\n" + if op == OpXGETBV { + body = "BYTE $0x0f\nBYTE $0x01\nBYTE $0xd0\n" + } + } + for index, seed := range []uint64{0x8c7, 0x12} { + entry := name + if index != 0 { + entry += "_af" + } + declarations += fmt.Sprintf("func %s(out *[8]uint64)\n", entry) + assembly += fmt.Sprintf(`TEXT ·%s(SB),4,$0-8 + MOVQ out+0(FP),DI + MOVQ $0xa1b2c3d400000000,AX + MOVQ $0xa1b2c3d400000000,CX + MOVQ $0x7e6d5c4b3a291807,BX + MOVQ $-1,DX + PUSHQ $%#x + POPFQ + %s + MOVQ AX,0(DI) + MOVQ BX,8(DI) + MOVQ CX,16(DI) + MOVQ DX,24(DI) + PUSHFQ + POPQ R8 + MOVQ R8,32(DI) + STMXCSR 40(DI) + RET +`, entry, seed, body) + } + } + if arch == "386" { + declarations = strings.ReplaceAll(declarations, "uint64", "uint32") + assembly = strings.NewReplacer( + "$0-8", "$0-4", "MOVQ", "MOVL", "PUSHQ", "PUSHL", "POPFQ", "POPFL", + "PUSHFQ", "PUSHFL", "POPQ R8", "POPL SI", "R8,32(DI)", "SI,16(DI)", + "$0xa1b2c3d400000000", "$0", "$0x7e6d5c4b3a291807", "$0x3a291807", + "BX,8(DI)", "BX,4(DI)", "CX,16(DI)", "CX,8(DI)", "DX,24(DI)", "DX,12(DI)", + "40(DI)", "20(DI)", + ).Replace(assembly) + } else if arch != "amd64" { + panic("CPU-query fixture requires an x86 architecture") + } + return +} + +const x86CPUQueryOriginalDriver = `package main + +import ( + "fmt" + "os" + "syscall" +) + +func main() { + if err := syscall.Setrlimit(syscall.RLIMIT_CORE, &syscall.Rlimit{}); err != nil { + panic(err) + } + cpu, state, flags := query, fault, uint64(0x8c5) + if len(os.Args) == 2 && os.Args[1] == "_af" { + cpu, state, flags = query_af, fault_af, 0x10 + } + sample := func(call func(*[8]uint64)) [8]uint64 { + out := [8]uint64{0,0,0,0,0,0,0xabcdef0123456789,0x13579bdf2468ace0} + call(&out) + if out[4]&0x8d5 != flags || out[6] != 0xabcdef0123456789 || out[7] != 0x13579bdf2468ace0 { + panic(fmt.Sprintf("original query flags/canary: %x", out)) + } + return out + } + out := sample(cpu) + for _, value := range out[:4] { + if value>>32 != 0 { panic("query output did not zero-extend") } + } + fmt.Printf("cpu query %x %x %x %x\n", out[0],out[1],out[2],out[3]) + out = sample(state) + if out[0]>>32 != 0 || out[3]>>32 != 0 || out[1] != 0x7e6d5c4b3a291807 || out[2] != 0xa1b2c3d400000000 { + panic(fmt.Sprintf("original XGETBV implicit registers: %x", out)) + } + fmt.Printf("xgetbv %x %x\n", out[0],out[3]) +} +` + +func x86CPUQuery386OriginalDriver() string { + return strings.NewReplacer( + "uint64", "uint32", "0xabcdef0123456789", "0x23456789", "0x13579bdf2468ace0", "0x2468ace0", + "0x7e6d5c4b3a291807", "0x3a291807", "0xa1b2c3d400000000", "0", + ).Replace(x86CPUQueryOriginalDriver) +} diff --git a/amd64_cpu_query_runtime_unix_test.go b/amd64_cpu_query_runtime_unix_test.go new file mode 100644 index 000000000..3df4dde82 --- /dev/null +++ b/amd64_cpu_query_runtime_unix_test.go @@ -0,0 +1,186 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86CPUQueryOriginalAndLLVM(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + referenceSource, referenceExecutable := filepath.Join(dir, "reference.c"), filepath.Join(dir, "reference") + if err := os.WriteFile(referenceSource, []byte("#define REFERENCE_ONLY 1\n"+x86CPUQueryIndependentC), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, referenceSource, referenceExecutable) + reference := string(execution.run(t, referenceExecutable)) + lines := strings.Split(strings.TrimSpace(reference), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "cpu query ") || !strings.HasPrefix(lines[1], "xgetbv ") { + t.Fatalf("independent native CPU/OS-state reference: %q", reference) + } + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + declarations, assembly := x86CPUQueryRuntimeFixtures(raw, "amd64") + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + original := x86CPUQueryBuildOriginal(t, execution, declarations, assembly) + for _, suffix := range []string{"", "_af"} { + arguments := execution.command(original) + arguments = append(arguments, suffix) + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, arguments[0], arguments[1:]...) + command.Env = append(os.Environ(), "GOTRACEBACK=system") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if lines[1] == "xgetbv SIGILL" { + if !x86UndefinedTrapOriginalSignal(output, err) || !strings.HasPrefix(string(output), lines[0]+"\n") { + t.Fatalf("unchanged Go XGETBV did not retain its actual unavailable-ISA fault: %v\n%s", err, output) + } + } else if err != nil || string(output) != reference { + t.Fatalf("original Go query values/zero extension/flags/canaries differ from native reference: %v\n%s\nreference=%s", err, output, reference) + } + } + ir := x86NarrowOracleIR(t, assembly, "package cpuquery\n"+declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "cpuquery", input, x86CPUQueryIndependentC, level) + if string(output) != reference { + t.Fatalf("LLVM query result/fault differs from independent native reference: %q / %q", output, reference) + } + }) + } + }) + } +} + +func x86CPUQueryBuildOriginal(t *testing.T, execution x86FloatCompareExecution, declarations, assembly string) string { + t.Helper() + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module cpuquery\n\ngo 1.20\n", "main.go": x86CPUQueryOriginalDriver, + "decl.go": "package main\n" + declarations, "query_amd64.s": assembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-p=1", "-o", executable, ".") + command.Dir = dir + goos := runtime.GOOS + if strings.Contains(execution.triple, "linux") { + goos = "linux" + } + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+goos, "GOARCH=amd64", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + return executable +} + +const x86CPUQueryIndependentC = ` +#include +#include +#include +#include +#include +#include +#include +#include + +// Observe the actual instruction, not an advertised CPUID capability. An +// emulator may mask OSXSAVE while still implementing XGETBV successfully. +static int native_state(unsigned *low, unsigned *high) { + int channel[2]; + if (pipe(channel)) return -1; + pid_t child=fork(); + if (child<0) { + close(channel[0]); + close(channel[1]); + return -1; + } + if (child==0) { + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) _exit(120); + if (close(channel[0])) _exit(121); + unsigned values[2]; + __asm__ volatile("xgetbv" : "=a"(values[0]),"=d"(values[1]) : "c"(0)); + if (write(channel[1],values,sizeof(values))!=sizeof(values) || close(channel[1])) _exit(122); + _exit(0); + } + if (close(channel[1])) return -1; + int status=0; + pid_t waited; + do { waited=waitpid(child,&status,0); } while (waited<0 && errno==EINTR); + unsigned values[2]={0,0}; + ssize_t count=read(channel[0],values,sizeof(values)); + if (close(channel[0]) || waited!=child) return -1; + if (WIFSIGNALED(status) && WTERMSIG(status)==SIGILL && count==0) return 0; + if (!WIFEXITED(status) || WEXITSTATUS(status)!=0 || count!=sizeof(values)) return -1; + *low=values[0]; + *high=values[1]; + return 1; +} + +#if !REFERENCE_ONLY +extern void query(uint64_t *); +extern void query_af(uint64_t *); +extern void fault(uint64_t *); +extern void fault_af(uint64_t *); + +static int check(uint64_t *out, uint64_t flags, unsigned csr) { + return (out[4]&0x8d5)==flags && out[5]==csr && + out[6]==UINT64_C(0xabcdef0123456789) && out[7]==UINT64_C(0x13579bdf2468ace0); +} +#endif + +int main(void) { + unsigned a,b,c,d,low=0,high=0; + int available=native_state(&low,&high); + if (available<0) return 10; + if (!__get_cpuid(0,&a,&b,&c,&d)) return 11; +#if !REFERENCE_ONLY + void (*cpu[])(uint64_t *)={query,query_af}; + void (*state[])(uint64_t *)={fault,fault_af}; + const uint64_t flags[]={0x8c5,0x10}; + for (unsigned index=0;index<2;index++) { + uint64_t out[8]={0,0,0,0,0,0,UINT64_C(0xabcdef0123456789),UINT64_C(0x13579bdf2468ace0)}; + unsigned before,after; + __asm__ volatile("stmxcsr %0" : "=m"(before)); + cpu[index](out); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + if (!check(out,flags[index],before) || after!=before || out[0]!=a || out[1]!=b || out[2]!=c || out[3]!=d) return 12; + if (available) { + state[index](out); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + if (!check(out,flags[index],before) || after!=before || out[0]!=low || out[3]!=high || + out[1]!=UINT64_C(0x7e6d5c4b3a291807) || out[2]!=UINT64_C(0xa1b2c3d400000000)) return 13; + } else { + pid_t child=fork(); + if (child<0) return 14; + if (child==0) { + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) _exit(120); + state[index](out); + _exit(91); + } + int status=0; + if (waitpid(child,&status,0)!=child || !WIFSIGNALED(status) || WTERMSIG(status)!=SIGILL) return 15; + } + } +#endif + printf("cpu query %x %x %x %x\n",a,b,c,d); + if (available) printf("xgetbv %x %x\n",low,high); + else puts("xgetbv SIGILL"); + return 0; +} +` diff --git a/amd64_ctx.go b/amd64_ctx.go index 3eeef7edb..6e032f45c 100644 --- a/amd64_ctx.go +++ b/amd64_ctx.go @@ -16,9 +16,12 @@ type amd64Ctx struct { goarch string targetTriple string - resolve func(string) string - sigs map[string]FuncSig - annotate bool + resolve func(string) string + sigs map[string]FuncSig + annotate bool + continuations map[string]x86Continuation + indirectLabels []string + indirectABI0 *FuncSig // proven straight-line stack-argument callback adapter tmp int @@ -31,9 +34,8 @@ type amd64Ctx struct { usedRegs map[Reg]bool regSlot map[Reg]string // gp reg -> alloca name - zeroUpperVector bool // VEX/EVEX writes clear the upper register bits - usedXRegs map[int]bool - xRegSlot map[int]string // xmm reg index -> alloca name (<16 x i8>) + usedXRegs map[int]bool + xRegSlot map[int]string // xmm reg index -> alloca name (<16 x i8>) usedYRegs map[int]bool yRegSlot map[int]string // ymm reg index -> alloca name (<32 x i8>) @@ -51,28 +53,41 @@ type amd64Ctx struct { x87IntegerSlot string // hardware FISTP result for explicit 386 x87 lowering x87Mode X87Mode - flagsZSlot string - flagsSltSlot string // signed negative-style bit for J{L,LE,G,GE}-like checks - flagsCFSlot string // carry/borrow style bit for J{B,BE,A,AE,NC,C}-like checks - flagsOFSlot string // overflow-style bit used by ADOX carry chain modeling - flagsIDSlot string // CPUID availability bit preserved by PUSHFL/POPFL - flagsWritten bool - allowSPWrite bool // current instruction has an explicitly modeled 386 SP destination - directionSlot string // x86 DF, used by MOVS/STOS/SCAS instructions - repeatPrefix string // pending REP/REPN prefix for the following instruction - vstackSlot string // [64 x i64] virtual stack for PUSHQ/POPQ - vspSlot string // i64 virtual stack pointer (next free slot) - localStackSlot string // byte-addressable backing storage for x86 SP references - classicFrame string // contiguous classic Go ABI frame used by 386 FP addressing - classicSize int64 - classicBias int64 - - fpParams map[int64]FrameSlot // off(FP) -> slot - fpResults []FrameSlot - fpResAllocaOff map[int64]string // off(FP) -> alloca - fpResAllocaIdx map[int]string // result index -> alloca - fpResWritten map[int]bool // result index -> whether written via +off(FP) - fpResAddrTaken map[int]bool // result index -> address of fp_ret_* escaped + flagsZSlot string + flagsSltSlot string // architectural SF; signed conditions combine it with OF + flagsCFSlot string // carry/borrow style bit for J{B,BE,A,AE,NC,C}-like checks + flagsPFSlot string // parity bit for J{P,PE,PS,NP,PO,PC}-like checks + flagsAFSlot string // auxiliary carry/borrow from bit 3, exposed by LAHF/PUSHF + flagsOFSlot string // overflow-style bit used by ADOX carry chain modeling + flagsIDSlot string // CPUID ID bit preserved by native-width flags stack forms + flagsWritten bool + allowSPWrite bool // current instruction has an explicitly modeled 386 SP destination + directionSlot string // x86 DF, used by MOVS/STOS/SCAS instructions + repeatPrefix string // pending REP/REPN prefix for the following instruction + namedAtomicLock bool // actual same-block source-adjacent LOCK for this instruction only + lockScalarNS string // compiler name allocation, not a source capability + vstackSlot string // [64 x i64] virtual stack for PUSHQ/POPQ + vspSlot string // i64 virtual stack pointer (next free slot) + localStackSlot string // byte-addressable backing storage for x86 SP references + sourceStack *x86SourceStackProof + fixedLocal *x86FixedLocalProof + fixedMemRoles map[MemRef]MemRef // exact certified current-instruction native role projection + frameSize int64 // local bytes below the Go assembler's implicit amd64 BP + classicFrame string // contiguous classic Go ABI frame used by 386 FP addressing + classicSize int64 + classicBias int64 + + fpParams map[int64]FrameSlot // off(FP) -> slot + fpParamAlloca map[int64]string // off(FP) -> mutable parameter shadow + fpResults []FrameSlot + fpResAllocaOff map[int64]string // off(FP) -> alloca + fpResAllocaIdx map[int]string // result index -> alloca + fpResWritten map[int]bool // result index -> whether written via +off(FP) + fpResAddrTaken map[int]bool // result index -> address of fp_ret_* escaped + fpScalarSource Func // source for all-path checks of new scalar result views + fpScalarData map[x86StaticSBAddress]int64 // exact effective SB address -> available emitted numerical bytes + fpScalarChecked bool + fpScalarCheckErr error } func newAMD64Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) string, sigs map[string]FuncSig, annotate bool) *amd64Ctx { @@ -88,6 +103,10 @@ func newX86Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) st resolve: resolve, sigs: sigs, annotate: annotate, + frameSize: fn.FrameSize, + sourceStack: fn.x86SourceStack, + fixedLocal: fn.x86FixedLocal, + indirectABI0: inferX86ABI0Forwarder(fn, sig, goarch), blocks: amd64SplitBlocks(fn), usedRegs: map[Reg]bool{}, regSlot: map[Reg]string{}, @@ -100,15 +119,23 @@ func newX86Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) st usedKRegs: map[int]bool{}, kRegSlot: map[int]string{}, fpParams: map[int64]FrameSlot{}, + fpParamAlloca: map[int64]string{}, fpResAllocaOff: map[int64]string{}, fpResAllocaIdx: map[int]string{}, fpResWritten: map[int]bool{}, fpResAddrTaken: map[int]bool{}, + fpScalarSource: fn, blockByIdx: map[int]int{}, } for _, s := range sig.Frame.Params { c.fpParams[s.Offset] = s } + c.continuations = make(map[string]x86Continuation, len(fn.x86ContinuationAddresses)) + for symbol, target := range fn.x86ContinuationAddresses { + target.root = resolve(target.root) + c.continuations[resolve(symbol)] = target + } + c.indirectLabels = fn.x86IndirectLabels c.fpResults = append([]FrameSlot(nil), sig.Frame.Results...) base := 0 for i, blk := range c.blocks { @@ -120,7 +147,7 @@ func newX86Ctx(b *strings.Builder, fn Func, sig FuncSig, resolve func(string) st } func (c *amd64Ctx) useHardwareX87() bool { - return c.goarch == "386" && c.x87Mode != X87Software + return c.x87Mode != X87Software } func (c *amd64Ctx) emitSourceComment(ins Instr) { @@ -136,7 +163,7 @@ func (c *amd64Ctx) newTmp() string { } func (c *amd64Ctx) slotName(r Reg) string { - return "%" + amd64LLVMBlockName("reg_"+string(r)) + return "%reg_" + amd64LLVMSafeName(string(r)) } func (c *amd64Ctx) xSlotName(i int) string { @@ -208,6 +235,10 @@ func (c *amd64Ctx) scanUsedRegs() { if r == "" { return } + // Byte aliases are read and written through their containing GP slot. + if base, _, ok := amd64ByteAlias(r); ok { + r = base + } if idx, ok := amd64ParseXReg(r); ok { c.usedXRegs[idx] = true return @@ -224,7 +255,7 @@ func (c *amd64Ctx) scanUsedRegs() { c.usedKRegs[idx] = true return } - if _, ok := amd64ParseX87Reg(r); ok && c.goarch == "386" { + if _, ok := amd64ParseX87Reg(r); ok { c.usedX87 = true return } @@ -235,9 +266,13 @@ func (c *amd64Ctx) scanUsedRegs() { case OpReg: markReg(op.Reg) case OpMem: - markReg(op.Mem.Base) - if op.Mem.Index != "" { - markReg(op.Mem.Index) + memory := op.Mem + if named, valid := x86SBMemoryAddressRole(op, c.goarch); valid { + memory = named // An ignored Reg must not allocate a virtual stack. + } + markReg(memory.Base) + if memory.Index != "" { + markReg(memory.Index) } case OpRegList: for _, r := range op.RegList { @@ -248,27 +283,124 @@ func (c *amd64Ctx) scanUsedRegs() { for _, blk := range c.blocks { for _, ins := range blk.instrs { + if form, known, err := x86ScalarNativeLoadProperties(ins.Op, ins, c.goarch); known && err == nil { + markReg(form.destination) + } + if effect, known, err := x86TypedPackedIntegerCompareByteEffect(ins.Op, ins, c.goarch); known && err == nil { + for _, operand := range effect.operands { + markOp(operand.operand) + } + } op := strings.ToUpper(string(ins.Op)) - if c.goarch == "386" && isX87Op(Op(op)) { + if dot := strings.IndexByte(op, '.'); dot >= 0 { + op = op[:dot] + } + if isX87Op(Op(op)) { c.usedX87 = true if op == "FMOVVP" { c.usesX87Convert = true } } + if op == "LAHF" || op == "SAHF" { + markReg(AX) + } + if spec, ok := amd64ImplicitSystemSpecs[Op(op)]; ok { + registers := spec.inputs | spec.outputs + if registers&amd64ImplicitAX != 0 { + markReg(AX) + } + if registers&amd64ImplicitCX != 0 { + markReg(CX) + } + if registers&amd64ImplicitDX != 0 { + markReg(DX) + } + } + if spec, known, err := x86TimestampProperties(Op(op), ins, c.goarch); known && err == nil { + for _, register := range x86TimestampRegisters(spec) { + markReg(register) + } + } + if spec, known, err := x86CPUQueryProperties(Op(op), ins, c.goarch); known && err == nil { + for _, register := range spec.inputs { + markReg(register) + } + for _, register := range spec.outputs { + markReg(register) + } + } + if spec, known, err := amd64PackedStringCompareProperties(Op(op), ins, c.goarch); known && err == nil { + output, _, _ := spec.resultEffect() + markReg(output) + if spec.explicitLength { + markReg(AX) + markReg(DX) + } + } + if spec, ok := amd64RTMSpecs[Op(op)]; ok && spec.outputAX { + markReg(AX) + } + if spec, ok := amd64SystemTransferSpecs[Op(op)]; ok { + if spec.inputs&amd64SystemTransferCX != 0 { + markReg(CX) + } + if spec.inputs&amd64SystemTransferDX != 0 { + markReg(DX) + } + if c.goarch != "386" && spec.inputs&amd64SystemTransferR11 != 0 { + markReg(Reg("R11")) + } + } + if _, ok := amd64LeaveSpecs[Op(op)]; ok { + markReg(BP) + markReg(SP) + } + if op == "XLAT" { + markReg(AX) + markReg(BX) + } if c.goarch == "386" { switch op { - case "PUSHL", "POPL", "PUSHFL", "POPFL", "PUSHAL", "POPAL", "ADJSP": + case "PUSHW", "POPW", "PUSHFW", "POPFW", "PUSHL", "POPL", "PUSHFL", "POPFL", "PUSHAL", "POPAL", "ADJSP": markReg(SP) } } for _, arg := range ins.Args { markOp(arg) } + if form, known, err := x86PackedAverageProperties(ins.Op, ins, c.goarch); known && err == nil { + markOp(form.source) + } + if form, recognized, err := x86ScalarExtensionProperties(Op(op), ins, c.goarch); recognized && err == nil && form.source.Kind == OpReg { + markReg(form.source.Reg) + } + if _, fourSource := amd64FourSourceSpecs[Op(op)]; fourSource && len(ins.Args) >= 2 && ins.Args[1].Kind == OpRegList && len(ins.Args[1].RegList) != 0 { + first := ins.Args[1].RegList[0] + if index, ok := amd64ParseXReg(first); ok { + base := index &^ 3 + for register := base; register < base+4; register++ { + markReg(Reg(fmt.Sprintf("X%d", register))) + } + } else if index, ok := amd64ParseZReg(first); ok { + base := index &^ 3 + for register := base; register < base+4; register++ { + markReg(Reg(fmt.Sprintf("Z%d", register))) + } + } + } + if len(ins.Args) == 1 && ins.Args[0].Kind == OpReg { + if spec, ok := x86UnaryYmbSpecs[Op(op)]; ok { + markReg(amd64YmbEffectiveRegister(ins.Args[0].Reg, spec.bits)) + } + } } } // Ensure a few common regs exist even if only used implicitly by helpers. markReg(AX) + if c.goarch != "386" && c.frameSize > 0 && c.usedRegs[BP] { + markReg(SP) + } // Ensure arg regs exist for ABIInternal-style stdlib asm. This matters for: // - functions like runtime·cmpstring that tail-call helpers @@ -296,7 +428,14 @@ func (c *amd64Ctx) scanUsedRegs() { } func (c *amd64Ctx) emitEntryAllocas() error { + if c.fixedLocal != nil && (c.fixedLocal.binding == nil || c.fixedLocal.binding != c.sig.x86FixedLocal || + c.goarch != "amd64" || c.targetTriple != c.fixedLocal.binding.target || c.frameSize != c.fixedLocal.frame) { + return goAMD64FrameContext("fixed local storage consumer lost its sealed source identity") + } c.scanUsedRegs() + if c.sourceStack != nil || c.fixedLocal != nil { + c.usedRegs[SP] = true + } regs := make([]string, 0, len(c.usedRegs)) for r := range c.usedRegs { @@ -304,7 +443,7 @@ func (c *amd64Ctx) emitEntryAllocas() error { } sort.Strings(regs) - c.b.WriteString(amd64LLVMBlockName("entry") + ":\n") + c.b.WriteString(amd64LLVMBlockName(c.blocks[0].name) + ":\n") for _, rs := range regs { r := Reg(rs) name := c.slotName(r) @@ -312,31 +451,73 @@ func (c *amd64Ctx) emitEntryAllocas() error { fmt.Fprintf(c.b, " %s = alloca i64\n", name) fmt.Fprintf(c.b, " store i64 0, ptr %s\n", name) } - if spSlot, ok := c.regSlot[SP]; ok && c.goarch == "386" { - minOff, maxOff := c.stackOffsetRange() - movement, err := c.stackMovementBudget() - if err != nil { - return err + if spSlot, ok := c.regSlot[SP]; ok { + if c.fixedLocal != nil { + // The complete proof excludes saved BP/return/incoming FP bytes, + // proves every observed word-aligned SP residue, and prevents local + // addresses escaping or being compared numerically. A word-aligned + // physical buffer preserves that residue domain, not a stronger + // SIMD-aligned shadow capable of repairing an uncertain source. + size := c.fixedLocal.frame + 64 + c.localStackSlot = "%fixed_local_native" + fmt.Fprintf(c.b, " %s = alloca [%d x i8], align 8\n", c.localStackSlot, size) + base := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr inbounds [%d x i8], ptr %s, i32 0, i64 32\n", base, size, c.localStackSlot) + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i64\n", address, base) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", address, spSlot) + } else { + minOff, maxOff := c.stackOffsetRange() + if c.sourceStack != nil && c.sourceStack.minimum < minOff { + minOff = c.sourceStack.minimum + } + var movement int64 + if c.goarch == "386" { + var err error + movement, err = c.stackMovementBudget() + if err != nil { + return err + } + } + minOff -= movement + maxOff += movement + const guard = int64(64) + bias := guard - minOff + if c.sourceStack != nil { + // A native Go entry SP is word-aligned. Preserve that same + // congruence so proved AND alignment has identical byte offsets. + word := c.sourceStack.word + bias = (bias + word - 1) & -word + } + size := bias + maxOff + guard + if size < 256 { + size = 256 + } + c.localStackSlot = "%local_stack" + if c.sourceStack != nil { + fmt.Fprintf(c.b, " %s = alloca [%d x i8], align %d\n", c.localStackSlot, size, c.sourceStack.word) + } else { + fmt.Fprintf(c.b, " %s = alloca [%d x i8]\n", c.localStackSlot, size) + } + base := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr inbounds [%d x i8], ptr %s, i32 0, i64 %d\n", base, size, c.localStackSlot, bias) + addr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i64\n", addr, base) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", addr, spSlot) + if bpSlot, ok := c.regSlot[BP]; ok && c.goarch != "386" && c.frameSize > 0 { + // Go saves BP before reserving TEXT's local frame. Thus a source + // -8(BP) aliases FrameSize-8(SP), including across outgoing calls. + bp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %d\n", bp, addr, c.frameSize) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", bp, bpSlot) + } } - minOff -= movement - maxOff += movement - const guard = int64(64) - bias := guard - minOff - size := bias + maxOff + guard - if size < 256 { - size = 256 - } - c.localStackSlot = "%local_stack" - fmt.Fprintf(c.b, " %s = alloca [%d x i8]\n", c.localStackSlot, size) - base := c.newTmp() - fmt.Fprintf(c.b, " %%%s = getelementptr inbounds [%d x i8], ptr %s, i32 0, i64 %d\n", base, size, c.localStackSlot, bias) - addr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i64\n", addr, base) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s\n", addr, spSlot) } - // XMM, YMM and ZMM are overlapping views of the same physical register. - // Allocate the widest view used by the function and share its low bytes. + // Keep all live X/Y/Z views on one physical byte bank. Encoding-specific + // writers still decide whether to preserve or clear the upper bytes; + // source mnemonic prefixes cannot select that effect (Go rewrites some + // V-prefixed scalar literals into legacy SSE instructions). widths := make(map[int]int) for i := range c.usedXRegs { widths[i] = 16 @@ -357,8 +538,7 @@ func (c *amd64Ctx) emitEntryAllocas() error { name := c.xSlotName(i) if width == 32 { name = fmt.Sprintf("%%y%d", i) - } - if width == 64 { + } else if width == 64 { name = fmt.Sprintf("%%z%d", i) } c.xRegSlot[i] = name @@ -401,21 +581,24 @@ func (c *amd64Ctx) emitEntryAllocas() error { fmt.Fprintf(c.b, " %s = alloca i64\n", c.x87IntegerSlot) } } - c.flagsZSlot = "%flags_z" c.flagsSltSlot = "%flags_slt" c.flagsCFSlot = "%flags_cf" + c.flagsPFSlot = "%flags_pf" + c.flagsAFSlot = "%flags_af" c.flagsOFSlot = "%flags_of" - if c.goarch == "386" { - c.directionSlot = "%direction_backward" - c.flagsIDSlot = "%flags_id" - } + c.directionSlot = "%direction_backward" + c.flagsIDSlot = "%flags_id" fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsZSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsZSlot) fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsSltSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsSltSlot) fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsCFSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsPFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsPFSlot) + fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsAFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsAFSlot) fmt.Fprintf(c.b, " %s = alloca i1\n", c.flagsOFSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) if c.flagsIDSlot != "" { @@ -445,12 +628,31 @@ func (c *amd64Ctx) emitEntryAllocas() error { return err } } else { + // Named FP parameters have mutable frame-slot semantics in Go assembly. + // Keep shadow storage even though LLVM function arguments are SSA values; + // assembly kernels commonly advance slice pointers by writing base+off(FP) + // and reading the updated value in a later loop iteration. + for _, slot := range c.sig.Frame.Params { + if slot.Index < 0 || slot.Index >= len(c.sig.Args) { + return fmt.Errorf("FP frame slot: invalid arg index %d at +%d(FP)", slot.Index, slot.Offset) + } + name := amd64FPParamSlotName(slot.Offset) + c.fpParamAlloca[slot.Offset] = name + value := fmt.Sprintf("%%arg%d", slot.Index) + if fields := frameSlotFields(slot); len(fields) != 0 { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s%s\n", extracted, c.sig.Args[slot.Index], value, frameSlotExtractSuffix(slot)) + value = "%" + extracted + } + fmt.Fprintf(c.b, " %s = alloca %s\n", name, x86FPStorageType(slot.Type)) + c.storeFPStorage(slot.Type, value, name, "", "") + } for _, r := range c.fpResults { name := fmt.Sprintf("%%fp_ret_%d", r.Index) c.fpResAllocaIdx[r.Index] = name c.fpResAllocaOff[r.Offset] = name - fmt.Fprintf(c.b, " %s = alloca %s\n", name, r.Type) - fmt.Fprintf(c.b, " store %s %s, ptr %s\n", r.Type, llvmZeroValue(r.Type), name) + fmt.Fprintf(c.b, " %s = alloca %s\n", name, x86FPStorageType(r.Type)) + c.storeFPStorage(r.Type, llvmZeroValue(r.Type), name, "", "") } } @@ -525,6 +727,13 @@ func (c *amd64Ctx) emitEntryAllocas() error { return nil } +func amd64FPParamSlotName(off int64) string { + if off < 0 { + return fmt.Sprintf("%%fp_arg_n%d", -off) + } + return fmt.Sprintf("%%fp_arg_%d", off) +} + func x86FrameTypeSize(ty LLVMType) int64 { switch ty { case I1, I8: @@ -591,29 +800,33 @@ func (c *amd64Ctx) emit386ClassicFrame() error { return fmt.Errorf("FP frame slot: invalid arg index %d at +%d(FP)", slot.Index, slot.Offset) } value := fmt.Sprintf("%%arg%d", slot.Index) - if slot.Field >= 0 { + if fields := frameSlotFields(slot); len(fields) != 0 { extracted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", extracted, c.sig.Args[slot.Index], value, slot.Field) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s%s\n", extracted, c.sig.Args[slot.Index], value, frameSlotExtractSuffix(slot)) value = "%" + extracted } - fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", slot.Type, value, c.classicFramePtr(slot.Offset)) + c.storeFPStorage(slot.Type, value, c.classicFramePtr(slot.Offset), ", align 1", "") } for _, slot := range c.fpResults { ptr := c.classicFramePtr(slot.Offset) c.fpResAllocaIdx[slot.Index] = ptr c.fpResAllocaOff[slot.Offset] = ptr - fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", slot.Type, llvmZeroValue(slot.Type), ptr) + c.storeFPStorage(slot.Type, llvmZeroValue(slot.Type), ptr, ", align 1", "") } return nil } func (c *amd64Ctx) stackOffsetRange() (minOff, maxOff int64) { + maxOff = c.frameSize for _, block := range c.blocks { for _, ins := range block.instrs { for _, arg := range ins.Args { if arg.Kind != OpMem || arg.Mem.Base != SP { continue } + if _, static := x86SBMemoryAddressRole(arg, c.goarch); static { + continue + } if arg.Mem.Off < minOff { minOff = arg.Mem.Off } @@ -648,6 +861,10 @@ func (c *amd64Ctx) stackMovementBudget() (int64, error) { if err := add(4, ins); err != nil { return 0, err } + case "PUSHW", "POPW", "PUSHFW", "POPFW": + if err := add(2, ins); err != nil { + return 0, err + } case "PUSHAL", "POPAL": if err := add(32, ins); err != nil { return 0, err @@ -664,6 +881,26 @@ func (c *amd64Ctx) stackMovementBudget() (int64, error) { return 0, err } } + case "INCL", "DECL": + if is386SPDestination(ins) { + if err := add(1, ins); err != nil { + return 0, err + } + } + case "INCW", "DECW": + if is386SPDestination(ins) { + // A low-word wrap can move the modeled SP by as much as 65535. + if err := add(65535, ins); err != nil { + return 0, err + } + } + case "ADDW", "SUBW": + if is386SPDestination(ins) { + // A low-word update can move the modeled SP by up to 65535. + if err := add(65535, ins); err != nil { + return 0, err + } + } case "ANDL": if movement, ok := bounded386SPAndMovement(ins); ok { if err := add(movement, ins); err != nil { @@ -709,15 +946,68 @@ func bounded386SPAndMovement(ins Instr) (int64, bool) { } func models386SPWrite(ins Instr) bool { + op := strings.ToUpper(string(ins.Op)) + if dot := strings.IndexByte(op, '.'); dot >= 0 { + op = op[:dot] + } + if op == "MOVQ" && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg && ins.Args[1].Reg == SP && ins.Args[0].Kind == OpReg { + class, _, special := x86MachineRegister(ins.Args[0].Reg) + return special && (class == "cr" || class == "dr") + } + if len(ins.Args) == 1 && ins.Args[0].Kind == OpReg { + if spec, ok := x86UnaryYmbSpecs[Op(op)]; ok && spec.bits != 64 { + return amd64YmbEffectiveRegister(ins.Args[0].Reg, spec.bits) == SP + } + } if !is386SPDestination(ins) { return false } - switch strings.ToUpper(string(ins.Op)) { - case "MOVL", "ADDL", "SUBL", "LEAL", "POPL": + if _, ok := amd64BMI2ShiftSpecs[op]; ok { + return true + } + if _, known, err := x86ScalarFloatIntegerProperties(ins, "386"); known { + return err == nil + } + switch op { + case "MOVL", "ADDW", "SUBW", "ADCW", "SBBW", "ADDL", "SUBL", "ADCL", "SBBL", "ADCXL", "ADOXL", "LEAW", "LEAL", "POPW", "POPL", + "CRC32B", "CRC32W", "CRC32L", + "CMPXCHGW", "CMPXCHGL", + "POPCNTW", "POPCNTL", "TZCNTW", "TZCNTL", "LZCNTW", "LZCNTL", "PMOVMSKB", "VPMOVMSKB", + "BEXTRL", "BEXTRQ", "BZHIL", "BZHIQ", "BLSIL", "BLSIQ", "BLSMSKL", "BLSMSKQ", "BLSRL", "BLSRQ", + "ANDNL", "ANDNQ", + "RORXL", "RORXQ", + "PDEPL", "PDEPQ", "PEXTL", "PEXTQ": // These are the direct SP forms used by the official 386 corpus. MOVL // and non-SP LEAL rebase to an explicitly supplied stack context; // arithmetic and SP-relative LEAL retain the current stack model. return true + case "RDPID": + return len(ins.Args) == 1 && ins.Args[0].Kind == OpReg && ins.Args[0].Reg == SP + case "RCLW", "RCLL", "RCRW", "RCRL", "ROLW", "ROLL", "RORW", "RORL", "SARW", "SARL", + "SALW", "SALL", "SHLW", "SHLL", "SHRW", "SHRL": + return true + case "INCW", "DECW", "INCL", "DECL": + return true + case "BTCW", "BTCL", "BTRW", "BTRL", "BTSW", "BTSL": + return true + case "SLDTW", "SLDTL", "SMSWW", "SMSWL", "STRW", "STRL": + // Go's Yml destination class includes SP in 386 mode. These system + // instructions write their result directly to that destination. + return true + case "LARW", "LARL", "LSLW", "LSLL": + // LAR/LSL use the same Yrl destination class, including SP. + return true + case "LFSW", "LFSL", "LGSW", "LGSL", "LSSW", "LSSL": + // Far-pointer loads also use Yrl, including SP, for their offset. + return true + case "RDFSBASEL", "RDGSBASEL": + // The 386 FSGSBASE read forms use Yrl destinations, including SP. + return true + case "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", + "MOVWLSX", "MOVWLZX", "MOVLQZX", "MOVSWW", "MOVZWW": + return true + case "KMOVB", "KMOVW", "KMOVD", "KMOVQ": + return len(ins.Args) == 2 && amd64IsKOperand(ins.Args[0]) case "ANDL": _, ok := bounded386SPAndMovement(ins) return ok @@ -760,6 +1050,37 @@ func (c *amd64Ctx) popI64() string { return "%" + val } +func (c *amd64Ctx) pushI16(v string) error { + sp, err := c.loadReg(SP) + if err != nil { + return err + } + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %s, 2\n", next, sp) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return err + } + p := c.ptrFromAddrI64("%" + next) + fmt.Fprintf(c.b, " store i16 %s, ptr %s, align 1\n", v, p) + return nil +} + +func (c *amd64Ctx) popI16() (string, error) { + sp, err := c.loadReg(SP) + if err != nil { + return "", err + } + p := c.ptrFromAddrI64(sp) + v := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s, align 1\n", v, p) + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, 2\n", next, sp) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return "", err + } + return "%" + v, nil +} + func (c *amd64Ctx) pushI32(v string) error { sp, err := c.loadReg(SP) if err != nil { @@ -838,6 +1159,28 @@ func amd64ByteAlias(rr Reg) (base Reg, shift uint, ok bool) { return DX, 0, true case DH: return DX, 8, true + case BPB: + return BP, 0, true + case SIB: + return SI, 0, true + case DIB: + return DI, 0, true + case R8B: + return Reg("R8"), 0, true + case R9B: + return Reg("R9"), 0, true + case R10B: + return Reg("R10"), 0, true + case R11B: + return Reg("R11"), 0, true + case R12B: + return Reg("R12"), 0, true + case R13B: + return Reg("R13"), 0, true + case R14B: + return Reg("R14"), 0, true + case R15B: + return Reg("R15"), 0, true default: return "", 0, false } @@ -877,6 +1220,9 @@ func amd64ByteRegBase(r Reg) (base Reg, shift uint, ok bool) { } func (c *amd64Ctx) loadReg(r Reg) (string, error) { + if r == TLS { + return "", fmt.Errorf("%w: TLS pseudo-register needs its native Go TLS source contract", ErrProbeNeedsContext) + } // Model byte aliases used by stdlib asm. Reads return the selected byte as a // zero-extended i64. if base, shift, ok := amd64ByteAlias(r); ok { @@ -1000,21 +1346,6 @@ func (c *amd64Ctx) storeRegSized(r Reg, ty LLVMType, v string) error { } } -// clearVectorUpper clears bytes above the written width while leaving the -// low view unchanged. Legacy SSE writes deliberately preserve those bytes. -func (c *amd64Ctx) clearVectorUpper(idx, lowBytes int) { - slot, width := c.zRegSlot[idx], 64 - if slot == "" { - slot, width = c.yRegSlot[idx], 32 - } - if slot == "" || width <= lowBytes { - return - } - upper := c.newTmp() - fmt.Fprintf(c.b, " %%%s = getelementptr i8, ptr %s, i64 %d\n", upper, slot, lowBytes) - fmt.Fprintf(c.b, " store <%d x i8> zeroinitializer, ptr %%%s, align 1\n", width-lowBytes, upper) -} - func (c *amd64Ctx) loadX(r Reg) (string, error) { idx, ok := amd64ParseXReg(r) if !ok { @@ -1022,7 +1353,7 @@ func (c *amd64Ctx) loadX(r Reg) (string, error) { } slot, ok := c.xRegSlot[idx] if !ok { - return "<16 x i8> zeroinitializer", nil + return "zeroinitializer", nil } t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s\n", t, slot) @@ -1038,9 +1369,6 @@ func (c *amd64Ctx) storeX(r Reg, v string) error { if !ok { return nil } - if c.zeroUpperVector { - c.clearVectorUpper(idx, 16) - } fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s\n", v, slot) return nil } @@ -1052,7 +1380,7 @@ func (c *amd64Ctx) loadY(r Reg) (string, error) { } slot, ok := c.yRegSlot[idx] if !ok { - return "<32 x i8> zeroinitializer", nil + return "zeroinitializer", nil } t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <32 x i8>, ptr %s\n", t, slot) @@ -1060,19 +1388,12 @@ func (c *amd64Ctx) loadY(r Reg) (string, error) { } func (c *amd64Ctx) storeY(r Reg, v string) error { - idx, ok := amd64ParseYReg(r) - if !ok { + if _, ok := amd64ParseYReg(r); !ok { return fmt.Errorf("not a Y reg: %s", r) } - slot, ok := c.yRegSlot[idx] - if !ok { - return nil - } - if c.zeroUpperVector { - c.clearVectorUpper(idx, 32) - } - fmt.Fprintf(c.b, " store <32 x i8> %s, ptr %s\n", v, slot) - return nil + // A 256-bit VEX write also replaces X and clears the upper Z view. + // Raw and named instructions must observe the same physical register. + return c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: r}, 32, v) } func (c *amd64Ctx) loadZ(r Reg) (string, error) { @@ -1082,7 +1403,7 @@ func (c *amd64Ctx) loadZ(r Reg) (string, error) { } slot, ok := c.zRegSlot[idx] if !ok { - return "<64 x i8> zeroinitializer", nil + return "zeroinitializer", nil } t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <64 x i8>, ptr %s\n", t, slot) @@ -1142,6 +1463,7 @@ func (c *amd64Ctx) setZSFlagsFromI64(v string) { slt := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp slt i64 %s, 0\n", slt, v) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) + c.setParityFlagSized(I64, v) } func (c *amd64Ctx) setZSFlagsFromI32(v string) { @@ -1151,6 +1473,33 @@ func (c *amd64Ctx) setZSFlagsFromI32(v string) { slt := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp slt i32 %s, 0\n", slt, v) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) + c.setParityFlagSized(I32, v) +} + +func (c *amd64Ctx) setParityFlagSized(ty LLVMType, v string) { + low := v + if ty != I8 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to i8\n", truncated, ty, v) + low = "%" + truncated + } + shift4 := c.newTmp() + fold4 := c.newTmp() + shift2 := c.newTmp() + fold2 := c.newTmp() + shift1 := c.newTmp() + fold1 := c.newTmp() + bit := c.newTmp() + parity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i8 %s, 4\n", shift4, low) + fmt.Fprintf(c.b, " %%%s = xor i8 %s, %%%s\n", fold4, low, shift4) + fmt.Fprintf(c.b, " %%%s = lshr i8 %%%s, 2\n", shift2, fold4) + fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, %%%s\n", fold2, fold4, shift2) + fmt.Fprintf(c.b, " %%%s = lshr i8 %%%s, 1\n", shift1, fold2) + fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, %%%s\n", fold1, fold2, shift1) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, 1\n", bit, fold1) + fmt.Fprintf(c.b, " %%%s = icmp eq i8 %%%s, 0\n", parity, bit) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", parity, c.flagsPFSlot) } func (c *amd64Ctx) setCmpFlags(a, b string) { @@ -1165,6 +1514,9 @@ func (c *amd64Ctx) setCmpFlags(a, b string) { ult := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %s\n", ult, a, b) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", ult, c.flagsCFSlot) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %s, %s\n", result, a, b) + c.setParityFlagSized(I64, "%"+result) } func (c *amd64Ctx) loadFlag(slot string) string { @@ -1181,6 +1533,25 @@ func (c *amd64Ctx) fpParam(off int64) (slot FrameSlot, ok bool) { return s, true } +func (c *amd64Ctx) loadFPParamValue(slot FrameSlot) (string, error) { + if slot.Index < 0 || slot.Index >= len(c.sig.Args) { + return "", fmt.Errorf("FP read slot: invalid arg index %d at +%d(FP)", slot.Index, slot.Offset) + } + if c.classicFrame != "" { + return c.loadFPStorage(slot.Type, c.classicFramePtr(slot.Offset), ", align 1"), nil + } + if shadow := c.fpParamAlloca[slot.Offset]; shadow != "" { + return c.loadFPStorage(slot.Type, shadow, ""), nil + } + value := fmt.Sprintf("%%arg%d", slot.Index) + if fields := frameSlotFields(slot); len(fields) != 0 { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s%s\n", extracted, c.sig.Args[slot.Index], value, frameSlotExtractSuffix(slot)) + value = "%" + extracted + } + return value, nil +} + func (c *amd64Ctx) fpResultAlloca(off int64) (string, LLVMType, bool) { s, ok := c.fpParams[off] _ = s @@ -1233,13 +1604,11 @@ func (c *amd64Ctx) evalFPToI64(off int64) (string, error) { } } if alloca, ty, rok := c.fpResultAlloca(off); rok && ty != "" { - ld := c.newTmp() align := "" if c.classicFrame != "" { align = ", align 1" } - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s%s\n", ld, ty, alloca, align) - v := "%" + ld + v := c.loadFPStorage(ty, alloca, align) switch ty { case I64: return v, nil @@ -1283,24 +1652,10 @@ func (c *amd64Ctx) evalFPToI64(off int64) (string, error) { // inference (common in low-level runtime assembly). return "0", nil } - idx := slot.Index - if idx < 0 || idx >= len(c.sig.Args) { - return "", fmt.Errorf("FP read slot: invalid arg index %d at +%d(FP)", idx, off) - } - arg := fmt.Sprintf("%%arg%d", idx) - - // If this FP slot refers to a field within an aggregate argument (string/slice), - // extract that field first. ty := slot.Type - if c.classicFrame != "" { - loaded := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", loaded, ty, c.classicFramePtr(off)) - arg = "%" + loaded - } else if slot.Field >= 0 { - aggTy := c.sig.Args[idx] - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", t, aggTy, arg, slot.Field) - arg = "%" + t + arg, err := c.loadFPParamValue(slot) + if err != nil { + return "", err } switch ty { case Ptr: @@ -1342,33 +1697,139 @@ func (c *amd64Ctx) evalFPToI64(off int64) (string, error) { } } +func amd64IntegerTypeBits(ty LLVMType) (int, bool) { + switch ty { + case I1: + return 1, true + case I8: + return 8, true + case I16: + return 16, true + case I32: + return 32, true + case I64: + return 64, true + default: + return 0, false + } +} + +func amd64IntegerTypeForBits(bits int) LLVMType { + switch bits { + case 1: + return I1 + case 8: + return I8 + case 16: + return I16 + case 32: + return I32 + case 64: + return I64 + default: + return "" + } +} + +func (c *amd64Ctx) coerceFPStoreValue(from, to LLVMType, value string) (string, error) { + if from == to { + return value, nil + } + pointerBits := 64 + if c.goarch == "386" { + pointerBits = 32 + } + + rawType := from + rawValue := value + switch from { + case Ptr: + rawType = amd64IntegerTypeForBits(pointerBits) + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to %s\n", cast, value, rawType) + rawValue = "%" + cast + case LLVMType("float"): + rawType = I32 + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", cast, value) + rawValue = "%" + cast + case LLVMType("double"): + rawType = I64 + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", cast, value) + rawValue = "%" + cast + default: + if _, ok := amd64IntegerTypeBits(from); !ok { + return "", fmt.Errorf("unsupported FP store source type %s", from) + } + } + + targetBits := 0 + switch to { + case Ptr: + targetBits = pointerBits + case LLVMType("float"): + targetBits = 32 + case LLVMType("double"): + targetBits = 64 + default: + var ok bool + targetBits, ok = amd64IntegerTypeBits(to) + if !ok { + return "", fmt.Errorf("unsupported FP store destination type %s", to) + } + } + + fromBits, _ := amd64IntegerTypeBits(rawType) + targetIntType := amd64IntegerTypeForBits(targetBits) + if fromBits > targetBits { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", cast, rawType, rawValue, targetIntType) + rawType, rawValue = targetIntType, "%"+cast + } else if fromBits < targetBits { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", cast, rawType, rawValue, targetIntType) + rawType, rawValue = targetIntType, "%"+cast + } + + switch to { + case Ptr: + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr %s %s to ptr\n", cast, rawType, rawValue) + return "%" + cast, nil + case LLVMType("float"), LLVMType("double"): + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", cast, rawType, rawValue, to) + return "%" + cast, nil + default: + return rawValue, nil + } +} + func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { + return c.storeFPResultWithMetadata(off, ty, v, "") +} + +func (c *amd64Ctx) storeFPResultWithMetadata(off int64, ty LLVMType, v, metadata string) error { align := "" if c.classicFrame != "" { // The 386 ABI frame is a byte array, and FP offsets need not provide // the natural alignment required by the value stored in a slot. align = ", align 1" } - if slot, ok := c.fpParams[off]; ok && c.classicFrame != "" { - // 386 ABIInternal stubs write register arguments back into the - // caller-provided FP argument area before tail-calling an ABI0 - // handler (for example runtime.panicIndex). The classic frame is - // deliberately mutable, unlike an LLVM SSA argument. - value := v - switch { - case ty == slot.Type: - case ty == I32 && slot.Type == Ptr: - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = inttoptr i32 %s to ptr\n", t, v) - value = "%" + t - case ty == Ptr && slot.Type == I32: - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i32\n", t, v) - value = "%" + t - default: - return fmt.Errorf("FP parameter write type mismatch: have %s want %s at +%d(FP)", ty, slot.Type, off) + if slot, ok := c.fpParams[off]; ok { + ptr := c.fpParamAlloca[off] + if c.classicFrame != "" { + ptr = c.classicFramePtr(off) } - fmt.Fprintf(c.b, " store %s %s, ptr %s%s\n", slot.Type, value, c.classicFramePtr(off), align) + if ptr == "" { + return fmt.Errorf("missing mutable FP parameter slot at +%d(FP)", off) + } + value, err := c.coerceFPStoreValue(ty, slot.Type, v) + if err != nil { + return fmt.Errorf("FP parameter write type mismatch: have %s want %s at +%d(FP): %w", ty, slot.Type, off, err) + } + c.storeFPStorage(slot.Type, value, ptr, align, metadata) return nil } if c.goarch == "386" && ty == I32 { @@ -1403,9 +1864,9 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { if slot.Type == LLVMType("double") { cast := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast i64 %%%s to double\n", cast, merged) - fmt.Fprintf(c.b, " store double %%%s, ptr %s%s\n", cast, alloca, align) + fmt.Fprintf(c.b, " store double %%%s, ptr %s%s%s\n", cast, alloca, align, metadata) } else { - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s\n", merged, alloca, align) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s%s\n", merged, alloca, align, metadata) } c.fpResWritten[slot.Index] = true return nil @@ -1415,11 +1876,34 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { if !ok { return fmt.Errorf("unsupported FP write slot: +%d(FP)", off) } + if slotTy == I1 { + value, err := c.coerceFPStoreValue(ty, I1, v) + if err != nil { + return fmt.Errorf("FP bool result write at +%d(FP): %w", off, err) + } + c.storeFPStorage(I1, value, alloca, align, metadata) + c.markFPResultWritten(off) + return nil + } if slotTy != "" && slotTy != ty { if c.goarch == "386" && ty == I32 && slotTy == Ptr { p := c.newTmp() fmt.Fprintf(c.b, " %%%s = inttoptr i32 %s to ptr\n", p, v) - fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s\n", p, alloca, align) + fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s%s\n", p, alloca, align, metadata) + c.markFPResultWritten(off) + return nil + } + if ty == I32 && slotTy == LLVMType("float") { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %s to float\n", cast, v) + fmt.Fprintf(c.b, " store float %%%s, ptr %s%s%s\n", cast, alloca, align, metadata) + c.markFPResultWritten(off) + return nil + } + if ty == LLVMType("float") && slotTy == I32 { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", cast, v) + fmt.Fprintf(c.b, " store i32 %%%s, ptr %s%s%s\n", cast, alloca, align, metadata) c.markFPResultWritten(off) return nil } @@ -1451,41 +1935,68 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", t, ty, v, slotTy) cast = "%" + t } - fmt.Fprintf(c.b, " store %s %s, ptr %s%s\n", slotTy, cast, alloca, align) + fmt.Fprintf(c.b, " store %s %s, ptr %s%s%s\n", slotTy, cast, alloca, align, metadata) c.markFPResultWritten(off) return nil } } + if (ty == I64 || ty == LLVMType("double")) && slotTy == LLVMType("float") { + secondAlloca, secondTy, secondOK := c.fpResultAlloca(off + 4) + if !secondOK || secondTy != LLVMType("float") { + return fmt.Errorf("FP write at +%d(FP) cannot split a 64-bit memory operand", off) + } + bits := v + if ty == LLVMType("double") { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", cast, v) + bits = "%" + cast + } + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", low, bits) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, 32\n", shifted, bits) + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", high, shifted) + lowFloat := c.newTmp() + highFloat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", lowFloat, low) + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", highFloat, high) + fmt.Fprintf(c.b, " store float %%%s, ptr %s%s%s\n", lowFloat, alloca, align, metadata) + fmt.Fprintf(c.b, " store float %%%s, ptr %s%s%s\n", highFloat, secondAlloca, align, metadata) + c.markFPResultWritten(off) + c.markFPResultWritten(off + 4) + return nil + } // Cast integer sizes when needed (common: i64 reg -> i32 return slot). switch { case ty == I64 && slotTy == I32: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", t, v) - fmt.Fprintf(c.b, " store i32 %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store i32 %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == I64 && slotTy == LLVMType("double"): t := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", t, v) - fmt.Fprintf(c.b, " store double %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store double %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == LLVMType("double") && slotTy == I64: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", t, v) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == I64 && slotTy == Ptr: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", t, v) - fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store ptr %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == Ptr && slotTy == I64: t := c.newTmp() fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", t, v) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s\n", t, alloca, align) + fmt.Fprintf(c.b, " store i64 %%%s, ptr %s%s%s\n", t, alloca, align, metadata) c.markFPResultWritten(off) return nil case ty == I64 && slotTy == I64: @@ -1494,7 +2005,7 @@ func (c *amd64Ctx) storeFPResult(off int64, ty LLVMType, v string) error { return fmt.Errorf("FP write type mismatch: have %s want %s at +%d(FP)", ty, slotTy, off) } } - fmt.Fprintf(c.b, " store %s %s, ptr %s%s\n", ty, v, alloca, align) + fmt.Fprintf(c.b, " store %s %s, ptr %s%s%s\n", ty, v, alloca, align, metadata) c.markFPResultWritten(off) return nil } @@ -1508,13 +2019,11 @@ func (c *amd64Ctx) loadFPResult(slot FrameSlot) (string, error) { if !ok { return "", fmt.Errorf("missing fp result alloca for index %d", slot.Index) } - t := c.newTmp() align := "" if c.classicFrame != "" { align = ", align 1" } - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s%s\n", t, slot.Type, alloca, align) - return "%" + t, nil + return c.loadFPStorage(slot.Type, alloca, align), nil } func isAMD64FloatRetTy(ty LLVMType) bool { @@ -1665,6 +2174,15 @@ func (c *amd64Ctx) addrFromMem(mem MemRef) (addrI64 string, err error) { } func (c *amd64Ctx) addrFromPlainMem(mem MemRef) (addrI64 string, err error) { + if mem.Base == TLS || mem.Index == TLS { + return "", fmt.Errorf("%w: TLS memory needs its native Go progedit, relocation and TLS address contract", ErrProbeNeedsContext) + } + if physical, certified := c.fixedMemRoles[mem]; certified { + mem = physical + } + if err := x86SymbolIndexedMemoryError(mem, c.goarch); err != nil { + return "", err + } var cur string if mem.Sym != "" { base, err := c.ptrFromSB(mem.Sym) @@ -1735,18 +2253,24 @@ func (c *amd64Ctx) ptrFromAddrI64(addrI64 string) string { } func (c *amd64Ctx) ptrFromSB(sym string) (ptr string, err error) { + // This entry point consumes memory, not Go's TYPE_ADDR immediate. Address + // evaluators must first select that role explicitly; silently removing '$' + // here lets direct OpSym load/store dispatch bypass the memory grammar. + if strings.HasPrefix(strings.TrimSpace(sym), "$") { + return "", fmt.Errorf("address immediate is not symbolic memory: %q", sym) + } base, off, ok := parseSBRef(sym) if !ok { return "", fmt.Errorf("invalid (SB) sym ref: %q", sym) } - base = strings.TrimPrefix(base, "$") - res := base - if strings.Contains(base, "·") || strings.Contains(base, "/") || strings.Contains(base, ".") { - res = c.resolve(base) - } else { - res = c.resolve("·" + base) - } + res := resolveAssemblyDataSymbol(base, c.resolve) p := llvmGlobal(res) + if continuation, ok := c.continuations[res]; ok { + if off != 0 { + return "", fmt.Errorf("continuation address cannot have a byte offset: %q", sym) + } + return fmt.Sprintf("blockaddress(%s, %%%s)", llvmGlobal(continuation.root), amd64LLVMBlockName(continuation.label)), nil + } if off == 0 { return p, nil } diff --git a/amd64_defined_status_consumers_test.go b/amd64_defined_status_consumers_test.go new file mode 100644 index 000000000..d76747e34 --- /dev/null +++ b/amd64_defined_status_consumers_test.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86StatusFlagMaskClearsExactlyItsPhysicalSubset(t *testing.T) { + // Independent architectural bit positions, not the production slot list. + flags := []struct { + bit uint + slot string + }{ + {0, "%flags_cf"}, {2, "%flags_pf"}, {4, "%flags_af"}, + {6, "%flags_z"}, {7, "%flags_slt"}, {11, "%flags_of"}, + } + for _, arch := range []string{"386", "amd64"} { + for subset := 0; subset < 64; subset++ { + t.Run(fmt.Sprintf("%s/subset=%02x", arch, subset), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + output.Reset() + temporary := ctx.tmp + var mask x86StatusFlagMask + var want strings.Builder + for index, flag := range flags { + if subset&(1<= la && i >= lb || i < la && i < lb", "if int64(i) >= la && int64(i) >= lb || int64(i) < la && int64(i) < lb", + ).Replace(x86DefinedStatusGoDriver) + formatted, err := format.Source([]byte(source)) + if err != nil { + t.Fatal(err) + } + return string(formatted) +} + +func x86DefinedStatusIA32CDriver() string { + var prototypes, kernels strings.Builder + for _, kernel := range x86DefinedStatusIA32Kernels() { + fmt.Fprintf(&prototypes, "extern void %s(uint32_t *, uint32_t, uint32_t, uint32_t);\n", kernel.name) + fmt.Fprintf(&kernels, "{%s,%d,%d,\"%s\"},\n", kernel.name, kernel.kind, kernel.bits, kernel.instruction) + } + source := strings.NewReplacer( + "KERNEL_PROTOTYPES", prototypes.String(), "KERNELS", kernels.String(), + "uint64_t", "uint32_t", "UINT64_MAX", "UINT32_MAX", "UINT64_C", "UINT32_C", + "0xa7a7a7a7a7a7a7a7", "0xa7a7a7a7", + "bits == 64 ?", "bits == 32 ?", + "a[i] = right>>(8*i);", "a[i] = ((uint64_t)right)>>(8*i);", + "b[i] = left>>(8*i);", "b[i] = ((uint64_t)left)>>(8*i);", + ).Replace(x86DefinedStatusCDriver) + return source +} + +func x86DefinedStatusIA32BuildOriginal(t *testing.T) x87IA32Executable { + t.Helper() + dir := t.TempDir() + declarations, assembly := x86DefinedStatusIA32Fixture("386") + inputs := map[string]string{ + "go.mod": "module definedstatus386\n\ngo 1.20\n", + "main.go": x86DefinedStatusIA32GoDriver(t), + "decl.go": "package main\n" + declarations, + "status_386.s": assembly, + } + for name, contents := range inputs { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + t.Logf("original IA32 input=%s bytes=%d sha256=%x", name, len(contents), sha256.Sum256([]byte(contents))) + } + goBinary := filepath.Join(runtime.GOROOT(), "bin", "go") + env := append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", + "GOROOT="+runtime.GOROOT(), "GOFLAGS=", "GOEXPERIMENT=", "GOOS=linux", + "GOARCH=386", "CGO_ENABLED=0", "GOMAXPROCS=2") + version, err := x87IA32Capture(t, goBinary, []string{"version"}, dir, env) + if err != nil || !strings.HasPrefix(string(version), "go version "+runtime.Version()+" ") { + t.Fatalf("original IA32 SDK identity mismatch: %v\n%s", err, version) + } + executable := filepath.Join(dir, "original-go") + output, err := x87IA32Capture(t, goBinary, + []string{"build", "-p=2", "-o", executable, "."}, dir, env) + if err != nil || len(output) != 0 { + t.Fatalf("original IA32 Go build failed: %v\n%s", err, output) + } + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + t.Logf("original Go %s static IA32 ELF bytes=%d sha256=%x; compile-only receipt", + runtime.Version(), sealed.size, sealed.hash) + return sealed +} + +func TestX86DefinedStatusIA32OriginalGoStaticELFActualCompile(t *testing.T) { + executable := x86DefinedStatusIA32BuildOriginal(t) + if err := executable.verify(); err != nil { + t.Fatal(err) + } + t.Log("seven kernels and 896 mathematical observations built; native execution is a separate required root") +} + +func TestX86DefinedStatusIA32RequiredNativeCounterpartSource(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_defined_status_linux_amd64_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var counterpart *ast.FuncDecl + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == "TestCrossLinuxRuntimeMatrixX86DefinedStatusNative386" { + counterpart = function + } + } + if counterpart == nil { + t.Fatal("required original-Go/LLVM O0/O2 native IA32 defined-status counterpart is missing") + } + want := map[string]bool{ + "x87IA32RequireNativeHost": false, + "x86DefinedStatusIA32BuildOriginal": false, + "x87IA32RunDirect": false, + "x86DefinedStatusIA32CompileAndRun": false, + } + levels := map[string]bool{"-O0": false, "-O2": false} + ast.Inspect(counterpart.Body, func(node ast.Node) bool { + if literal, ok := node.(*ast.BasicLit); ok && literal.Kind == token.STRING { + value, err := strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + if _, required := levels[value]; required { + levels[value] = true + } + } + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if function, ok := call.Fun.(*ast.Ident); ok { + if _, required := want[function.Name]; required { + want[function.Name] = true + } + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok && strings.HasPrefix(selector.Sel.Name, "Skip") { + t.Error("required native IA32 counterpart cannot skip failed tools or execution") + } + return true + }) + for function, found := range want { + if !found { + t.Errorf("required native IA32 counterpart lacks %s", function) + } + } + for level, found := range levels { + if !found { + t.Errorf("required native IA32 counterpart lacks %s", level) + } + } + ast.Inspect(file, func(node ast.Node) bool { + literal, ok := node.(*ast.BasicLit) + if ok && literal.Kind == token.STRING { + value, err := strconv.Unquote(literal.Value) + if err != nil { + t.Fatal(err) + } + if value == "qemu-i386" || value == "-exec" { + t.Error("required native IA32 counterpart cannot substitute a guest executor") + } + } + return true + }) +} + +func x86DefinedStatusIA32Capabilities(cpuinfo string) error { + flags := make(map[string]bool) + for _, line := range strings.Split(cpuinfo, "\n") { + key, value, ok := strings.Cut(line, ":") + if ok && strings.TrimSpace(key) == "flags" { + for _, flag := range strings.Fields(value) { + flags[flag] = true + } + } + } + for _, required := range []string{"sse4_1", "sse4_2", "popcnt"} { + if !flags[required] { + return fmt.Errorf("required native IA32 capability missing: %s", required) + } + } + return nil +} + +func x86DefinedStatusIA32RequireCapabilities(t *testing.T) { + t.Helper() + file, err := os.Open("/proc/cpuinfo") + if err != nil { + t.Fatal(err) + } + defer file.Close() + contents, err := io.ReadAll(io.LimitReader(file, (4<<20)+1)) + if err != nil || len(contents) > 4<<20 { + t.Fatal("unbound native IA32 capability observation", err) + } + if err := x86DefinedStatusIA32Capabilities(string(contents)); err != nil { + t.Fatal(err) + } +} + +func TestX86DefinedStatusIA32CapabilitiesAreRequired(t *testing.T) { + if err := x86DefinedStatusIA32Capabilities("flags: fpu cmov popcnt sse4_1 sse4_2\n"); err != nil { + t.Fatal(err) + } + for _, observation := range []string{ + "flags: fpu cmov sse4_1 sse4_2\n", "flags: fpu cmov popcnt sse4_2\n", + "flags: fpu cmov popcnt sse4_1\n", "notflags: popcnt sse4_1 sse4_2\n", + "flags: popcnt_extended sse4_1_extended sse4_2_extended\n", "", + } { + if x86DefinedStatusIA32Capabilities(observation) == nil { + t.Errorf("missing/foreign CPU capability was accepted: %q", observation) + } + } +} + +func TestX86DefinedStatusIA32ReferenceTransportWidth(t *testing.T) { + kernels := x86DefinedStatusIA32Kernels() + if len(kernels) != 7 || kernels[0].bits != 16 || kernels[1].bits != 32 { + t.Fatal("native IA32 kernel denominator changed", kernels) + } + declarations, source := x86DefinedStatusIA32Fixture("386") + if strings.Count(declarations, "*[8]uint32") != 7 || + strings.Count(source, "$0-16") != 7 || strings.Count(source, "MOVB AH,4(DI)") != 7 || + strings.Contains(source, "Q ") || strings.Contains(source, "PUSHFQ") { + t.Fatal("IA32 fixture lost actual four-byte transport or physical instruction width") + } + goSource := x86DefinedStatusIA32GoDriver(t) + if !strings.Contains(goSource, "int64(int32(left))") || !strings.Contains(goSource, "int64(int32(right))") { + t.Fatal("minimum signed IA32 length must be widened before negation") + } + cSource := x86DefinedStatusIA32CDriver() + if strings.Contains(cSource, "= right>>(8*i)") || strings.Contains(cSource, "= left>>(8*i)") || + !strings.Contains(cSource, "((uint64_t)right)>>(8*i)") || !strings.Contains(cSource, "((uint64_t)left)>>(8*i)") { + t.Fatal("independent C vector filling must not shift a uint32 by 32 or more") + } + if !strings.Contains(cSource, "bits == 32 ? UINT32_MAX") || !strings.Contains(goSource, "if bits == 32") { + t.Fatal("full IA32 mask must not shift a uint32 by its width") + } +} + +func TestX86DefinedStatusIA32IndependentCActualCompile(t *testing.T) { + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("required LLVM 22 clang missing") + } + dir := t.TempDir() + input, object := filepath.Join(dir, "model.c"), filepath.Join(dir, "model.o") + // Freestanding syntax/object checking needs no host libc headers or fake + // sysroot. The required native counterpart uses the unchanged stdio driver. + source := strings.Replace(x86DefinedStatusIA32CDriver(), "#include ", + "extern int printf(const char *, ...);", 1) + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output, err := x87IA32Capture(t, clang, []string{"-target", "i386-unknown-linux-gnu", + "-ffreestanding", "-Wall", "-Wextra", "-Werror", level, "-c", input, "-o", object}, dir, os.Environ()) + if err != nil || len(output) != 0 { + t.Fatalf("independent IA32 C model object failed: %v\n%s", err, output) + } + parsed, err := elf.Open(object) + if err != nil { + t.Fatal(err) + } + defer parsed.Close() + if parsed.Class != elf.ELFCLASS32 || parsed.Machine != elf.EM_386 || parsed.Type != elf.ET_REL { + t.Fatal("independent C model did not produce an IA32 object") + } + contents, err := os.ReadFile(object) + if err != nil || len(contents) == 0 { + t.Fatal("missing IA32 C model object", err) + } + t.Logf("independent IA32 C model %s bytes=%d sha256=%x; compile-only, not executed", + level, len(contents), sha256.Sum256(contents)) + }) + } +} + +// These objects cover every affected LLVM target/API independently of the +// required direct Linux/386 executable. No cross-object is a runtime receipt. +func TestX86DefinedStatusIA32ThreeAPIsFiveTargetsObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("required LLVM 22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + declarations, source := x86DefinedStatusIA32Fixture(target.arch) + goAssemblerOracleOutput(t, target.arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, target.arch, target.triple, api) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprint("O2=", optimized), func(t *testing.T) { + input := ir + if optimized { + input = x86UndefinedTrapOptimizedIR(t, input) + } + compileLLVMToObject(t, llc, target.triple, "status.ll", "status.o", input) + }) + } + }) + } + } +} + +func x86DefinedStatusIA32CompileAndRun(t *testing.T, llc, ir, level string) { + t.Helper() + dir := t.TempDir() + llPath, object := filepath.Join(dir, "status.ll"), filepath.Join(dir, "status.o") + mainPath, executable := filepath.Join(dir, "main.c"), filepath.Join(dir, "translated") + for name, contents := range map[string]string{llPath: ir, mainPath: x86DefinedStatusIA32CDriver()} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + output, err := x87IA32Capture(t, llc, + []string{level, "-mtriple=i386-unknown-linux-gnu", "-filetype=obj", llPath, "-o", object}, dir, os.Environ()) + if err != nil || len(output) != 0 { + t.Fatalf("LLVM22 IA32 %s object failed: %v\n%s", level, err, output) + } + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if err != nil { + t.Fatal(err) + } + output, err = x87IA32Capture(t, compiler, + []string{"-static", "-no-pie", level, "-Wall", "-Wextra", "-Werror=shift-count-overflow", + object, mainPath, "-o", executable}, dir, os.Environ()) + if err != nil || len(output) != 0 { + t.Fatalf("static IA32 %s independent C link failed: %v\n%s", level, err, output) + } + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != "LLVM defined-status observations 896 failures 0\n" { + t.Fatalf("LLVM IA32 %s six-flag/LAHF/full canary oracle: %q", level, output) + } +} diff --git a/amd64_defined_status_linux_amd64_test.go b/amd64_defined_status_linux_amd64_test.go new file mode 100644 index 000000000..4459675f6 --- /dev/null +++ b/amd64_defined_status_linux_amd64_test.go @@ -0,0 +1,51 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "os" + "testing" +) + +// Unlike emulated-host observation agreement, these required native roots +// retain the full independent ISA expectations, including PTEST's cleared AF. +// Missing tools, instructions or actual native backing are failures. +func TestCrossLinuxRuntimeMatrixX86DefinedStatusOriginalGoNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86DefinedStatusOriginalGoNumerical(t) +} + +func TestCrossLinuxRuntimeMatrixX86DefinedStatusIndependentOriginalCNumerical(t *testing.T) { + x87IA32RequireNativeHost(t) + x86DefinedStatusIndependentOriginalCNumerical(t) +} + +// The native kernel, original ABI0 executable and both translated ELF32 +// executables are required. No Rosetta/QEMU or cross-object substitutes them. +func TestCrossLinuxRuntimeMatrixX86DefinedStatusNative386(t *testing.T) { + x87IA32RequireNativeHost(t) + x86DefinedStatusIA32RequireCapabilities(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("required LLVM 22 llc missing") + } + t.Run("OriginalGo", func(t *testing.T) { + original := x86DefinedStatusIA32BuildOriginal(t) + output := x87IA32RunDirect(t, original, nil, os.Environ()) + if string(output) != "original defined-status observations 896 failures 0\n" { + t.Fatalf("original Go IA32 ABI0 six-flag/LAHF/full canary oracle: %q", output) + } + }) + declarations, source := x86DefinedStatusIA32Fixture("386") + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "386", "i386-unknown-linux-gnu", "Go") + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + x86DefinedStatusIA32CompileAndRun(t, llc, input, level) + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_defined_status_numerical_test.go b/amd64_defined_status_numerical_test.go new file mode 100644 index 000000000..5f31cd839 --- /dev/null +++ b/amd64_defined_status_numerical_test.go @@ -0,0 +1,541 @@ +//go:build unix + +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +type x86DefinedStatusKernel struct { + name, instruction string + kind, bits int +} + +func x86DefinedStatusKernels() []x86DefinedStatusKernel { + var kernels []x86DefinedStatusKernel + add := func(instruction string, kind, bits int) { + kernels = append(kernels, x86DefinedStatusKernel{ + fmt.Sprintf("status%d", len(kernels)), instruction, kind, bits, + }) + } + for _, width := range []struct { + suffix string + bits int + }{{"W", 16}, {"L", 32}, {"Q", 64}} { + add("POPCNT"+width.suffix+" AX,BX", 0, width.bits) + } + add("PTEST X0,X1", 1, 128) + for _, op := range []string{"PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM"} { + kind := 2 + if strings.HasPrefix(op, "PCMPI") { + kind = 3 + } + add(op+" $8,X0,X1", kind, 128) + } + for _, op := range []string{"VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM"} { + kind := 2 + if strings.HasPrefix(op, "VPCMPI") { + kind = 3 + } + add(op+" $8,X0,X1", kind, 128) + } + for _, stem := range []string{"KTEST", "KORTEST"} { + for _, width := range []struct { + suffix string + bits int + }{{"B", 8}, {"W", 16}, {"D", 32}, {"Q", 64}} { + kind := 4 + if stem == "KORTEST" { + kind = 5 + } + add(stem+width.suffix+" K1,K2", kind, width.bits) + } + } + return kernels +} + +func x86DefinedStatusNumericalFixture() (declarations, source string) { + var decl, assembly strings.Builder + for _, kernel := range x86DefinedStatusKernels() { + fmt.Fprintf(&decl, "func %s(out *[8]uint64, left, right, seed uint64)\n", kernel.name) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-32 +MOVQ out+0(FP),DI +MOVQ left+8(FP),AX +MOVQ right+16(FP),BX +MOVQ seed+24(FP),SI +MOVQ BX,DX +MOVQ AX,X0 +MOVQ BX,X1 +`, kernel.name) + if kernel.kind >= 4 { + assembly.WriteString("KMOVQ AX,K1\nKMOVQ BX,K2\n") + } + fmt.Fprintf(&assembly, ` +PUSHQ SI +POPFQ +%s +PUSHFQ +POPQ SI +MOVQ SI,0(DI) +LAHF +MOVB AH,8(DI) +RET +`, kernel.instruction) + } + return decl.String(), assembly.String() +} + +// The baseline original-Go root executes SSE4.2/POPCNT, not AVX512 masks. +// Its exact source includes the mask objects, but no mask body is called here. +func x86DefinedStatusOriginalGoNumerical(t *testing.T) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + declarations, source := x86DefinedStatusNumericalFixture() + driver := x86DefinedStatusOriginalGoDriver(false) + original := x86DefinedStatusBuildOriginal(t, execution, declarations, source, driver) + output := execution.run(t, original) + if string(output) != "original defined-status observations 1024 failures 0\n" { + t.Fatalf("original Go six-bit status/LAHF/canary model: %s", output) + } +} + +func x86DefinedStatusOriginalGoDriver(observe bool) string { + kernels := x86DefinedStatusKernels()[:8] + var names, kinds, widths strings.Builder + for _, kernel := range kernels { + fmt.Fprintf(&names, "%s,\n", kernel.name) + fmt.Fprintf(&kinds, "%d,", kernel.kind) + fmt.Fprintf(&widths, "%d,", kernel.bits) + } + return strings.NewReplacer( + "KERNEL_NAMES", names.String(), "KERNEL_KINDS", kinds.String(), + "KERNEL_WIDTHS", widths.String(), "ACTUAL_OBSERVATIONS", fmt.Sprint(observe), + ).Replace(x86DefinedStatusGoDriver) +} + +func x86DefinedStatusBuildOriginal(t *testing.T, execution x86FloatCompareExecution, declarations, assembly, driver string) string { + t.Helper() + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module definedstatus\n\ngo 1.20\n", + "main.go": driver, + "decl.go": "package main\n" + declarations, + "status_amd64.s": assembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "build", "-p=1", "-o", executable, ".") + command.Dir = dir + goos := runtime.GOOS + if strings.Contains(execution.triple, "linux") { + goos = "linux" + } + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+goos, "GOARCH=amd64", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + return executable +} + +func TestX86DefinedStatusLLVMNumerical(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, source := x86DefinedStatusNumericalFixture() + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "amd64", execution.triple, "Go") + var prototypes, kernels strings.Builder + for _, kernel := range x86DefinedStatusKernels() { + fmt.Fprintf(&prototypes, "extern void %s(uint64_t *, uint64_t, uint64_t, uint64_t);\n", kernel.name) + fmt.Fprintf(&kernels, "{%s,%d,%d,\"%s\"},\n", kernel.name, kernel.kind, kernel.bits, kernel.instruction) + } + driver := strings.NewReplacer("KERNEL_PROTOTYPES", prototypes.String(), + "KERNELS", kernels.String()).Replace(x86DefinedStatusCDriver) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "defined-status", input, driver, level) + want := fmt.Sprintf("LLVM defined-status observations %d failures 0\n", len(x86DefinedStatusKernels())*128) + if string(output) != want { + t.Fatalf("independent six-bit status/LAHF/canary model: %s", output) + } + }) + } +} + +// The C producer executes entire physical opcodes and captures flags in the +// same inline block, independently of Go's ABI0 wrapper and LLVM lowering. +func x86DefinedStatusOriginalCSource() string { + instructions := []string{ + "popcntw %%ax,%%bx", "popcntl %%eax,%%ebx", "popcntq %%rax,%%rbx", + "ptest %%xmm0,%%xmm1", + "pcmpestri $8,%%xmm0,%%xmm1", "pcmpestrm $8,%%xmm0,%%xmm1", + "pcmpistri $8,%%xmm0,%%xmm1", "pcmpistrm $8,%%xmm0,%%xmm1", + } + var functions, registrations strings.Builder + for index, kernel := range x86DefinedStatusKernels()[:8] { + fmt.Fprintf(&functions, `static void %s(uint64_t *out, uint64_t left, uint64_t right, uint64_t seed) { + uint64_t length = right; + __asm__ volatile( + "movq %%%%rax,%%%%xmm0\n\t" + "movq %%%%rbx,%%%%xmm1\n\t" + "pushq %%%%rsi\n\t" + "popfq\n\t" + "%s\n\t" + "pushfq\n\t" + "popq %%%%rsi\n\t" + "movq %%%%rsi,0(%%%%rdi)\n\t" + "lahf\n\t" + "movb %%%%ah,8(%%%%rdi)\n\t" + : "+a"(left), "+b"(right), "+d"(length), "+S"(seed) + : "D"(out) + : "cc", "memory", "xmm0", "xmm1", "rcx"); +} + +`, kernel.name, instructions[index]) + fmt.Fprintf(®istrations, "{%s,%d,%d,%q},\n", kernel.name, kernel.kind, kernel.bits, kernel.instruction) + } + source := strings.NewReplacer("KERNEL_PROTOTYPES", functions.String(), + "KERNELS", registrations.String()).Replace(x86DefinedStatusCDriver) + return strings.ReplaceAll(source, "LLVM", "original-C") +} + +func x86DefinedStatusIndependentOriginalCNumerical(t *testing.T) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source := filepath.Join(dir, "original.c") + if err := os.WriteFile(source, []byte(x86DefinedStatusOriginalCSource()), 0600); err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := filepath.Join(dir, "original-c"+level) + arguments := append([]string(nil), execution.compiler...) + arguments = append(arguments, level, source, "-o", executable) + x86FloatCompareCaptureCommand(t, exec.Command(arguments[0], arguments[1:]...)) + output := execution.run(t, executable) + if string(output) != "original-C defined-status observations 1024 failures 0\n" { + t.Fatalf("actual C opcodes versus independent six-bit status model: %s", output) + } + }) + } +} + +// Common emulated-host evidence is exact original-Go/C agreement, not ISA +// conformance. Every original result is retained; native Linux separately +// runs the unchanged strict mathematical oracle. No deviation is a skip. +func TestX86DefinedStatusOriginalGoAndCObservationAgreement(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly := x86DefinedStatusNumericalFixture() + original := x86DefinedStatusBuildOriginal(t, execution, declarations, assembly, x86DefinedStatusOriginalGoDriver(true)) + observations := execution.run(t, original) + lines := strings.Split(strings.TrimSuffix(string(observations), "\n"), "\n") + summary := regexp.MustCompile(`^observed-status observations 1024 model-deviations ([0-9]+) invalid-observations 0$`) + if len(lines) != 1025 || !summary.MatchString(lines[len(lines)-1]) { + t.Fatalf("incomplete or invalid actual original-Go observation transcript: %s", observations) + } + dir := t.TempDir() + source := filepath.Join(dir, "original.c") + if err := os.WriteFile(source, []byte("#define OBSERVE_ACTUAL 1\n"+x86DefinedStatusOriginalCSource()), 0600); err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + executable := filepath.Join(dir, "original-c"+level) + arguments := append([]string(nil), execution.compiler...) + arguments = append(arguments, level, source, "-o", executable) + x86FloatCompareCaptureCommand(t, exec.Command(arguments[0], arguments[1:]...)) + actual := execution.run(t, executable) + if string(actual) != string(observations) { + t.Fatalf("original Go/C full status/LAHF/canary transcript differs: Go=%x C=%x\n%s\n%s", + sha256.Sum256(observations), sha256.Sum256(actual), observations, actual) + } + t.Logf("original observation agreement sha=%x; %s; not native ISA conformance", + sha256.Sum256(actual), lines[len(lines)-1]) + }) + } +} + +const x86DefinedStatusGoDriver = `package main + +import ( + "fmt" + "os" +) + +func mask(bits uint) uint64 { + if bits == 64 { + return ^uint64(0) + } + return uint64(1)<> (8*uint(i))) + b[i] = byte(left >> (8*uint(i))) + } + la, lb := 16, 16 + if kind == 2 { + la, lb = int(int32(left)), int(int32(right)) + if la < 0 { + la = -la + } + if lb < 0 { + lb = -lb + } + if la > 16 { + la = 16 + } + if lb > 16 { + lb = 16 + } + } else { + la, lb = 0, 0 + for la < 16 && a[la] != 0 { + la++ + } + for lb < 16 && b[lb] != 0 { + lb++ + } + } + var matches uint64 + for i := 0; i < 16; i++ { + if i >= la && i >= lb || i < la && i < lb && a[i] == b[i] { + matches |= 1 << uint(i) + } + } + var result uint64 + if matches != 0 { + result |= 1 + } + if lb < 16 { + result |= 64 + } + if la < 16 { + result |= 128 + } + if matches&1 != 0 { + result |= 2048 + } + return result +} + +func main() { + const observeActual = ACTUAL_OBSERVATIONS + kernels := []func(*[8]uint64,uint64,uint64,uint64){KERNEL_NAMES} + kinds := []int{KERNEL_KINDS} + widths := []uint{KERNEL_WIDTHS} + values := []uint64{0,1,0x55,0xaa,0xff,0xffff,0x80000000,^uint64(0)} + const canary = uint64(0xa7a7a7a7a7a7a7a7) + observations, failures, invalid := 0, 0, 0 + for index, kernel := range kernels { + kernelFailures := 0 + for _, left := range values { + for _, right := range values { + for _, seed := range []uint64{0x8d7,0x12} { + var out [8]uint64 + for i := range out { + out[i] = canary + } + kernel(&out,left,right,seed) + want := flags(kinds[index],widths[index],left,right) + invalidObservation := byte(out[1]) != byte(out[0]&0xd5|2) || + out[1]>>8 != canary>>8 + for i := 2; i < len(out); i++ { + invalidObservation = invalidObservation || out[i] != canary + } + if invalidObservation { + invalid++ + } + if observeActual { + fmt.Printf("observed-status %d %016x", index, out[0]&0x8d5) + for i := 1; i < len(out); i++ { + fmt.Printf(" %016x", out[i]) + } + fmt.Println() + } + bad := out[0]&0x8d5 != want || invalidObservation + if bad { + if !observeActual && failures < 12 { + fmt.Printf("original status kernel=%d left=%x right=%x flags=%x want=%x\n",index,left,right,out[0]&0x8d5,want) + } + failures++ + kernelFailures++ + } + observations++ + } + } + } + if !observeActual && kernelFailures != 0 { + fmt.Printf("original status kernel=%d failures=%d\n",index,kernelFailures) + } + } + if observeActual { + fmt.Printf("observed-status observations %d model-deviations %d invalid-observations %d\n",observations,failures,invalid) + if invalid != 0 { + os.Exit(1) + } + return + } + fmt.Printf("original defined-status observations %d failures %d\n",observations,failures) + if failures != 0 { + os.Exit(1) + } +} +` + +const x86DefinedStatusCDriver = ` +#include +#include +#ifndef OBSERVE_ACTUAL +#define OBSERVE_ACTUAL 0 +#endif +KERNEL_PROTOTYPES + +typedef struct { + void (*run)(uint64_t *, uint64_t, uint64_t, uint64_t); + unsigned kind, bits; + const char *name; +} Kernel; + +static uint64_t mask(unsigned bits) { + return bits == 64 ? UINT64_MAX : (UINT64_C(1)<= 4) { + uint64_t m = mask(bits); + left &= m; + right &= m; + if (kind == 4) { + return (left&right ? 0 : 64) | (left&~right ? 0 : 1); + } + uint64_t combined = left|right; + return (combined ? 0 : 64) | (combined == m ? 1 : 0); + } + unsigned char a[16] = {0}, b[16] = {0}; + for (unsigned i = 0; i < 8; i++) { + a[i] = right>>(8*i); + b[i] = left>>(8*i); + } + unsigned la = 16, lb = 16; + if (kind == 2) { + int64_t na = (int32_t)left, nb = (int32_t)right; + if (na < 0) { + na = -na; + } + if (nb < 0) { + nb = -nb; + } + la = na > 16 ? 16 : (unsigned)na; + lb = nb > 16 ? 16 : (unsigned)nb; + } else { + la = lb = 0; + while (la < 16 && a[la]) { + la++; + } + while (lb < 16 && b[lb]) { + lb++; + } + } + unsigned matches = 0; + for (unsigned i = 0; i < 16; i++) { + if ((i >= la && i >= lb) || (i < la && i < lb && a[i] == b[i])) { + matches |= 1u<>8) != (canary>>8); + for (unsigned i = 2; i < 8; i++) { + invalid_observation |= out[i] != canary; + } + invalid += !!invalid_observation; + if (OBSERVE_ACTUAL) { + printf("observed-status %u %016llx",index,(unsigned long long)(out[0]&0x8d5)); + for (unsigned i = 1; i < 8; i++) { + printf(" %016llx",(unsigned long long)out[i]); + } + printf("\n"); + } + int bad = (out[0]&0x8d5) != want || invalid_observation; + if (bad) { + if (!OBSERVE_ACTUAL && failures < 12) { + printf("LLVM status kernel=%u left=%llx right=%llx flags=%llx want=%llx\n", + index,(unsigned long long)values[l],(unsigned long long)values[r], + (unsigned long long)(out[0]&0x8d5),(unsigned long long)want); + } + failures++; + kernel_failures++; + } + observations++; + } + } + } + if (!OBSERVE_ACTUAL && kernel_failures) { + printf("LLVM status family=%s failures=%u\n",kernels[index].name,kernel_failures); + } + } + if (OBSERVE_ACTUAL) { + printf("observed-status observations %u model-deviations %u invalid-observations %u\n",observations,failures,invalid); + return invalid ? 1 : 0; + } + printf("LLVM defined-status observations %u failures %u\n",observations,failures); + return failures ? 1 : 0; +} +` diff --git a/amd64_defined_zero_af_test.go b/amd64_defined_zero_af_test.go new file mode 100644 index 000000000..3124fa4bc --- /dev/null +++ b/amd64_defined_zero_af_test.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86DefinedZeroAFCase struct { + op, instruction string + amd64Only bool +} + +// These independent ISA witnesses cover every current non-arithmetic writer +// whose AF result is defined to be zero. Go encoding tables establish syntax; +// they do not describe the status effects. ENQCMD/S currently use raw source. +func x86DefinedZeroAFWriters() []x86DefinedZeroAFCase { + cases := []x86DefinedZeroAFCase{ + {"UMWAIT", "UMWAIT BX", false}, + {"TPAUSE", "TPAUSE BX", false}, + {"PTEST", "PTEST X0,X1", false}, + {"VPTEST", "VPTEST X0,X1", false}, + {"VTESTPD", "VTESTPD X0,X1", false}, + {"VTESTPS", "VTESTPS X0,X1", false}, + {"XTEST", "XTEST", false}, + {"ENQCMD", "BYTE $0xf2\nBYTE $0x0f\nBYTE $0x38\nBYTE $0xf8\nBYTE $0x38", false}, + {"ENQCMDS", "BYTE $0xf3\nBYTE $0x0f\nBYTE $0x38\nBYTE $0xf8\nBYTE $0x38", false}, + } + for _, stem := range []string{"RDRAND", "RDSEED", "POPCNT"} { + for _, width := range []string{"W", "L", "Q"} { + op := stem + width + operand := "BX" + if stem == "POPCNT" { + operand = "AX,BX" + } + cases = append(cases, x86DefinedZeroAFCase{op, op + " " + operand, width == "Q"}) + } + } + for _, stem := range []string{"KTEST", "KORTEST"} { + for _, width := range []string{"B", "W", "D", "Q"} { + op := stem + width + cases = append(cases, x86DefinedZeroAFCase{op, op + " K1,K2", false}) + } + } + for _, stem := range []string{"PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM"} { + for _, prefix := range []string{"", "V"} { + op := prefix + stem + cases = append(cases, x86DefinedZeroAFCase{op, op + " $0,X0,X1", false}) + } + } + return cases +} + +func x86DefinedZeroAFSource(arch, name, instruction string) string { + word := "Q" + if arch == "386" { + word = "L" + } + return fmt.Sprintf("TEXT %s(SB),4,$0-0\nMOVL $0,AX\nMOVL $0,DX\nMOVL $0,BX\n"+ + "PUSH%s $0x8d7\nPOPF%s\n%s\nPUSHF%s\nPOP%s CX\nRET\n", + name, word, word, instruction, word, word) +} + +func TestX86DefinedZeroAFWritersClearAfterOperation(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86DefinedZeroAFWriters() { + if arch == "386" && row.amd64Only { + continue // Go's explicit 64-bit operand row is not a 386 form. + } + t.Run(arch+"/"+row.op, func(t *testing.T) { + source := x86DefinedZeroAFSource(arch, "status", row.instruction) + requireX86GoAssemblerResult(t, arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"status": {Name: "status", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + // One false store initializes the slot before the explicit AF=1 + // seed. A defined-zero writer needs its own subsequent store. + if strings.Count(ir, "store i1 false, ptr %flags_af") != 2 { + t.Fatalf("%s retained seeded AF instead of its defined-zero status result", row.op) + } + }) + } + } +} + +func TestX86DefinedZeroAFExistingCompareWriterControls(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, op := range []string{"COMISS", "COMISD", "UCOMISS", "UCOMISD", "VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + t.Run(arch+"/"+op, func(t *testing.T) { + source := x86DefinedZeroAFSource(arch, "status", op+" X0,X1") + requireX86GoAssemblerResult(t, arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"status": {Name: "status", Ret: Void}}, + }) + if err != nil || strings.Count(ir, "store i1 false, ptr %flags_af") != 2 { + t.Fatal("already correct compare writer lost AF clearing", err) + } + }) + } + } +} diff --git a/amd64_directive_effects.go b/amd64_directive_effects.go new file mode 100644 index 000000000..214d1f463 --- /dev/null +++ b/amd64_directive_effects.go @@ -0,0 +1,97 @@ +package plan9asm + +import ( + "fmt" + "math" + "strings" +) + +type x86DirectiveEffect uint8 + +const ( + x86DirectiveLayout x86DirectiveEffect = iota + 1 + x86DirectiveMetadata +) + +type x86DirectiveForm struct { + effect x86DirectiveEffect + alignment int64 +} + +// These are the existing metadata directives and the complete PCALIGN family, +// not a blanket pseudo-op exemption. They preserve numerical availability but +// still have native layout/GC effects. This grammar grants no caller, raw-PC, +// stack, metadata ownership or physical function-alignment permission. +func x86DirectiveProperties(ins Instr) (x86DirectiveForm, bool, error) { + var form x86DirectiveForm + base, _, _ := strings.Cut(string(ins.Op), ".") + count := 2 + switch base { + case "PCALIGN": + form.effect, count = x86DirectiveLayout, 1 + case "PCALIGNMAX": + form.effect = x86DirectiveLayout + case "PCDATA", "FUNCDATA": + form.effect = x86DirectiveMetadata + default: + return form, false, nil + } + fail := func() (x86DirectiveForm, bool, error) { + return form, true, fmt.Errorf("invalid x86 %s directive operands: %q", base, ins.Raw) + } + if string(ins.Op) != base || len(ins.Args) != count { + return fail() + } + constant := func(arg Operand) bool { + return arg.Kind == OpImm && arg.ImmRaw == "" && !arg.ImmIsFloat + } + if !constant(ins.Args[0]) { + return fail() + } + if form.effect == x86DirectiveLayout { + // obj.AlignmentPaddingLength: both spellings retain the function's + // required alignment, even when PCALIGNMAX emits no padding here. + form.alignment = ins.Args[0].Imm + if form.alignment < 8 || form.alignment > 2048 || form.alignment&(form.alignment-1) != 0 { + return fail() + } + if count == 2 && (!constant(ins.Args[1]) || ins.Args[1].Imm < 0 || ins.Args[1].Imm >= form.alignment) { + return fail() + } + return form, true, nil + } + if base == "PCDATA" { + // The backend's ypcdata row is Yi32,Yi32 and linkpcln retains an + // int32 value. Frontend constant syntax alone is insufficient. + if ins.Args[0].Imm < 0 || ins.Args[0].Imm > math.MaxInt32 || !constant(ins.Args[1]) || + ins.Args[1].Imm < math.MinInt32 || ins.Args[1].Imm > math.MaxInt32 { + return fail() + } + return form, true, nil + } + // cmd/asm.validSymbol alone is not the full contract: x86's yfuncdata + // row requires memory and linkpcln requires its offset to be zero. An + // accepted frontend offset otherwise panics during object production. + arg := ins.Args[1] + symbol := strings.TrimSpace(arg.Sym) + if ins.Args[0].Imm < 0 || ins.Args[0].Imm > math.MaxInt32 || + arg.Kind != OpSym || !strings.HasSuffix(symbol, "(SB)") { + return fail() + } + name, offset := splitSymPlusOff(strings.TrimSpace(strings.TrimSuffix(symbol, "(SB)"))) + if offset != 0 || !validPPIdentifier(strings.TrimSuffix(name, "<>")) { + return fail() + } + return form, true, nil +} + +func validateX86DirectiveFile(file *File) error { + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if _, known, err := x86DirectiveProperties(ins); known && err != nil { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + } + } + return nil +} diff --git a/amd64_directive_effects_go127_test.go b/amd64_directive_effects_go127_test.go new file mode 100644 index 000000000..1a7e1e5c1 --- /dev/null +++ b/amd64_directive_effects_go127_test.go @@ -0,0 +1,41 @@ +//go:build go1.27 + +package plan9asm + +import ( + "fmt" + "testing" +) + +// PCALIGNMAX was added after the oldest supported SDK. Current Go checks its +// real encoding; untagged tests still check every consumer on historical Go. +func TestX86DirectiveCurrentGoCompleteFormats(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86DirectiveValidForms() { + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT value(SB),4,$0-0\n"+row+"\nRET\n", "", true) + }) + } + for _, row := range []string{ + "PCALIGN", "PCALIGN $16,$1", "PCALIGN AX", "PCALIGN $16.0", + "PCDATA $0", "PCDATA AX,$1", "PCDATA $0,$1.0", + "FUNCDATA $0", "FUNCDATA AX,metadata(SB)", "FUNCDATA $0,$1", + "FUNCDATA $0,metadata(SB)(AX)", "FUNCDATA $0,metadata(SB)(AX*1)", + } { + t.Run(arch+"/invalid/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT value(SB),4,$0-0\n"+row+"\nRET\n", "", false) + }) + } + } +} + +func TestX86DirectiveCurrentGoFiveTargetObjects(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, row := range []string{"PCALIGN $16", "PCALIGNMAX $16,$7", "PCDATA $0,$-1", "FUNCDATA $0,metadata(SB)"} { + t.Run(target.triple+"/"+row, func(t *testing.T) { + source := fmt.Sprintf("TEXT value(SB),4,$0-4\nMOVL $1,AX\n%s\nMOVL AX,ret+0(FP)\nRET\n", row) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + }) + } + } +} diff --git a/amd64_directive_effects_test.go b/amd64_directive_effects_test.go new file mode 100644 index 000000000..f474fa92b --- /dev/null +++ b/amd64_directive_effects_test.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86DirectiveResultInitialization(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, directive := range []string{ + "PCALIGN $16", + "PCALIGNMAX $16,$7", + "PCDATA $0,$-1", + "FUNCDATA $0,metadata(SB)", + } { + t.Run(arch+"/"+directive, func(t *testing.T) { + body := fmt.Sprintf("MOVL $1,AX\n%s\nMOVL AX,ret0+0(FP)\nRET", directive) + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + for name, check := range map[string]func(Func, FuncSig, string) error{ + "public": CheckX86ABI0ResultInitialization, + "private": checkX86ScalarFPResultInitialization, + } { + if err := check(file.Funcs[0], sig, arch); err != nil { + t.Errorf("%s: valid directive changed numerical availability: %v", name, err) + } + } + }) + } + } +} + +func x86DirectiveValidForms() []string { + forms := []string{ + "PCDATA $0,$-1", "PCDATA $1,$2", + "FUNCDATA $0,metadata(SB)", "FUNCDATA $1,metadata<>+0(SB)", + "FUNCDATA $2,metadata+(4-4)(SB)", + } + for _, alignment := range []int{8, 16, 32, 64, 128, 256, 512, 1024, 2048} { + forms = append(forms, fmt.Sprintf("PCALIGN $%d", alignment)) + for _, limit := range []int{0, alignment - 1} { + forms = append(forms, fmt.Sprintf("PCALIGNMAX $%d,$%d", alignment, limit)) + } + } + return forms +} + +func TestX86DirectiveNumericalStateUnchanged(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86DirectiveValidForms() { + t.Run(arch+"/"+row, func(t *testing.T) { + ins := x86PureGPUnknownInstruction(t, row) + for _, transfer := range []func(Instr, *x86ResultState, []bool, int64, string) bool{ + x86ResultTransfer, x86ScalarFPResultTransfer, + } { + for _, shape := range []bool{false, true} { + state := x86ResultState{fp: []bool{true, false}, auto: []bool{false, true}, + pointerFP: map[int64]bool{0: true}, shapeOnly: shape, flags: x86ByteFlagCF} + for index := range state.gp { + state.gp[index] = uint8(index * 7) + state.pointerGP[index] = index%2 == 0 + } + for index := range state.xmm { + state.xmm[index] = uint16(index * 31) + state.vectorUpper[index] = uint64(index) << 40 + } + state.masks[3] = true + before := state.clone() + if !transfer(ins, &state, nil, 0, arch) || !reflect.DeepEqual(state, before) { + t.Fatal("directive changed definedness or pointer/flag identity") + } + } + } + }) + } + } +} + +func TestX86DirectiveMalformedAndUnknownEffects(t *testing.T) { + rows := []string{ + "PCALIGN", "PCALIGN $16,$1", "PCALIGN $0", "PCALIGN $4", "PCALIGN $24", "PCALIGN $4096", + "PCALIGN $-16", "PCALIGN $16.0", "PCALIGN $(missing)", "PCALIGN AX", "PCALIGN 16(AX)", "PCALIGN.Z $16", + "PCALIGNMAX", "PCALIGNMAX $16", "PCALIGNMAX $16,$0,$1", "PCALIGNMAX $16,$16", "PCALIGNMAX $16,$-1", + "PCALIGNMAX $24,$1", "PCALIGNMAX $16,$1.0", "PCALIGNMAX $16,AX", "PCALIGNMAX AX,$1", "PCALIGNMAX $16,$missing", + "PCDATA $0", "PCDATA $0,$1,$2", "PCDATA AX,$1", "PCDATA $0,$1.0", "PCDATA $0,$missing", "PCDATA.Z $0,$1", + "FUNCDATA $0", "FUNCDATA $0,$1", "FUNCDATA AX,metadata(SB)", "FUNCDATA $0.0,metadata(SB)", + "FUNCDATA $0,metadata", "FUNCDATA $0,metadata(FP)", "FUNCDATA $0,metadata(SB)(AX)", + "FUNCDATA $0,metadata(SB)(AX*1)", "FUNCDATA $0,metadata+missing(SB)", "FUNCDATA.Z $0,metadata(SB)", + "FUNCDATA $0,$metadata(SB)", "FUNCDATA $0,metadata+4(SB)", "FUNCDATA $-1,metadata(SB)", + "PCDATA $-1,$0", "PCDATA $0,$2147483648", "PCDATA $0,$-2147483649", + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + ins := x86PureGPUnknownInstruction(t, row) + if _, known, err := x86DirectiveProperties(ins); !known || err == nil { + t.Fatal("invalid directive accepted", known, err) + } + for _, target := range x86RawReturnTargets { + for _, dead := range []bool{false, true} { + body := row + "\nMOVL $1,ret+0(FP)\nRET" + if dead { + body = "MOVL $1,ret+0(FP)\nRET\n" + row + } + source := "TEXT ·view(SB),4,$0-4\n" + body + "\n" + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32", target.arch, target.triple, api); err == nil { + t.Fatal("malformed source bypassed full source grammar", target.triple, dead, api) + } + } + } + } + }) + } + for _, row := range []string{"BYTE $0x90", "WORD $0x9090", "LONG $0", "QUAD $0", "NO_LOCAL_POINTERS", "UNDEF", "NOT_A_DIRECTIVE"} { + if _, known, _ := x86DirectiveProperties(x86PureGPUnknownInstruction(t, row)); known { + t.Fatal("unregistered pseudo/native operation acquired numerical noop", row) + } + } +} + +func TestX86DirectiveDoesNotInitializeOrAuthorizeEffects(t *testing.T) { + for _, row := range []string{"PCALIGN $16", "PCALIGNMAX $16,$7", "PCDATA $0,$-1", "FUNCDATA $0,metadata(SB)"} { + for name, body := range map[string]string{ + "missing result": row + "\nRET", + "unknown GP": row + "\nMOVL AX,ret0+0(FP)\nRET", + "unknown vector": row + "\nMOVL X0,ret0+0(FP)\nRET", + "unknown flags": row + "\nJE missing\nMOVL $1,ret0+0(FP)\nmissing:\nRET", + "native pointer": "MOVL $1,AX\n" + row + "\nMOVL (AX),BX\nMOVL BX,ret0+0(FP)\nRET", + "raw bytes": row + "\nBYTE $0x90\nMOVL $1,ret0+0(FP)\nRET", + "unknown dead": "MOVL $1,ret0+0(FP)\nRET\n" + row + "\nNOT_A_DIRECTIVE\nRET", + } { + t.Run(row+"/"+name, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value() uint32", body) + for _, check := range []func(Func, FuncSig, string) error{ + CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization, + } { + if err := check(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("directive supplied missing numerical/native authority", err) + } + } + } + }) + } + } +} + +func TestX86DirectiveThreeAPIsFiveLLVMTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + body := strings.Join(x86DirectiveValidForms(), "\n") + source := "TEXT ·view(SB),4,$0-4\nMOVL $305419896,AX\n" + body + "\nMOVL AX,X0\nMOVL X0,ret+0(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32", target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, api) + } + }) + } +} + +func TestX86DirectiveCommonGoFormats(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + // Old x86 backends predate alignment directives. The shared grammar + // and LLVM tests above still run there, while these original-Go forms + // are genuinely common to every supported compiler. + for _, row := range []string{"PCDATA $0,$-1", "FUNCDATA $0,metadata<>+0(SB)"} { + goAssemblerOracleOutput(t, arch, "TEXT value(SB),4,$0-0\n"+row+"\nRET\n", "", true) + } + } +} + +func TestX86DirectiveLayoutIsNotNumericalAuthority(t *testing.T) { + for _, row := range []string{"PCALIGN $32", "PCALIGNMAX $32,$0", "PCALIGNMAX $32,$31"} { + t.Run(row, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT refer(SB),4,$0-0\nMOVL data(SB),AX\nRET\nTEXT data(SB),4,$0-0\n" + row + "\nLONG $0xdeadbeef\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for pass := 0; pass < 2; pass++ { + file, err = normalizeX86RawFile(file, arch) + if err != nil { + t.Fatal(err) + } + if file.Funcs[1].X86RawAlign != 32 || string(file.Funcs[1].X86RawText) != string([]byte{0xef, 0xbe, 0xad, 0xde}) { + t.Fatal("layout-only directive lost exact raw TEXT alignment/bytes") + } + } + framed, err := Parse(ArchAMD64, strings.Replace(source, "TEXT data(SB),4,$0-0", "TEXT data(SB),4,$16-0", 1)) + if err != nil { + t.Fatal(err) + } + if _, err := normalizeX86RawFile(framed, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("layout-only directive authorized a native source frame", err) + } + // A native byte jump cannot infer its displacement across an + // alignment boundary from ordinary numerical no-op semantics. + crossing, err := Parse(ArchAMD64, "TEXT value(SB),4,$0-0\nBYTE $0xeb\nBYTE $2\n"+row+"\nBYTE $0x90\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := normalizeX86RawFile(crossing, arch); err == nil { + t.Fatal("unknown native byte jump crossed layout boundary") + } + } + }) + } +} diff --git a/amd64_dword_product_displacement_test.go b/amd64_dword_product_displacement_test.go new file mode 100644 index 000000000..b4d3ceded --- /dev/null +++ b/amd64_dword_product_displacement_test.go @@ -0,0 +1,305 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86DwordProductDisplacementOracle struct { + row string + width int + tuple int + units int + mask bool + high bool + mem MemRef + code []byte +} + +// These tuples and offsets are independently prescribed from the actual +// Go encoder's Full/N16,N32,N64 and BcstN8 rows, not production properties. +func x86DwordProductDisplacementOracles(t *testing.T) []x86DwordProductDisplacementOracle { + t.Helper() + var result []x86DwordProductDisplacementOracle + addresses := []MemRef{ + {Base: AX}, {Base: BP}, {Base: SP, Index: CX, Scale: 2}, + {Base: AX, Index: CX, Scale: 4}, {Base: "R13", Index: "R12", Scale: 8}, + } + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + for masking := 0; masking < 3; masking++ { + for _, units := range []int{-129, -128, -1, 1, 127, 128} { + for _, memory := range addresses { + tuple, op := width, name + if broadcast { + tuple, op = 8, name+".BCST" + } + if masking == 2 { + op += ".Z" + } + memory.Off = int64(units * tuple) + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + second, destination := bank+"1", bank+"2" + high := masking == 0 && !broadcast && width != 64 + if high { + second, destination = bank+"17", bank+"18" // Force actual EVEX. + } + args := (Operand{Kind: OpMem, Mem: memory}).String() + "," + second + if masking == 1 { + args += ",K1" + } else if masking == 2 { + args += ",K7" + } + row := op + " " + args + "," + destination + code := x86OrderingHintGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) < 8 || code[0] != 0x62 || code[2]&0x80 == 0 || code[len(code)-1] != 0xc3 { + t.Fatalf("independent original EVEX.W1 object changed: %s %x", row, code) + } + mod, end := code[5]>>6, 6 + if code[5]&7 == 4 { + end++ // Independently observe SIB, not decoded source. + } + if units >= -128 && units <= 127 { + if mod != 1 || int(int8(code[end])) != units || len(code) != end+2 { + t.Fatalf("original compressed disp8/%d changed: %s %x", tuple, row, code) + } + } else if mod != 2 || len(code) != end+5 { + t.Fatalf("original unscaled disp32 boundary changed: %s %x", row, code) + } + result = append(result, x86DwordProductDisplacementOracle{row, width, tuple, units, masking != 0, high, memory, code[:len(code)-1]}) + } + } + } + } + } + } + return result +} + +func TestX86DwordProductCompressedDisp8IndependentPhysicalSource(t *testing.T) { + fixtures := x86DwordProductDisplacementOracles(t) + for _, fixture := range fixtures { + for _, mode := range []int{32, 64} { + for _, prefix := range []struct { + byte byte + segment Reg + }{{0, ""}, {0x64, FS}, {0x65, GS}} { + t.Run(fmt.Sprintf("%s/mode%d/prefix%x", fixture.row, mode, prefix.byte), func(t *testing.T) { + code := append([]byte(nil), fixture.code...) + if prefix.byte != 0 { + code = append([]byte{prefix.byte}, code...) + } + ins, length, known, err := decodedX86DwordToQwordMultiplyInstruction(code, mode) + if mode == 32 && (fixture.high || fixture.mem.Base == "R13") { + if !known || err == nil { + t.Fatal("extended physical mode32 bank/address acquired permission", ins, err) + } + return + } + want := fixture.mem + want.Segment = prefix.segment + if !known || err != nil || length != len(code) || !ins.x86Encoded || len(ins.Args) == 0 || ins.Args[0].Kind != OpMem || !reflect.DeepEqual(ins.Args[0].Mem, want) { + t.Fatalf("native physical tuple source changed: %x got=%+v length%d known%t err%v want=%+v", code, ins, length, known, err, want) + } + }) + } + for _, bad := range [][]byte{ + append([]byte{0x67}, fixture.code...), + append([]byte{0x64, 0x67}, fixture.code...), + append([]byte{0x65, 0x67}, fixture.code...), + } { + if _, _, known, err := decodedX86DwordToQwordMultiplyInstruction(bad, mode); !known || err == nil { + t.Fatal("address override acquired unbound source permission", fixture.row, mode, err) + } + } + } + wrongW := append([]byte(nil), fixture.code...) + wrongW[2] &^= 0x80 + if _, _, known, err := decodedX86DwordToQwordMultiplyInstruction(wrongW, 64); !known || err == nil { + t.Fatal("full/broadcast EVEX.W0 accepted", fixture.row, err) + } + } + t.Logf("%d independently assembled full/broadcast,unmasked/merge/zero,disp8/disp32,SIB/bank witnesses ×two modes ×three segment prefixes; address/W rejection retained", len(fixtures)) +} + +func x86DwordProductDirectives(code []byte) string { + var out strings.Builder + for _, value := range code { + fmt.Fprintf(&out, "BYTE $%d\n", value) + } + return out.String() +} + +func TestX86DwordProductTupleOffsetsThreeAPIsFiveTargetObjects(t *testing.T) { + fixtures := x86DwordProductDisplacementOracles(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw%t", triple, raw), func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + var source, declarations strings.Builder + declarations.WriteString("package tuples\n") + count := 0 + for _, fixture := range fixtures { + // SP/BP decoder identity is proved above, without using it + // to confer an unbound native frame in this object fixture. + if fixture.mem.Base == SP || fixture.mem.Base == BP || arch == "386" && (fixture.high || fixture.mem.Base == "R13" || !raw && fixture.mask) { + continue + } + row := fixture.row + if raw { + row = x86DwordProductDirectives(fixture.code) + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", count, row) + fmt.Fprintf(&declarations, "func F%d()\n", count) + count++ + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "tuples", declarations.String()) + sigs := map[string]FuncSig{} + for i := 0; i < count; i++ { + name := fmt.Sprintf("F%d", i) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + resolve := func(name string) string { return strings.TrimPrefix(name, "·") } + options := Options{Goarch: arch, TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "tuple-text.ll", "tuple-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "tuple-module.ll", "tuple-module.o", module.String()) + goModule, err := translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: resolve}) + if goModule != nil && !goModule.Module.IsNil() { + defer goModule.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "tuple-go.ll", "tuple-go.o", goModule.Module.String()) + t.Logf("%d independently assembled tuple/disp32 boundary forms ×three APIs", count) + }) + } + } +} + +func TestX86DwordProductTupleRawRejectedPrefixesThreeAPIs(t *testing.T) { + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + op, tuple := name, width + if broadcast { + op, tuple = name+".BCST", 8 + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row := fmt.Sprintf("%s %d(AX),%s1,K1,%s2", op, tuple, bank, bank) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + wrongW := append([]byte(nil), code[:len(code)-1]...) + wrongW[2] &^= 0x80 + for _, bad := range [][]byte{append([]byte{0x67}, code[:len(code)-1]...), append([]byte{0x64, 0x67}, code[:len(code)-1]...), wrongW} { + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(fmt.Sprintf("%s/%x/%s", row, bad, triple), func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT ·F(SB),4,$0-0\n" + x86DwordProductDirectives(bad) + "RET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"F": {Name: "F", Ret: Void}}, ResolveSym: func(name string) string { return strings.TrimPrefix(name, "·") }} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("reserved tuple/address text accepted", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil || !module.IsNil() { + t.Fatal("reserved tuple/address module accepted", err) + } + pkg := mustGoPackage(t, "tuples", "package tuples\nfunc F()\n") + result, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err == nil || result != nil { + t.Fatal("reserved tuple/address Go module accepted", err) + } + }) + } + } + } + } + } +} + +func TestX86DwordProductVEXDisplacementsRemainUnscaled(t *testing.T) { + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, bank := range []string{"X", "Y"} { + for _, units := range []int{-129, -128, -1, 1, 127, 128} { + for _, memory := range []MemRef{{Base: AX}, {Base: BP}, {Base: AX, Index: CX, Scale: 4}, {Base: SP, Index: CX, Scale: 2}, {Base: "R13", Index: "R12", Scale: 8}} { + memory.Off = int64(units) + row := fmt.Sprintf("%s %s,%s1,%s2", name, (Operand{Kind: OpMem, Mem: memory}).String(), bank, bank) + original := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+row+"\nRET\n") + if original[0] != 0xc4 && original[0] != 0xc5 { + t.Fatal("actual Go no longer emitted VEX", row, original) + } + for _, mode := range []int{32, 64} { + for _, segment := range []Reg{"", FS, GS} { + t.Run(fmt.Sprintf("%s/mode%d/segment%s", row, mode, segment), func(t *testing.T) { + code := append([]byte(nil), original[:len(original)-1]...) + if segment != "" { + prefix := byte(0x64) + if segment == GS { + prefix = 0x65 + } + code = append([]byte{prefix}, code...) + } + ins, length, known, err := decodedX86DwordToQwordMultiplyInstruction(code, mode) + if mode == 32 && memory.Base == "R13" { + if !known || err == nil { + t.Fatal("VEX extended address acquired mode32 binding") + } + return + } + want := memory + want.Segment = segment + if !known || err != nil || length != len(code) || !ins.x86Encoded || ins.Args[0].Kind != OpMem || !reflect.DeepEqual(ins.Args[0].Mem, want) { + t.Fatal("VEX source displacement incorrectly scaled", row, ins, err, want) + } + }) + } + } + } + } + } + } +} diff --git a/amd64_dword_product_fault_runtime_unix_test.go b/amd64_dword_product_fault_runtime_unix_test.go new file mode 100644 index 000000000..935188954 --- /dev/null +++ b/amd64_dword_product_fault_runtime_unix_test.go @@ -0,0 +1,179 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Independent fault envelopes include all eight bytes of each potentially +// active qword, even though only its low dword affects the numerical product. +// Original SSE instructions are the bytes emitted by Go, not mnemonic aliases +// assembled by the C compiler. No original AVX512 execution is claimed here. +func TestCrossLinuxRuntimeMatrixX86DwordProductDeadResultFaultEnvelopes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, originals, cases strings.Builder + declarations.WriteString("package faultoracle\n") + count := 0 + for _, name := range x86DwordProductOracleNames { + legacy := !strings.HasPrefix(name, "V") + widths := []int{16} + if !legacy { + widths = append(widths, 32, 64) + } + for _, width := range widths { + for broadcast := 0; broadcast < 2; broadcast++ { + if legacy && broadcast != 0 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy && masking != 0 { + continue + } + tuple := width + if broadcast != 0 { + tuple = 8 + } + for _, displacement := range []int{0, tuple, -tuple} { + if legacy && displacement != 0 { + continue + } + fn := fmt.Sprintf("F%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + op, args := name, fmt.Sprintf("%d(BX),%s1", displacement, bank) + if broadcast != 0 { + op += ".BCST" + } + if masking == 2 { + op += ".Z" + } + if !legacy { + if masking != 0 { + args += ",K1" + } + args += "," + bank + "2" + } + instruction := op + " " + args + if displacement != 0 { + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+instruction+"\nRET\n") + instruction = x86DwordProductDirectives(actual[:len(actual)-1]) + } + fmt.Fprintf(&declarations, "func %s(p *[64]byte,mask uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),BX\nMOVQ mask+8(FP),AX\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\nKMOVQ AX,K1\n%s\nRET\n", fn, instruction) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint64_t);\n", fn) + original := "NULL" + if legacy { + original = "original_" + fn + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\nPXOR X1,X1\n"+name+" (DI),X1\nRET\n") + var bytes []string + for _, value := range actual { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "extern void %s(const uint8_t*,uint64_t);\n__asm__(%q);\n", original, ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + } + fmt.Fprintf(&cases, "{%s,%s,%d,%d,%d,%t,%d},\n", fn, original, width, broadcast, masking, legacy, displacement) + } + } + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + strict, deviations := 1, 0 + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + strict, deviations = 0, 8 + } + reference := fmt.Sprintf(x86DwordProductFaultC, prototypes.String(), originals.String(), cases.String(), strict) + for _, pipeline := range []struct{ name, backend, ir string }{ + {"backend_O0", "-O0", ir}, {"backend_O2", "-O2", ir}, + {"IR_O2/backend_O2", "-O2", x86DwordProductIROptimize(t, ir)}, + } { + t.Run(pipeline.name, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "dword-product-fault", pipeline.ir, reference, pipeline.backend) + want := fmt.Sprintf("dword-product fault outcomes %d original SSE outcomes 80 alignment deviations %d failures 0\n", count*10*4, deviations) + if string(output) != want { + t.Fatalf("native width/alignment, dead result or qword masked fault suppression: %s", output) + } + t.Log(strings.TrimSpace(string(output))) + }) + } +} + +const x86DwordProductFaultC = ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +%s +%s +typedef struct {void(*translated)(const uint8_t*,uint64_t);void(*original)(const uint8_t*,uint64_t);unsigned width,broadcast,masking,legacy;int displacement;} Kernel; +static Kernel kernels[]={%s}; +static const uint64_t masks[]={0,1,0x80,UINT64_MAX}; +static unsigned need(Kernel k,uint64_t mask) { + if(!k.masking) return k.broadcast?8:k.width; + unsigned high=0; + for(unsigned i=0;i>i)&1) high=k.broadcast?8:(i+1)*8; + return high; +} +static int observe(Kernel k,unsigned address,uint64_t mask,int original) { + pid_t child=fork(); + if(child<0) return -1; + if(child==0) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + _Alignas(64) uint8_t data[256];memset(data,0xa7,sizeof(data)); + const uint8_t *p=data+64; + if(address==1) p++; + if(address==2) p=NULL; + if(address>=3) { + long page=sysconf(_SC_PAGESIZE);if(page<=0) _exit(121); + uint8_t *mapping=mmap(NULL,2*(size_t)page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(mapping==MAP_FAILED) _exit(122); + memset(mapping,0xa7,(size_t)page); + if(mprotect(mapping+page,(size_t)page,PROT_NONE)) _exit(123); + unsigned available[]={0,4,7,8,k.width-1,k.width,64}; + p=mapping+page-available[address-3]; + } + if(original) k.original(p,mask);else k.translated(p,mask); + _exit(0); + } + int status=0;if(waitpid(child,&status,0)!=child) return -2; + if(WIFEXITED(status)) return WEXITSTATUS(status)?-3:0; + if(WIFSIGNALED(status)&&(WTERMSIG(status)==SIGSEGV||WTERMSIG(status)==SIGBUS)) return 1; + return -4; +} +static int expected(Kernel k,unsigned address,uint64_t mask) { + unsigned bytes=need(k,mask),available[]={0,4,7,8,k.width-1,k.width,64}; + if(k.legacy&&(address==1||(address>=3&&(available[address-3]%%16)!=0))) return 1; + return (address==2&&bytes)||(address>=3&&bytes&&(int)bytes+k.displacement>(int)available[address-3]); +} +int main(void) { + unsigned count=0,native=0,deviations=0,failures=0;const int strict=%d; + for(unsigned i=0;i= 0 && !x86PackedShiftMemoryValid(form.source, arch) { + err = fmt.Errorf("386 %s physical source has an unbound address", op) + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && binary.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: dword product has no bound address-size override", ErrProbeNeedsContext) + } + } + return form, true, err +} + +func x86TypedDwordProductByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86DwordProductProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: form.first, operand: form.source, width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), read: true, write: true}, + }}, true, nil +} diff --git a/amd64_dword_product_forms_test.go b/amd64_dword_product_forms_test.go new file mode 100644 index 000000000..d3d7d1985 --- /dev/null +++ b/amd64_dword_product_forms_test.go @@ -0,0 +1,452 @@ +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +var x86DwordProductOracleNames = []string{"PMULDQ", "PMULULQ", "VPMULDQ", "VPMULUDQ"} + +func x86DwordProductOracleRows(t *testing.T, arch string) []string { + t.Helper() + var rows []string + for _, name := range x86DwordProductOracleNames[:2] { + for _, source := range []string{"X0", "X7", "(AX)", "16(AX)(CX*2)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+",X1") + } + if arch == "amd64" { + rows = append(rows, name+" X15,X1") + } + } + if arch == "amd64" { + for _, source := range []string{"M0", "M7", "(AX)", "value+0(FP)", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + rows = append(rows, "PMULULQ "+source+",M1") + } + } + official := x86OfficialPackedBinaryWitnesses(t) + for _, name := range x86DwordProductOracleNames[2:] { + if len(official[name]) == 0 { + t.Fatal("independent complete Go rules omitted", name) + } + for _, row := range official[name] { + if arch != "386" || strings.Count(row, ",") != 3 { + rows = append(rows, row) + } + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{"value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+","+bank+"1,"+bank+"2", + name+".BCST "+source+","+bank+"1,"+bank+"2") + } + if arch != "386" || bank != "Z" { + rows = append(rows, name+" "+bank+"20,"+bank+"21,"+bank+"22") + } + } + } + return rows +} + +func TestX86DwordProductCompleteIndependentEncoderRows(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, table := range []struct{ name, body string }{ + {"ymm", "{Zm_r_xm, 1, argList{Ymm, Ymr}},\n{Zm_r_xm, 2, argList{Yxm, Yxr}},"}, + {"yxm_q4", "{Zm_r, 1, argList{Yxm, Yxr}},"}, + } { + start := strings.Index(string(legacy), "var "+table.name+" = []ytab{") + if start < 0 { + t.Fatal("actual Go omitted table", table.name) + } + actual := strings.SplitN(string(legacy)[start:], "\n}", 2)[0] + actual = strings.SplitN(actual, "[]ytab{", 2)[1] + actualTokens, err := x86SignedPackedFloatGoOpcodeTokens(actual) + if err != nil { + t.Fatal(err) + } + wantTokens, err := x86SignedPackedFloatGoOpcodeTokens(table.body) + if err != nil || !reflect.DeepEqual(actualTokens, wantTokens) { + t.Fatal("actual Go complete legacy zcase/zoffset/classes changed", table.name, actual) + } + } + for _, row := range []string{"{APMULDQ, yxm_q4, Pq4, opBytes{0x28}}", "{APMULULQ, ymm, Py1, opBytes{0xf4, Pe, 0xf4}}"} { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual Go legacy prefix/opbytes changed", row) + } + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + classes := []string{"Yxm,Yxr,Yxr", "Yym,Yyr,Yyr", "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr"} + counts := map[string]int{} + for _, rule := range rules { + if rule.Opcode != "VPMULDQ" && rule.Opcode != "VPMULUDQ" { + continue + } + mapName, opcode := "0F", "0xF4" + if rule.Opcode == "VPMULDQ" { + mapName, opcode = "0F38", "0x28" + } + var opbytes []string + for _, width := range []int{128, 256} { + opbytes = append(opbytes, fmt.Sprintf("avxEscape | vex%d | vex66 | vex%s | vexW0", width, mapName), opcode) + } + for _, width := range []int{128, 256, 512} { + opbytes = append(opbytes, fmt.Sprintf("avxEscape | evex%d | evex66 | evex%s | evexW1", width, mapName), fmt.Sprintf("evexN%d | evexBcstN8 | evexZeroingEnabled", width/8), opcode) + } + index := counts[rule.Opcode] + zcase := "Zevex_rm_v_r" + if index < 2 { + zcase = "Zvex_rm_v_r" + } else if index%2 == 1 { + zcase = "Zevex_rm_v_k_r" + } + if index >= 8 || rule.RowIndex != index || rule.Ytab != "_yvandnpd" || rule.Prefix != "Pavx" || + rule.ZCase != zcase || rule.ZOffset != []int{2, 2, 0, 3, 0, 3, 0, 3}[index] || + rule.EncodingOffset != []int{0, 2, 4, 4, 7, 7, 10, 10}[index] || + strings.Join(rule.Classes, ",") != classes[index] || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatal("independent whole Go encoder row changed", rule) + } + counts[rule.Opcode]++ + } + if len(x86DwordProductSpecs) != 4 || counts["VPMULDQ"] != 8 || counts["VPMULUDQ"] != 8 { + t.Fatal("entire signed/unsigned family is not closed", counts) + } + for _, name := range x86DwordProductOracleNames { + if _, exists := x86DwordProductSpecs[Op(name)]; !exists { + t.Fatal("production omitted independent family member", name) + } + } + // PMULUDQ is a decoder alias, never another accepted Go source mnemonic. + arch, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || strings.Contains(string(arch), "instructions[\"PMULUDQ\"]") { + t.Fatal("actual Go alias inventory changed", err) + } +} + +func TestX86DwordProductWholeFormatsEffectsAndMemoryShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + rows := x86DwordProductOracleRows(t, arch) + var source strings.Builder + for i, row := range rows { + fmt.Fprintf(&source, "TEXT f%d(SB),4,$64-64\n%s\nRET\n", i, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for index, fn := range file.Funcs { + ins := fn.Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86DwordProductProperties(ins.Op, ins, arch) + high := arch == "386" && strings.Contains(rows[index], "20") + if !known || high && !errors.Is(err, ErrProbeNeedsContext) || !high && err != nil { + t.Fatal("Go-accepted source lost syntax/context classification", arch, rows[index], err) + } + if high { + continue + } + for i, operand := range ins.Args { + width, write, valid := x86InstructionMemoryShape(ins, i, arch) + if valid != (i == form.memory && !form.mmx) || valid && (write || width != int64(form.memoryBytes)) { + t.Fatal("native tuple shape changed", rows[index], width, write, valid) + } + if operand.Kind == OpFP { + width, write, valid = X86FPAccessBytes(ins, i, arch) + if !valid || write || width != int64(form.memoryBytes) { + t.Fatal("FP observation lost native memory span", rows[index], width, write, valid) + } + } + } + if form.legacy { + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 || effect.flagsDefined != 0 || effect.flagsKilled != 0 || effect.flagsRead != 0 || + !effect.operands[0].read || effect.operands[0].write || !effect.operands[1].read || !effect.operands[1].write || + effect.operands[0].width != int64(form.byteWidth) || effect.operands[1].width != int64(form.byteWidth) || !reflect.DeepEqual(effect.operands[0].operand, form.source) { + t.Fatal("legacy typed read/modify/write/flags/bank mismatch", effect, err) + } + } + if before != fmt.Sprintf("%#v", ins) || x86CallerVectorMoveKnown(ins) != !form.mmx { + t.Fatal("form modified source or borrowed MMX input permission", ins) + } + } + t.Logf("%s: %d independently Go-accepted complete-family forms; syntax/context/effects distinct", arch, len(rows)) + } +} + +func TestX86DwordProductThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/raw=%t", triple, raw), func(t *testing.T) { + rows := x86DwordProductOracleRows(t, arch) + if raw { + rows = nil + for _, name := range x86DwordProductOracleNames[2:] { + for _, row := range x86OfficialPackedBinaryWitnesses(t)[name] { + code := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + var directives strings.Builder + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&directives, "BYTE $%d\n", value) + } + rows = append(rows, directives.String()) + } + } + } + var source, declarations strings.Builder + declarations.WriteString("package product\n") + count := 0 + for _, row := range rows { + if !raw { + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := parsed.Funcs[0].Instrs[1] + form, _, err := x86DwordProductProperties(ins.Op, ins, arch) + if errors.Is(err, ErrProbeNeedsContext) || form.mmx || form.legacy && form.memory >= 0 && (form.source.Kind == OpFP || form.source.Kind == OpMem && form.source.Mem.Base == SP) { + continue + } + if err != nil { + t.Fatal(err) + } + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$64-64\n%s\nRET\n", count, row) + fmt.Fprintf(&declarations, "func F%d(value [64]byte)\n", count) + count++ + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + pkg := mustGoPackage(t, "product", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for i := 0; i < count; i++ { + name := fmt.Sprintf("F%d", i) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "product-text.ll", "product-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "product-module.ll", "product-module.o", module.String()) + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: resolve}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "product-go.ll", "product-go.o", translated.Module.String()) + t.Logf("%d independent forms compiled through each of the three APIs", count) + }) + } + } +} + +func x86DwordProductReservedZeroObservation(t *testing.T, name string) { + t.Helper() + source := "TEXT f(SB),4,$0-0\n" + name + ".Z (AX),X1,X2\nRET\n" + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatal(failure) + } + if err != nil { + if !bytes.Contains(output, []byte("mask register must be specified for .Z instructions")) { + t.Fatal("unknown Go rejection", string(output)) + } + return + } + // Historical Go1.20 emits reserved EVEX.z=1,aaa=0. It is not a legal + // format and remains rejected by every translation API and raw decoder. + mapByte, opcode := byte(0xf1), byte(0xf4) + if name == "VPMULDQ" { + mapByte, opcode = 0xf2, 0x28 + } + code := []byte{0x62, mapByte, 0xf5, 0x88, opcode, 0x10, 0xc3} + var listing []string + for _, value := range code { + listing = append(listing, fmt.Sprintf("%02x", value)) + } + if !bytes.Contains(output, []byte(strings.Join(listing, " "))) { + t.Fatal("historical reserved encoding changed", string(output)) + } + if _, _, known, err := decodedX86DwordToQwordMultiplyInstruction(code[:len(code)-1], 64); !known || err == nil { + t.Fatal("historical maskless zero encoding gained physical permission", known, err) + } +} + +func TestX86DwordProductDispatchIdentityBeforeEmission(t *testing.T) { + for _, name := range x86DwordProductOracleNames { + opcode := Op(name) + row := name + " X0,X1" + if strings.HasPrefix(name, "V") { + row += ",X2" + } + for _, sourceOp := range []Op{"", opcode, Op(strings.ToLower(name)), "UNKNOWN", "MOVQ"} { + file, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ins.Op = sourceOp + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + known, term, err := ctx.lowerDwordToQwordMultiply(opcode, ins) + if sourceOp == "UNKNOWN" || sourceOp == "MOVQ" { + if !known || term || err == nil || before != out.String() { + t.Fatal("conflicting dispatch/source identity emitted IR", opcode, sourceOp, err) + } + } else if !known || term || err != nil { + t.Fatal("compatible helper identity rejected", opcode, sourceOp, err) + } + if ins.Op != sourceOp { + t.Fatal("helper mutated input identity") + } + } + } +} + +func TestX86DwordProductMalformedAndContextThreeAPIsBeforeIR(t *testing.T) { + for _, name := range x86DwordProductOracleNames { + vector := strings.HasPrefix(name, "V") + rows := []string{name + " $1,X1", name + " Y0,Y1", name + " X16,X0", name + " X0,(AX)", name + " (AX)(SP*2),X1"} + if vector { + rows = []string{name + " $1,X1,X2", name + " X0,Y1,X2", name + " X0,X1,K0,X2", name + " X0,X1,(AX)", name + ".BCST X0,X1,X2", name + ".SAE X0,X1,X2", name + ".RN_SAE X0,X1,X2", name + ".Z.BCST (AX),X1,K1,X2", name + " (AX)(SP*2),X1,X2"} + x86DwordProductReservedZeroObservation(t, name) + rows = append(rows, name+".Z (AX),X1,X2") + } + for _, row := range rows { + if !strings.Contains(row, ".Z (") { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(row+"/"+triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + file, err := Parse(ArchAMD64, "TEXT ·F(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + return + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"F": {Name: "F", Ret: Void}}, ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("malformed text emitted IR", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil || !module.IsNil() { + t.Fatal("malformed direct module accepted", err) + } + pkg := mustGoPackage(t, "product", "package product\nfunc F()\n") + result, err := translateGoModuleInContext(ctx, pkg, []byte("TEXT ·F(SB),4,$0-0\n"+row+"\nRET\n"), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err == nil || result != nil { + t.Fatal("malformed Go module accepted", err) + } + }) + } + } + } +} + +func TestX86DwordProductMode32PhysicalMarkerAndMMXBank(t *testing.T) { + for _, name := range x86DwordProductOracleNames[2:] { + for _, row := range x86OfficialPackedBinaryWitnesses(t)[name] { + if strings.Count(row, ",") != 3 { + continue + } + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins, length, known, err := decodedX86DwordToQwordMultiplyInstruction(code[:len(code)-1], 32) + if !known || err != nil || length != len(code)-1 || !ins.x86Encoded { + t.Fatal("actual physical low-bank mode32 decode missing", row, ins, err) + } + if _, _, err := x86DwordProductProperties(ins.Op, ins, "386"); err != nil { + t.Fatal("physical mask inherited named frontend limit", row, err) + } + data, err := json.Marshal(ins) + if err != nil { + t.Fatal(err) + } + var restored Instr + if err := json.Unmarshal(data, &restored); err != nil { + t.Fatal(err) + } + restored.Raw = "BYTE genuine-looking decoder text" + if _, _, err := x86DwordProductProperties(restored.Op, restored, "386"); err == nil { + t.Fatal("JSON/raw text fabricated private physical marker") + } + } + } + for _, source := range []string{"AX", "SP", "BP", "R14", "R15"} { + row := "PMULULQ " + source + ",M1" + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + if known, _, err := ctx.lowerDwordToQwordMultiply(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]); !known || !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Fatal("GP spelling became initialized MMX transport", row, err) + } + } +} diff --git a/amd64_dword_product_runtime_test.go b/amd64_dword_product_runtime_test.go new file mode 100644 index 000000000..55bda4703 --- /dev/null +++ b/amd64_dword_product_runtime_test.go @@ -0,0 +1,243 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86DwordProductIROptimize(t *testing.T, ir string) string { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := parseIRModuleInContext(ctx, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + if err := module.RunPasses("default", llvm.TargetMachine{}, options); err != nil { + t.Fatalf("LLVM22 dword-product IR default: %v", err) + } + return module.String() +} + +// This scalar reference is independent of the production lane model, typed +// specifications and LLVM operations. All products fit widened 64-bit math. +const x86DwordProductCReference = ` +#include +#include +#include +#include +#include +static uint32_t dword(const uint8_t *p) { + return (uint32_t)p[0]|(uint32_t)p[1]<<8|(uint32_t)p[2]<<16|(uint32_t)p[3]<<24; +} +static uint64_t product(uint32_t x,uint32_t y,int sign) { + if(!sign) return (uint64_t)x*y; + int64_t a=x81) {x=(uint32_t)(sample*7919+i*104729);y=(uint32_t)(sample*65537+i*37);} + for(int j=0;j<4;j++) {a.value[i+j]=(uint8_t)(x>>(j*8));b.value[i+j]=(uint8_t)(y>>(j*8));} + } + struct Aligned beforeA=a,beforeB=b,beforeOld=old; + uint64_t seed=state?UINT64_C(0x8d5):0; + uint32_t profile=(uint32_t)(0x1f80|((sample%4)<<13)|(sample%64)),saved=_mm_getcsr(); + _mm_setcsr(profile); + program(out.value,a.value,b.value,old.value,masks[m],seed,profile); + uint32_t after=_mm_getcsr();_mm_setcsr(saved); + uint64_t flags=0;uint32_t csr=0;memcpy(&flags,out.value+64,8);memcpy(&csr,out.value+72,4); + if((flags&0x8d5)!=seed||csr!=profile||after!=profile||memcmp(&a,&beforeA,sizeof(a))||memcmp(&b,&beforeB,sizeof(b))||memcmp(&old,&beforeOld,sizeof(old))) return 1; + const uint8_t *retained=alias==0?a.value:alias==1?b.value:old.value; + for(int i=0;i<64;i++) { + int active=!masking||((masks[m]>>(i/8))&1); + int broadcast=source==2||source>=5; + int displacement=source==3?width:source==4?-width:source==5?8:source==6?-8:0; + const uint8_t *sourceBytes=(const uint8_t *)&a+offsetof(struct Aligned,value); + uint64_t value=product(dword(sourceBytes+displacement+(broadcast?0:i/8*8)),dword(b.value+i/8*8),sign); + uint8_t want=i>((i%8)*8)):(masking==2?0:retained[i])):(legacy?retained[i]:0); + if(out.value[i]!=want) {fprintf(stderr,"%s sample%d mask%llx byte%d got%x want%x\n",name,sample,(unsigned long long)masks[m],i,out.value[i],want);return 2;} + } + for(int i=0;i<64;i++) if(out.before[i]!=0xc3||out.after[i]!=0xc3) return 3; + for(int i=76;i<80;i++) if(out.value[i]!=0xc3) return 4; + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86DwordProductIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, reference, checks strings.Builder + declarations.WriteString("package narroworacle\n") + reference.WriteString(x86DwordProductCReference) + count := 0 + for _, name := range x86DwordProductOracleNames { + legacy, sign := 0, 0 + if !strings.HasPrefix(name, "V") { + legacy = 1 + } + if strings.Contains(name, "MULDQ") { + sign = 1 + } + widths := []int{16} + if legacy == 0 { + widths = append(widths, 32, 64) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for source := 0; source < 7; source++ { + if legacy != 0 && source >= 2 { + continue + } + for alias := 0; alias < 3; alias++ { + if legacy != 0 && alias != 1 { + continue + } + for masking := 0; masking < 3; masking++ { + if legacy != 0 && masking != 0 { + continue + } + fn := fmt.Sprintf("P%d", count) + count++ + first := bank + "0" + op := name + if source != 0 { + first = "(CX)" + } + if source == 2 || source >= 5 { + op += ".BCST" + } + if source == 3 || source == 4 { + displacement := width + if source == 4 { + displacement = -width + } + first = fmt.Sprintf("%d(CX)", displacement) + } else if source >= 5 { + displacement := 8 + if source == 6 { + displacement = -8 + } + first = fmt.Sprintf("%d(CX)", displacement) + } + if masking == 2 { + op += ".Z" + } + args := first + "," + bank + "1" + if legacy == 0 { + if masking != 0 { + args += ",K1" + } + args += fmt.Sprintf(",%s%d", bank, alias) + } + instruction := op + " " + args + if source >= 3 { + actual := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+instruction+"\nRET\n") + instruction = x86DwordProductDirectives(actual[:len(actual)-1]) + } + fmt.Fprintf(&declarations, "func %s(out *[80]byte,a,b,old *[64]byte,mask,flags uint64,csr uint32)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\nLDMXCSR csr+48(FP)\nPUSHQ flags+40(FP)\nPOPFQ\n%s\nVMOVUPS Z%d,(AX)\nPUSHFQ\nPOPQ R9\nMOVQ R9,64(AX)\nSTMXCSR 72(AX)\nRET\n", fn, instruction, alias) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t,uint32_t);\n", fn) + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,%d,%d,%d,%d))return 1;\n", fn, op+" "+args, width, sign, source, alias, masking, legacy) + } + } + } + } + } + reference.WriteString("int main(void){\n" + checks.String() + "return 0;}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "dword-product-full", ir, reference.String(), level) + }) + } + t.Run("IR_O2/backend_O2", func(t *testing.T) { + execution.compileAndRunLLVM(t, "dword-product-ir-o2", x86DwordProductIROptimize(t, ir), reference.String(), "-O2") + }) + t.Logf("%d LLVM kernels ×97 initialized samples ×2 six-flag seeds ×4 masks; all64 output bytes, both operand/destination aliases, actual-original-Go raw ±Full/±m64bcst offsets, odd dwords, full MXCSR/input/canaries; explicit IR default distinct from llc backend O0/O2. Portable LLVM execution does not execute original AVX512.", count) +} + +func TestCrossLinuxRuntimeMatrixX86DwordProductOriginalGoSSE(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\nfunc ReadCSR() uint32\nfunc SetCSR(value uint32)\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\nTEXT ·SetCSR(SB),4,$0-4\nLDMXCSR value+0(FP)\nRET\n") + tests.WriteString(`package oracle +import("testing";"unsafe";"runtime") +func TestOriginal(t *testing.T) { + runtime.LockOSThread();defer runtime.UnlockOSThread() + // Exercise the runtime's floating-point heap-sampling path between kernels. + // LockOSThread alone does not preserve MXCSR across ordinary Go allocations. + oldRate:=runtime.MemProfileRate;runtime.MemProfileRate=2 + defer func(){runtime.MemProfileRate=oldRate}() + saved:=ReadCSR();defer SetCSR(saved) + edges:=[...]uint32{0,1,2,0x7fffffff,0x80000000,0x80000001,0xfffffffe,0xffffffff} + for sample:=0;sample<97;sample++ {for _,seed:=range []uint64{0,0x8d5} { + var a,b,out [160]byte + align:=func(value *[160]byte)(*[64]byte,int){at:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15);return (*[64]byte)(unsafe.Pointer(&value[at])),at} + ap,_:=align(&a);bp,_:=align(&b);op,offset:=align(&out) + for i:=range a {a[i]=0xa7;b[i]=0xb9;out[i]=0xc3} + for i:=0;i<64;i+=4 {x,y:=edges[(sample+i/4)%8],edges[(sample/8+i*3)%8];for j:=0;j<4;j++ {ap[i+j]=byte(x>>uint(j*8));bp[i+j]=byte(y>>uint(j*8))}} + originalA,originalB:=a,b + profile:=uint32(0x1f80|((sample%4)<<13)|(sample%64)) +`) + count := 0 + for _, name := range x86DwordProductOracleNames[:2] { + for _, source := range []string{"X0", "X1", "(DI)"} { + fn := fmt.Sprintf("S%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed uint64,profile uint32)\n", fn) + // Own the complete state window in assembly. A Go allocation (even + // on a locked thread) can set MXCSR.PE in runtime heap sampling. + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\nMOVOU (DI),X0\nMOVOU (SI),X1\nLDMXCSR profile+48(FP)\nPUSHQ seed+40(FP)\nPOPFQ\n%s %s,X1\nMOVOU X1,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", fn, name, source) + alias, signed := 0, 0 + if source == "X1" { + alias = 1 + } + if name == "PMULDQ" { + signed = 1 + } + fmt.Fprintf(&tests, `{ + flags:=struct{before,value,after uint64}{0x11223344,0,0x44332211};csr:=struct{before,value,after uint32}{0x55667788,0,0x88776655} + // This deliberate prior-state difference makes loss of per-kernel setup + // deterministic, without relying on when the runtime happens to sample. + SetCSR(profile^0x20) + %s(op,ap,bp,&flags.value,&csr.value,seed,profile) + after:=ReadCSR() + if flags.value&0x8d5!=seed||csr.value!=profile||after!=profile||flags.before!=0x11223344||flags.after!=0x44332211||csr.before!=0x55667788||csr.after!=0x88776655||a!=originalA||b!=originalB { + t.Fatalf("%s sample%%d flags=%%x want=%%x csr=%%x after=%%x want=%%x flagCanaries=%%x/%%x csrCanaries=%%x/%%x inputChanges=%%t/%%t", + sample,flags.value,seed,csr.value,after,profile,flags.before,flags.after,csr.before,csr.after,a!=originalA,b!=originalB) + } + for i,value:=range out {want:=byte(0xc3);if i>=offset&&i>uint((index%%8)*8)) + };if value!=want {t.Fatalf("%s sample%%d byte%%d got%%x want%%x",sample,i,value,want)}} +} +`, fn, fn, alias, signed, fn) + } + } + tests.WriteString("}}\n}\n") + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests.String()) + t.Logf("%d actual original Go SSE kernels ×97 samples ×2 seeds, complete bytes, signed/unsigned extreme products, ignored odd dwords, flags/MXCSR/canaries. Rosetta is diagnostic; native Linux axis remains required.", count) +} diff --git a/amd64_ecosystem_test.go b/amd64_ecosystem_test.go new file mode 100644 index 000000000..a8ca30786 --- /dev/null +++ b/amd64_ecosystem_test.go @@ -0,0 +1,7853 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestTranslateAMD64LegacyDoubleShiftRegisterPair(t *testing.T) { + src := ` +TEXT shiftpair(SB),NOSPLIT,$0-0 + MOVQ $1, SI + MOVQ $2, CX + SHLQ $13, CX:SI + RET +` + translateAMD64EcosystemCase(t, src, "shiftpair") +} + +func TestTranslateX86UD2CompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns UD2 to ynone on both 386 and amd64: its complete + // accepted surface is exactly the no-operand form. + file, err := Parse(ArchAMD64, "TEXT ud2forms(SB),NOSPLIT,$0-0\n\tUD2\n") + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"ud2forms": {Name: "ud2forms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "unreachable") { + t.Fatalf("%s UD2 translation is not terminating:\n%s", target.goarch, ll) + } + compileLLVMToObject(t, llc, target.triple, "ud2-"+target.goarch+".ll", "ud2-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86UD2RejectsFormsOutsideGoAssemblerTables(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\tUD2 AX\n") + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("%s UD2 with an operand unexpectedly translated", target.goarch) + } + } +} + +func TestTranslateAMD64ScalarShiftRotateCompleteGoAssemblerForms(t *testing.T) { + // ROL/ROR/SAR/SAL/SHL/SHR share yshb/yshl. Each width accepts $1, + // unsigned imm8, CL, and CX counts with a same-width GP-or-memory target. + var src strings.Builder + src.WriteString("TEXT scalarshiftrotateforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ROL", "ROR", "SAR", "SAL", "SHL", "SHR"} { + for _, width := range []string{"B", "W", "L", "Q"} { + op := stem + width + if width == "B" { + fmt.Fprintf(&src, "\t%s $1, AH\n", op) + fmt.Fprintf(&src, "\t%s $255, R11\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, AL\n", op) + } else { + fmt.Fprintf(&src, "\t%s $1, AX\n", op) + fmt.Fprintf(&src, "\t%s $255, R11\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, AX\n", op) + } + } + } + // Go's special double-shift table adds signed imm8/CL/CX, GP source, + // GP-or-memory destination rows to SHL/SHR W/L/Q only. + for _, op := range []string{"SHLW", "SHLL", "SHLQ", "SHRW", "SHRL", "SHRQ"} { + fmt.Fprintf(&src, "\t%s $-128, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s $127, R11, (DI)\n", op) + fmt.Fprintf(&src, "\t%s CL, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s CX, R11, (DI)\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "scalarshiftrotateforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-shift-rotate-forms.ll", "scalar-shift-rotate-forms.o", ll) +} + +func TestTranslate386ScalarShiftRotateCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalarshiftrotateforms386(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ROL", "ROR", "SAR", "SAL", "SHL", "SHR"} { + for _, width := range []string{"B", "W", "L"} { + op := stem + width + if width == "B" { + fmt.Fprintf(&src, "\t%s $1, AH\n", op) + fmt.Fprintf(&src, "\t%s $255, DX\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, AL\n", op) + fmt.Fprintf(&src, "\t%s $1, BP\n", op) + fmt.Fprintf(&src, "\t%s $255, SI\n", op) + fmt.Fprintf(&src, "\t%s CL, DI\n", op) + } else { + fmt.Fprintf(&src, "\t%s $1, AX\n", op) + fmt.Fprintf(&src, "\t%s $255, SI\n", op) + fmt.Fprintf(&src, "\t%s CL, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, DI\n", op) + } + } + } + for _, op := range []string{"SHLW", "SHLL", "SHRW", "SHRL"} { + fmt.Fprintf(&src, "\t%s $-128, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s $127, SI, (DI)\n", op) + fmt.Fprintf(&src, "\t%s CL, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s CX, SI, (DI)\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "scalarshiftrotateforms386": {Name: "scalarshiftrotateforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "scalar-shift-rotate-386-forms.ll", "scalar-shift-rotate-386-forms.o", ll) +} + +func TestTranslateX86PortIOCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT portioforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"INB", "INW", "INL", "OUTB", "OUTW", "OUTL"} { + fmt.Fprintf(&src, "\t%s\n", op) + fmt.Fprintf(&src, "\t%s $7\n", op) + } + for _, op := range []string{"INSB", "INSW", "INSL", "OUTSB", "OUTSW", "OUTSL"} { + fmt.Fprintf(&src, "\t%s\n", op) + fmt.Fprintf(&src, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&src, "\tSTD; REPN; %s\n", op) + fmt.Fprintln(&src, "\tCLD") + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "portioforms": {Name: "portioforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `asm sideeffect "inb`, `asm sideeffect "inw`, `asm sideeffect "inl`, + `asm sideeffect "outb`, `asm sideeffect "outw`, `asm sideeffect "outl`, + `asm sideeffect "insb`, `asm sideeffect "insw`, `asm sideeffect "insl`, + `asm sideeffect "outsb`, `asm sideeffect "outsw`, `asm sideeffect "outsl`, + `asm sideeffect "rep; insb`, `asm sideeffect "rep; outsb`, + `asm sideeffect "repne; insb`, `asm sideeffect "repne; outsb`, + } { + if !strings.Contains(ll, want) { + t.Fatalf("%s port-I/O translation missing %q:\n%s", target.goarch, want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "port-io-"+target.goarch+".ll", "port-io-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86PortIORejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + for _, instruction := range []string{ + "INB AX", + "INW $1, AX", + "OUTL AX", + "OUTB $1, AX", + "INSB AX", + "OUTSL (SI)", + "REP; INL", + } { + name := target.goarch + "/" + strings.NewReplacer(" ", "_", ";", "_").Replace(instruction) + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("%s %q unexpectedly translated", target.goarch, instruction) + } + }) + } + } +} + +func TestTranslateX86StringInstructionsCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + widths []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", widths: []string{"B", "W", "L"}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", widths: []string{"B", "W", "L", "Q"}}, + } { + t.Run(target.goarch, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT stringforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"MOVS", "STOS", "SCAS"} { + for _, width := range target.widths { + op := stem + width + fmt.Fprintf(&src, "\t%s\n", op) + fmt.Fprintf(&src, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&src, "\tSTD; REPN; %s\n", op) + fmt.Fprintln(&src, "\tCLD") + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"stringforms": {Name: "stringforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, stem := range []string{"movs", "rep_stos", "rep_scas"} { + for _, width := range target.widths { + want := "@__plan9asm_" + stem + strings.ToLower(width) + if !strings.Contains(ll, want) { + t.Fatalf("%s string translation missing %q:\n%s", target.goarch, want, ll) + } + } + } + compileLLVMToObject(t, llc, target.triple, "string-forms-"+target.goarch+".ll", "string-forms-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslate386RejectsQwordStringInstructions(t *testing.T) { + for _, op := range []string{"MOVSQ", "STOSQ", "SCASQ"} { + t.Run(op, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+op+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("386 %s unexpectedly translated", op) + } + }) + } +} + +func TestTranslate386RejectsStaleNamedFPResultOffsets(t *testing.T) { + src := `TEXT legacyresults(SB),NOSPLIT,$0-56 + MOVL AX, retax+28(FP) + MOVL BX, retbx+32(FP) + MOVL CX, retcx+40(FP) + MOVL DX, retdx+44(FP) + MOVL SI, retsi+48(FP) + MOVL DI, retdi+52(FP) + MOVL BP, retbp+56(FP) + RET +` + requireStaleNamedFPGoDiagnostic(t, src) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + results := make([]FrameSlot, 0, 7) + names := []string{"retax", "retbx", "retcx", "retdx", "retsi", "retdi", "retbp"} + for i, name := range names { + results = append(results, FrameSlot{Offset: int64(28 + 4*i), Type: I32, Index: i, Field: -1, Name: name}) + } + _, err = Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "legacyresults": { + Name: "legacyresults", + Args: []LLVMType{I32, I32, I32, I32, I32, I32, I32}, + Ret: LLVMType("{ i32, i32, i32, i32, i32, i32, i32 }"), + Frame: FrameLayout{Results: results}, + }, + }, + }) + if err == nil { + t.Fatal("stale explicit result offsets were silently rewritten by name") + } +} + +func TestTranslateX86MOVLScalarResultMemoryForms(t *testing.T) { + file, err := Parse(ArchAMD64, `TEXT movlresults(SB),NOSPLIT,$0-8 + MOVL $-1, first+0(FP) + MOVL AX, second+4(FP) + RET +`) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"movlresults": { + Name: "movlresults", + Ret: LLVMType("{ i32, i32 }"), + Frame: FrameLayout{Results: []FrameSlot{ + {Offset: 0, Type: I32, Index: 0, Field: -1, Name: "first"}, + {Offset: 4, Type: I32, Index: 1, Field: -1, Name: "second"}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "movl-result-forms-"+target.goarch+".ll", "movl-result-forms-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86MOVLRejectsMemoryToResultMemory(t *testing.T) { + for _, source := range []string{"arg+0(FP)", "(AX)", "global(SB)"} { + t.Run(strings.NewReplacer("(", "_", ")", "_", "+", "_").Replace(source), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-4\n\tMOVL "+source+", result+0(FP)\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": { + Name: "bad", + Ret: I32, + Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1, Name: "result"}}}, + }}, + }); err == nil { + t.Fatalf("MOVL %s, result(FP) unexpectedly translated", source) + } + }) + } +} + +func TestTranslateX86ImplicitMultiplyDivideCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's ydivb/ydivl tables give MUL, IMUL, DIV, and IDIV one + // GP-or-memory operand. IMUL W/L/Q additionally uses yimul/yimul3 for + // the two- and three-operand signed multiply forms. + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + widths []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", widths: []string{"B", "W", "L"}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", widths: []string{"B", "W", "L", "Q"}}, + } { + t.Run(target.goarch, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT muldivforms(SB),NOSPLIT,$0-0\n") + for _, width := range target.widths { + reg := "BX" + for _, stem := range []string{"MUL", "IMUL", "DIV", "IDIV"} { + fmt.Fprintf(&src, "\t%s%s %s\n", stem, width, reg) + fmt.Fprintf(&src, "\t%s%s (SI)\n", stem, width) + } + if width != "B" { + fmt.Fprintf(&src, "\tIMUL%s $-128, CX\n", width) + fmt.Fprintf(&src, "\tIMUL%s $4294967295, DI\n", width) + fmt.Fprintf(&src, "\tIMUL%s BX, CX\n", width) + fmt.Fprintf(&src, "\tIMUL%s (SI), DI\n", width) + fmt.Fprintf(&src, "\tIMUL3%s $-128, BX, CX\n", width) + fmt.Fprintf(&src, "\tIMUL3%s $4294967295, (SI), DI\n", width) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"muldivforms": {Name: "muldivforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mul-div-forms-"+target.goarch+".ll", "mul-div-forms-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86ImplicitMultiplyDivideRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + common := []string{ + "MULB $1", + "DIVW AX, BX", + "IDIVL AX, BX", + "IMULB AX, BX", + "IMUL3B $1, AX, BX", + "IMULW (AX), (BX)", + "IMUL3W AX, BX, CX", + "IMUL3L $1, AX, (BX)", + "IMULL X0", + } + for _, target := range []struct { + goarch string + triple string + instructions []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", instructions: append(append([]string{}, common...), "MULQ AX", "IMULQ AX", "DIVQ AX", "IDIVQ AX", "IMUL3Q $1, AX, BX")}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instructions: append(append([]string{}, common...), "IMULQ $4294967296, AX", "IMUL3Q $-2147483649, AX, BX")}, + } { + for _, instruction := range target.instructions { + name := target.goarch + "/" + strings.NewReplacer(" ", "_", ",", "_", "(", "_", ")", "_").Replace(instruction) + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("%s accepted %q outside Go 1.27's multiply/divide tables", target.goarch, instruction) + } + }) + } + } +} + +func TestTranslateAMD64ScalarShiftRotateRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "SHRB $-1, AL", + "SHLB $256, AL", + "SARW AX, BX", + "SALL (AX), BX", + "ROLL $1, X1", + "RORB $1, X1", + "RORQ.Z $1, AX, AX", + "SHLB $1, AL, BL", + "SARL $1, AX, BX", + "SALW $1, AX, BX", + "ROLQ $1, AX, BX", + "SHLL $128, AX, BX", + "SHRL DX, AX, BX", + "SHLW $1, (AX), BX", + "SHRQ $1, AX, $2", + "SHLB $1, AX, BX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidscalarshiftrotate(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidscalarshiftrotate": {Name: "invalidscalarshiftrotate", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's shift/rotate tables", instruction) + } + }) + } +} + +func TestTranslate386ScalarShiftRotateRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "SHRQ $1, AX", + "ROLQ CL, (BX)", + "SHRB $1, SP", + "SARL $1, R8", + "SHLL $1, R8, AX", + "SHLQ $1, AX, BX", + "SHRQ CL, AX, (BX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidscalarshiftrotate386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidscalarshiftrotate386": {Name: "invalidscalarshiftrotate386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64IncDecCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT incdecforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"INCB", "DECB"} { + fmt.Fprintf(&src, "\t%s AL\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s BPB\n", op) + fmt.Fprintf(&src, "\t%s R15B\n", op) + fmt.Fprintf(&src, "\t%s R11\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + } + for _, op := range []string{"INCW", "DECW", "INCL", "DECL", "INCQ", "DECQ"} { + fmt.Fprintf(&src, "\t%s AX\n", op) + fmt.Fprintf(&src, "\t%s R11\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "incdecforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "inc-dec-forms.ll", "inc-dec-forms.o", ll) +} + +func TestTranslate386IncDecCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT incdecforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"INCB", "DECB"} { + fmt.Fprintf(&src, "\t%s AL\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s DX\n", op) + fmt.Fprintf(&src, "\t%s BP\n", op) + fmt.Fprintf(&src, "\t%s SI\n", op) + fmt.Fprintf(&src, "\t%s DI\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + } + for _, op := range []string{"INCW", "DECW", "INCL", "DECL"} { + fmt.Fprintf(&src, "\t%s AX\n", op) + fmt.Fprintf(&src, "\t%s SI\n", op) + fmt.Fprintf(&src, "\t%s (BX)\n", op) + fmt.Fprintf(&src, "\t%s SP\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "incdecforms386": {Name: "incdecforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "inc-dec-386-forms.ll", "inc-dec-386-forms.o", ll) +} + +func TestTranslateAMD64IncDecRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "INCB", + "INCW AX, BX", + "DECL $1", + "DECQ X0", + "INCQ.Z AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidincdec(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidincdec": {Name: "invalidincdec", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's INC/DEC tables", instruction) + } + }) + } +} + +func TestTranslate386IncDecRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "INCB SP", + "INCW R8", + "DECL R11", + "INCQ AX", + "DECQ (BX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidincdec386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidincdec386": {Name: "invalidincdec386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedWordMultiplyCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedwordmulforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMULLW", "PMULHW", "PMULHUW"} { + fmt.Fprintf(&src, "\t%s M0, M1\n", op) + fmt.Fprintf(&src, "\t%s (AX), M1\n", op) + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1\n", op) + } + src.WriteString("\tPMULHRSW X0, X1\n\tPMULHRSW (AX), X1\n") + for _, op := range []string{"VPMULLW", "VPMULHW", "VPMULHUW", "VPMULHRSW"} { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s X0, X1, K1, X2\n", op) + fmt.Fprintf(&src, "\t%s.Z Y0, Y1, K2, Y2\n", op) + fmt.Fprintf(&src, "\t%s.Z Z0, Z1, K3, Z2\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedwordmulforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-word-multiply-forms.ll", "packed-word-multiply-forms.o", ll) +} + +func TestTranslate386PackedWordMultiplyCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedwordmulforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMULLW", "PMULHW", "PMULHUW", "PMULHRSW"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1\n", op) + } + for _, op := range []string{"VPMULLW", "VPMULHW", "VPMULHUW", "VPMULHRSW"} { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z1, Z2\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedwordmulforms386": {Name: "packedwordmulforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-word-multiply-386-forms.ll", "packed-word-multiply-386-forms.o", ll) +} + +func TestTranslateAMD64PackedWordMultiplyRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMULLW Y0, Y1", + "PMULHRSW M0, M1", + "PMULHW.Z X0, X1", + "VPMULLW.BCST (AX), X1, X2", + "VPMULHW.SAE X0, X1, X2", + "VPMULHUW.Z X0, X1, X2", + "VPMULLW X0, Y1, Y2", + "VPMULHW X0, X1, K0, X2", + "VPMULHUW X0, X1, AX, X2", + "VPMULHRSW X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedwordmul(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedwordmul": {Name: "invalidpackedwordmul", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-word multiply tables", instruction) + } + }) + } +} + +func TestTranslate386PackedWordMultiplyRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PMULLW M0, M1", + "PMULHW (AX), M1", + "PMULHUW X8, X0", + "PMULHRSW X0, X8", + "VPMULLW Z8, Z1, Z2", + "VPMULHW X0, X1, K1, X2", + "VPMULHUW.Z X0, X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedwordmul386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedwordmul386": {Name: "invalidpackedwordmul386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedWordMultiplyAddCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses ymm for PMADDWL and _yvandnpd for VPMADDWD. + src := `TEXT packedwordmaddforms(SB),NOSPLIT,$0-0 + PMADDWL M0, M1 + PMADDWL (AX), M2 + PMADDWL X0, X1 + PMADDWL (AX), X15 + VPMADDWD X0, X1, X2 + VPMADDWD (AX), X1, X2 + VPMADDWD Y0, Y1, Y2 + VPMADDWD (AX), Y1, Y2 + VPMADDWD Z0, Z1, Z2 + VPMADDWD (AX), Z1, Z2 + VPMADDWD X20, X21, X22 + VPMADDWD Y20, Y21, Y22 + VPMADDWD Z20, Z21, Z22 + VPMADDWD X20, X21, K1, X22 + VPMADDWD.Z (AX), X21, K2, X22 + VPMADDWD Y20, Y21, K3, Y22 + VPMADDWD.Z (AX), Y21, K4, Y22 + VPMADDWD Z20, Z21, K5, Z22 + VPMADDWD.Z (AX), Z21, K6, Z22 + RET +` + ll := translateAMD64EcosystemCase(t, src, "packedwordmaddforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-word-multiply-add-forms.ll", "packed-word-multiply-add-forms.o", ll) +} + +func TestTranslate386PackedWordMultiplyAddCompleteGoAssemblerForms(t *testing.T) { + src := `TEXT packedwordmaddforms386(SB),NOSPLIT,$0-0 + PMADDWL X0, X1 + PMADDWL (AX), X7 + VPMADDWD X0, X1, X2 + VPMADDWD (AX), X1, X2 + VPMADDWD Y0, Y1, Y2 + VPMADDWD (AX), Y1, Y2 + VPMADDWD Z0, Z1, Z7 + VPMADDWD (AX), Z6, Z7 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedwordmaddforms386": {Name: "packedwordmaddforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-word-multiply-add-386-forms.ll", "packed-word-multiply-add-386-forms.o", ll) +} + +func TestTranslateAMD64PackedWordMultiplyAddRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMADDWL.Z X0, X1", + "PMADDWL X0", + "PMADDWL Y0, Y1", + "PMADDWL X0, M1", + "PMADDWL X16, X1", + "PMADDWL X0, (AX)", + "VPMADDWD X0, X1", + "VPMADDWD M0, M1, M2", + "VPMADDWD X0, Y1, Y2", + "VPMADDWD X0, X1, (AX)", + "VPMADDWD.BCST (AX), X1, X2", + "VPMADDWD.Z X0, X1, X2", + "VPMADDWD X0, X1, K0, X2", + "VPMADDWD X0, X1, AX, X2", + "VPMADDWD X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidpackedwordmadd(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedwordmadd": {Name: "invalidpackedwordmadd", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMADDWL/VPMADDWD tables", instruction) + } + }) + } +} + +func TestTranslate386PackedWordMultiplyAddRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PMADDWL M0, M1", + "PMADDWL (AX), M1", + "PMADDWL X8, X0", + "PMADDWL X0, X8", + "VPMADDWD Z8, Z1, Z2", + "VPMADDWD Z0, Z1, Z8", + "VPMADDWD X0, X1, K1, X2", + "VPMADDWD.Z X0, X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidpackedwordmadd386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedwordmadd386": {Name: "invalidpackedwordmadd386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedFloatShuffleCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedfloatshuffleforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"SHUFPS", "SHUFPD"} { + fmt.Fprintf(&src, "\t%s $0, X0, X1\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), X15\n", op) + } + for _, op := range []string{"VSHUFPS", "VSHUFPD"} { + fmt.Fprintf(&src, "\t%s $0, X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s $1, Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $2, (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $3, Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s $4, (AX), Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s $5, X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s $6, X0, X1, K1, X2\n", op) + fmt.Fprintf(&src, "\t%s.Z $7, Y0, Y1, K2, Y2\n", op) + fmt.Fprintf(&src, "\t%s.BCST $8, (AX), Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z $9, (AX), Z1, K3, Z2\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedfloatshuffleforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-float-shuffle-forms.ll", "packed-float-shuffle-forms.o", ll) +} + +func TestTranslate386PackedFloatShuffleCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT packedfloatshuffleforms386(SB),NOSPLIT,$0-0 + SHUFPS $0, X0, X1 + SHUFPS $255, (AX), X7 + SHUFPD $0, X0, X1 + SHUFPD $255, (AX), X7 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedfloatshuffleforms386": {Name: "packedfloatshuffleforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-float-shuffle-386-forms.ll", "packed-float-shuffle-386-forms.o", ll) +} + +func TestTranslateAMD64PackedFloatShuffleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "SHUFPS $-1, X0, X1", + "SHUFPD $256, X0, X1", + "SHUFPS.Z $0, X0, X1", + "VSHUFPS $0, X0, X1", + "VSHUFPD.SAE $0, X0, X1, X2", + "VSHUFPS.Z $0, X0, X1, X2", + "VSHUFPD $0, X0, Y1, Y2", + "VSHUFPS $0, X0, X1, K0, X2", + "VSHUFPD $0, X0, X1, AX, X2", + "VSHUFPS.BCST $0, X0, X1, X2", + "VSHUFPD.BCST $0, (AX), X1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatshuffle(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedfloatshuffle": {Name: "invalidpackedfloatshuffle", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-float shuffle tables", instruction) + } + }) + } +} + +func TestTranslate386PackedFloatShuffleRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "SHUFPS $0, X8, X0", + "SHUFPD $0, X0, X8", + "VSHUFPS $0, X0, X1, X2", + "VSHUFPD $0, Y0, Y1, Y2", + "VSHUFPS.BCST $0, (AX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatshuffle386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedfloatshuffle386": {Name: "invalidpackedfloatshuffle386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64ImmediatePackedBlendCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT immediatepackedblendforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PBLENDW", "BLENDPS", "BLENDPD"} { + fmt.Fprintf(&src, "\t%s $0, X0, X1\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), X15\n", op) + } + for _, op := range []string{"VPBLENDW", "VPBLENDD", "VBLENDPS", "VBLENDPD"} { + fmt.Fprintf(&src, "\t%s $0, X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), X1, X15\n", op) + fmt.Fprintf(&src, "\t%s $1, Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $254, (AX), Y1, Y15\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "immediatepackedblendforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "immediate-packed-blend-forms.ll", "immediate-packed-blend-forms.o", ll) +} + +func TestTranslate386ImmediatePackedBlendCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT immediatepackedblendforms386(SB),NOSPLIT,$0-0 + PBLENDW $0, X0, X1 + PBLENDW $255, (AX), X7 + BLENDPS $0, X0, X1 + BLENDPS $255, (AX), X7 + BLENDPD $0, X0, X1 + BLENDPD $255, (AX), X7 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "immediatepackedblendforms386": {Name: "immediatepackedblendforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "immediate-packed-blend-386-forms.ll", "immediate-packed-blend-386-forms.o", ll) +} + +func TestTranslateAMD64ImmediatePackedBlendRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PBLENDW $-1, X0, X1", + "BLENDPS $256, X0, X1", + "BLENDPD.Z $0, X0, X1", + "PBLENDW $0, X20, X1", + "BLENDPS $0, X0, X20", + "VPBLENDW $-1, X0, X1, X2", + "VPBLENDD $256, X0, X1, X2", + "VBLENDPS $0, X0, X1", + "VBLENDPD $0, X0, Y1, Y2", + "VPBLENDW $0, X20, X1, X2", + "VPBLENDD $0, Y0, Y20, Y2", + "VBLENDPS $0, Z0, Z1, Z2", + "VBLENDPD.Z $0, X0, X1, X2", + "VPBLENDW.BCST $0, (AX), X1, X2", + "VPBLENDD $0, X0, X1, K1, X2", + "VBLENDPS $0, X0, (AX), X2", + "VBLENDPD $0, X0, X1, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidimmediatepackedblend(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidimmediatepackedblend": {Name: "invalidimmediatepackedblend", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's immediate packed-blend tables", instruction) + } + }) + } +} + +func TestTranslate386ImmediatePackedBlendRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PBLENDW $0, X8, X0", + "BLENDPS $0, X0, X8", + "BLENDPD.Z $0, X0, X1", + "VPBLENDW $0, X0, X1, X2", + "VPBLENDD $0, Y0, Y1, Y2", + "VBLENDPS $0, X0, X1, X2", + "VBLENDPD $0, Y0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidimmediatepackedblend386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidimmediatepackedblend386": {Name: "invalidimmediatepackedblend386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedUnpackCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedunpackforms(SB),NOSPLIT,$0-0\n") + legacyMMX := []string{"PUNPCKLBW", "PUNPCKHBW", "PUNPCKLWL", "PUNPCKHWL", "PUNPCKLLQ", "PUNPCKHLQ"} + for _, op := range legacyMMX { + fmt.Fprintf(&src, "\t%s M0, M1\n", op) + fmt.Fprintf(&src, "\t%s (AX), M7\n", op) + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + for _, op := range []string{"PUNPCKLQDQ", "PUNPCKHQDQ"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + vectorOps := []string{ + "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLWD", "VPUNPCKHWD", + "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", + } + for _, op := range vectorOps { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1, X15\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y15\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z1, Z31\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s Y20, Y21, K1, Y22\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), Z21, K2, Z22\n", op) + if strings.Contains(op, "DQ") { + fmt.Fprintf(&src, "\t%s.BCST (AX), X21, X22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z21, Z22\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), Z21, K3, Z22\n", op) + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedunpackforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-unpack-forms.ll", "packed-unpack-forms.o", ll) +} + +func TestTranslate386PackedUnpackCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedunpackforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLWL", "PUNPCKHWL", + "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PUNPCKHQDQ", + } { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + for _, op := range []string{ + "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLWD", "VPUNPCKHWD", + "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", + } { + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + // Named high-register mode32 acceptance is retained by the separate + // actual-Go/physical-Context oracle, not fabricated LLVM registers. + fmt.Fprintf(&src, "\t%s X5, X6, X7\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + if strings.Contains(op, "DQ") { + fmt.Fprintf(&src, "\t%s.BCST (AX), Y1, Y2\n", op) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedunpackforms386": {Name: "packedunpackforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-unpack-386-forms.ll", "packed-unpack-386-forms.o", ll) +} + +func TestTranslateAMD64PackedUnpackRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PUNPCKLBW.Z X0, X1", + "PUNPCKLWL X20, X1", + "PUNPCKHWL X0, X20", + "PUNPCKLLQ M0, X1", + "PUNPCKHLQ X0, M1", + "PUNPCKLQDQ M0, M1", + "PUNPCKHQDQ Y0, X1", + "VPUNPCKLBW X0, X1", + "VPUNPCKHWD X0, Y1, Y2", + "VPUNPCKLDQ X0, (AX), X2", + "VPUNPCKHQDQ X0, X1, (AX)", + "VPUNPCKLBW X32, X1, X2", + "VPUNPCKHWD Z0, Z1, K0, Z2", + "VPUNPCKLWD.Z Z0, Z1, Z2", + "VPUNPCKHBW.BCST (AX), X1, X2", + "VPUNPCKLWD.BCST (AX), Y1, Y2", + "VPUNPCKLDQ.BCST X0, X1, X2", + "VPUNPCKHQDQ.BCST (AX), X1, Y2", + "VPUNPCKHDQ.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedunpack(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedunpack": {Name: "invalidpackedunpack", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-unpack tables", instruction) + } + }) + } +} + +func TestTranslate386PackedUnpackRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PUNPCKLBW M0, M1", + "PUNPCKHWL (AX), M1", + "PUNPCKLQDQ M0, M1", + "PUNPCKHBW X8, X0", + "PUNPCKHLQ X0, X8", + "VPUNPCKLWD Z8, Z1, Z2", + "VPUNPCKHWD Z0, Z8, Z2", + "VPUNPCKLDQ Z0, Z1, Z8", + "VPUNPCKLBW X0, X1, K1, X2", + "VPUNPCKLDQ.Z Z0, Z1, K1, Z2", + "VPUNPCKHQDQ.BCST.Z (AX), Y1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedunpack386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedunpack386": {Name: "invalidpackedunpack386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64QwordPermuteCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT qwordpermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMQ", "VPERMPD"} { + fmt.Fprintf(&src, "\t%s $-128, Y0, Y1\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y15\n", op) + fmt.Fprintf(&src, "\t%s $0, Y20, Y21\n", op) + fmt.Fprintf(&src, "\t%s $1, Z20, Z21\n", op) + fmt.Fprintf(&src, "\t%s $2, Y20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z $3, Z20, K2, Z21\n", op) + fmt.Fprintf(&src, "\t%s.BCST $4, (AX), Y21\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z $5, (AX), K3, Z21\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s Y20, Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y21, K1, Y22\n", op) + fmt.Fprintf(&src, "\t%s.Z Z20, Z21, K2, Z22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), Z21, K3, Z22\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "qwordpermuteforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "qword-permute-forms.ll", "qword-permute-forms.o", ll) +} + +func TestTranslate386QwordPermuteCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT qwordpermuteforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMQ", "VPERMPD"} { + fmt.Fprintf(&src, "\t%s $-128, Y0, Y1\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y7\n", op) + fmt.Fprintf(&src, "\t%s $0, Y20, Y21\n", op) + fmt.Fprintf(&src, "\t%s $1, Z0, Z1\n", op) + fmt.Fprintf(&src, "\t%s.BCST $2, (AX), Y21\n", op) + fmt.Fprintf(&src, "\t%s.BCST $3, (AX), Z7\n", op) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Z0, Z1, Z2\n", op) + fmt.Fprintf(&src, "\t%s Y20, Y21, Y22\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z1, Z7\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "qwordpermuteforms386": {Name: "qwordpermuteforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "qword-permute-386-forms.ll", "qword-permute-386-forms.o", ll) +} + +func TestTranslateAMD64QwordPermuteRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPERMQ $-129, Y0, Y1", + "VPERMPD $256, Y0, Y1", + "VPERMQ $-1, Y20, Y21", + "VPERMPD $-1, Z0, Z1", + "VPERMQ.BCST $-1, (AX), Y1", + "VPERMQ $0, X0, X1", + "VPERMPD X0, X1, X2", + "VPERMQ $0, Y0, Z1", + "VPERMPD Y0, Z1, Z2", + "VPERMQ Y0, (AX), Y2", + "VPERMPD Y0, Y1, (AX)", + "VPERMQ $0, Y0, K0, Y1", + "VPERMPD Z0, Z1, K0, Z2", + "VPERMQ.Z $0, Y0, Y1", + "VPERMPD.Z Y0, Y1, Y2", + "VPERMQ.BCST $0, Y0, Y1", + "VPERMPD.BCST Y0, Y1, Y2", + "VPERMQ.SAE $0, Y0, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidqwordpermute(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidqwordpermute": {Name: "invalidqwordpermute", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvpermq table", instruction) + } + }) + } +} + +func TestTranslate386QwordPermuteRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "VPERMQ $-1, Y20, Y21", + "VPERMPD $0, Z8, Z1", + "VPERMQ $0, Z0, Z8", + "VPERMPD Z8, Z1, Z2", + "VPERMQ Z0, Z8, Z2", + "VPERMPD Z0, Z1, Z8", + "VPERMQ $0, Y0, K1, Y1", + "VPERMPD Z0, Z1, K1, Z2", + "VPERMQ.BCST.Z (AX), Z1, K1, Z2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidqwordpermute386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidqwordpermute386": {Name: "invalidqwordpermute386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedFloatMoveCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns VMOVAPD/APS/UPD/UPS the same 16-row _yvmovapd + // table: bidirectional X/Y/Z register-or-memory moves, EVEX high + // registers, and K1-K7 merge/zero masking for every width and direction. + var src strings.Builder + src.WriteString("TEXT packedfloatmoveforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VMOVAPD", "VMOVAPS", "VMOVUPD", "VMOVUPS"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s1, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s %s20, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s20, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s21\n", op, width) + } + fmt.Fprintf(&src, "\t%s Z0, Z1\n", op) + fmt.Fprintf(&src, "\t%s Z1, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z2\n", op) + fmt.Fprintf(&src, "\t%s Z20, Z21\n", op) + fmt.Fprintf(&src, "\t%s Z20, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z21\n", op) + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s1, K1, %s2\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s1, K2, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), K3, %s2\n", op, width) + fmt.Fprintf(&src, "\t%s.Z %s1, K4, %s2\n", op, width, width) + // Go accepts .Z memory stores, but their native EVEX.z encoding + // raises #UD. Independent-entry and fault oracles cover that effect. + fmt.Fprintf(&src, "\t%s.Z %s1, K5, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s.Z (AX), K6, %s2\n", op, width) + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedfloatmoveforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-float-move-forms.ll", "packed-float-move-forms.o", ll) +} + +func TestTranslate386PackedFloatMoveCompleteGoAssemblerForms(t *testing.T) { + // Go accepts high X/Y spellings and all three-operand mask rows. Observe + // their Context separately; actual mode32 objects use the physical low bank. + var src strings.Builder + src.WriteString("TEXT packedfloatmoveforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VMOVAPD", "VMOVAPS", "VMOVUPD", "VMOVUPS"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s20, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s20, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s21\n", op, width) + fmt.Fprintf(&src, "\t%s %s20, K1, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z %s20, K2, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s.Z (AX), K3, %s21\n", op, width) + } + fmt.Fprintf(&src, "\t%s Z0, Z7\n", op) + fmt.Fprintf(&src, "\t%s Z7, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z7\n", op) + fmt.Fprintf(&src, "\t%s Z0, K1, Z7\n", op) + fmt.Fprintf(&src, "\t%s.Z Z0, K2, (AX)\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K3, Z7\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, x86PhysicalMode32ObjectSource(t, src.String(), "386")) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedfloatmoveforms386": {Name: "packedfloatmoveforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-float-move-386-forms.ll", "packed-float-move-386-forms.o", ll) +} + +func TestTranslateAMD64PackedFloatMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVAPD X1, Y2", + "VMOVAPS Y1, Z2", + "VMOVUPD Z1, X2", + "VMOVUPS (AX), (BX)", + "VMOVAPD K1, X2", + "VMOVAPS X1, K0, X2", + "VMOVUPD (AX), K0, X2", + "VMOVUPS.Z X1, X2", + "VMOVAPD.BCST (AX), X2", + "VMOVAPS.SAE X1, X2", + "VMOVUPD.RN_SAE X1, X2", + "VMOVUPS X1, K1, Y2", + "VMOVAPD X1, X2, K1", + "VMOVAPS X1", + "VMOVUPD X1, K1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedfloatmove": {Name: "invalidpackedfloatmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvmovapd table", instruction) + } + }) + } +} + +func TestTranslate386PackedFloatMoveRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "VMOVAPD Z8, Z1", + "VMOVAPS Z0, Z8", + "VMOVUPD Z8, (AX)", + "VMOVUPS (AX), Z8", + "VMOVAPD Z0, K1, Z8", + "VMOVAPS.Z Z8, K1, (AX)", + "VMOVUPD.Z (AX), K1, Z8", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedfloatmove386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedfloatmove386": {Name: "invalidpackedfloatmove386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateX86EVEXPackedIntegerMoveCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns this complete family the same 12-row _yvmovdqa32 + // table. Each opcode accepts bidirectional X/Y/Z register-or-memory + // moves plus K1-K7 merge/zero masking for every width and direction. + for _, target := range []struct { + goarch string + triple string + zregs []int + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", zregs: []int{0, 7}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", zregs: []int{0, 21}}, + } { + t.Run(target.goarch, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT evexpackedintegermoveforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VMOVDQA32", "VMOVDQA64", "VMOVDQU8", "VMOVDQU16", "VMOVDQU32", "VMOVDQU64"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s20, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s21\n", op, width) + fmt.Fprintf(&src, "\t%s %s1, K1, %s2\n", op, width, width) + fmt.Fprintf(&src, "\t%s %s1, K2, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), K3, %s2\n", op, width) + fmt.Fprintf(&src, "\t%s.Z %s1, K4, %s2\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z %s1, K5, (AX)\n", op, width) + fmt.Fprintf(&src, "\t%s.Z (AX), K6, %s2\n", op, width) + } + fmt.Fprintf(&src, "\t%s Z%d, Z%d\n", op, target.zregs[0], target.zregs[1]) + fmt.Fprintf(&src, "\t%s Z%d, (AX)\n", op, target.zregs[0]) + fmt.Fprintf(&src, "\t%s (AX), Z%d\n", op, target.zregs[1]) + fmt.Fprintf(&src, "\t%s Z%d, K1, Z%d\n", op, target.zregs[0], target.zregs[1]) + fmt.Fprintf(&src, "\t%s.Z Z%d, K2, (AX)\n", op, target.zregs[0]) + fmt.Fprintf(&src, "\t%s.Z (AX), K3, Z%d\n", op, target.zregs[1]) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, x86PhysicalMode32ObjectSource(t, src.String(), target.goarch)) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "evexpackedintegermoveforms": {Name: "evexpackedintegermoveforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "evex-packed-integer-move-"+target.goarch+".ll", "evex-packed-integer-move-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86EVEXPackedIntegerMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVDQA32 X1, Y2", + "VMOVDQA64 Y1, Z2", + "VMOVDQU8 (AX), (BX)", + "VMOVDQU16 X1, K0, X2", + "VMOVDQU32.Z X1, X2", + "VMOVDQU64.BCST (AX), Z2", + "VMOVDQU8.SAE X1, X2", + "VMOVDQU16 X1, K1, Y2", + "VMOVDQU32 X1", + "VMOVDQA64 X1, K1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvmovdqa32 table", instruction) + } + }) + } + for _, instruction := range []string{ + "VMOVDQA32 Z8, Z0", + "VMOVDQA64 Z0, Z8", + "VMOVDQU8 Z8, (AX)", + "VMOVDQU16 (AX), Z8", + "VMOVDQU32 Z0, K1, Z8", + "VMOVDQU64.Z (AX), K1, Z8", + } { + file, err := Parse(ArchAMD64, "TEXT bad386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad386": {Name: "bad386", Ret: Void}}, + }); err == nil { + t.Fatalf("386 Translate accepted architecture-restricted %q", instruction) + } + } +} + +func TestTranslateAMD64PackedSignExtendMoveCompleteGoAssemblerForms(t *testing.T) { + // The complete signed widening family consists of six legacy yxm_q4 + // instructions plus six V instructions. BW/WD/DQ use _yvcvtdq2pd + // (X source for X/Y destinations and Y source for Z); BD/BQ/WQ use + // _yvbroadcastss (X source for every destination width). + var src strings.Builder + src.WriteString("TEXT packedsignextendmoveforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMOVSXBW", "PMOVSXBD", "PMOVSXBQ", "PMOVSXWD", "PMOVSXWQ", "PMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2\n", op) + fmt.Fprintf(&src, "\t%s X15, X14\n", op) + } + for _, op := range []string{"VPMOVSXBW", "VPMOVSXWD", "VPMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2\n", op) + fmt.Fprintf(&src, "\t%s X0, Y1\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y2\n", op) + fmt.Fprintf(&src, "\t%s Y0, Z1\n", op) + fmt.Fprintf(&src, "\t%s (AX), Z2\n", op) + fmt.Fprintf(&src, "\t%s X20, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, Y21\n", op) + fmt.Fprintf(&src, "\t%s Y20, Z21\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, X21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, K3, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K4, Y21\n", op) + fmt.Fprintf(&src, "\t%s Y20, K5, Z21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K6, Z21\n", op) + } + for _, op := range []string{"VPMOVSXBD", "VPMOVSXBQ", "VPMOVSXWQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s X0, %s1\n", op, width) + fmt.Fprintf(&src, "\t%s (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s X20, %s21\n", op, width) + fmt.Fprintf(&src, "\t%s X20, K1, %s21\n", op, width) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, %s21\n", op, width) + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedsignextendmoveforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-sign-extend-move-forms.ll", "packed-sign-extend-move-forms.o", ll) +} + +func TestTranslate386PackedSignExtendMoveCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedsignextendmoveforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PMOVSXBW", "PMOVSXBD", "PMOVSXBQ", "PMOVSXWD", "PMOVSXWQ", "PMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X0, X7\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + for _, op := range []string{"VPMOVSXBW", "VPMOVSXWD", "VPMOVSXDQ"} { + fmt.Fprintf(&src, "\t%s X20, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, Y21\n", op) + fmt.Fprintf(&src, "\t%s Y20, Z7\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, Z7\n", op) + } + for _, op := range []string{"VPMOVSXBD", "VPMOVSXBQ", "VPMOVSXWQ"} { + fmt.Fprintf(&src, "\t%s X20, X21\n", op) + fmt.Fprintf(&src, "\t%s X20, Y21\n", op) + fmt.Fprintf(&src, "\t%s X20, Z7\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, Z7\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedsignextendmoveforms386": {Name: "packedsignextendmoveforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-sign-extend-move-386-forms.ll", "packed-sign-extend-move-386-forms.o", ll) +} + +func TestTranslateAMD64PackedSignExtendMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMOVSXBW X16, X1", + "PMOVSXBD X1, X16", + "PMOVSXBQ Y1, X2", + "PMOVSXWD X1, Y2", + "PMOVSXWQ X1, (AX)", + "PMOVSXDQ.Z X1, X2", + "PMOVSXBW X1, K1, X2", + "VPMOVSXBW Y1, Y2", + "VPMOVSXWD X1, Z2", + "VPMOVSXDQ Z1, Z2", + "VPMOVSXBD Y1, Z2", + "VPMOVSXBQ Y1, Y2", + "VPMOVSXWQ Z1, X2", + "VPMOVSXBW X1, (AX)", + "VPMOVSXBD (AX), (BX)", + "VPMOVSXBQ X1, K0, Y2", + "VPMOVSXWD.Z X1, Y2", + "VPMOVSXDQ.BCST (AX), Y2", + "VPMOVSXWQ.SAE X1, Z2", + "VPMOVSXBW.RN_SAE X1, Y2", + "VPMOVSXBD X1", + "VPMOVSXBQ X1, K1, Y2, Y3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedsignextendmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsignextendmove": {Name: "invalidpackedsignextendmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's signed-extension move tables", instruction) + } + }) + } +} + +func TestTranslate386PackedSignExtendMoveRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PMOVSXBW X8, X1", + "PMOVSXBD X1, X8", + "PMOVSXBQ X15, X7", + "VPMOVSXBW Y1, Z8", + "VPMOVSXWD (AX), Z8", + "VPMOVSXDQ Y20, K1, Z8", + "VPMOVSXBD X20, Z8", + "VPMOVSXBQ.Z (AX), K1, Z8", + "VPMOVSXWQ X20, K1, Z8", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedsignextendmove386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedsignextendmove386": {Name: "invalidpackedsignextendmove386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64MOVDFromXRegister(t *testing.T) { + src := ` +TEXT movdfromx(SB),NOSPLIT,$0-0 + PXOR X12, X12 + MOVD X12, DX + RET +` + translateAMD64EcosystemCase(t, src, "movdfromx") +} + +func TestTranslateAMD64MOVDAliasCompleteGo127Forms(t *testing.T) { + // cmd/asm aliases MOVD to AMOVQ on x86. These cover every ymovq operand + // class that the alias can select in 64-bit mode: GP/immediate/memory, + // MMX/memory/GP/XMM, and XMM/memory/GP/XMM. + src := ` +TEXT movdaliasforms(SB),NOSPLIT,$0-0 + MOVD AX, BX + MOVD $1, AX + MOVD AX, 8(BX) + MOVD 8(BX), AX + MOVD (BX), M0 + MOVD M0, 8(BX) + MOVD AX, M1 + MOVD M1, AX + MOVD X0, M2 + MOVD M2, M3 + MOVD AX, X1 + MOVD 24(SP), X9 + MOVD X1, AX + MOVD X9, 32(SP) + MOVD X1, X2 + RET +` + ll := translateAMD64EcosystemCase(t, src, "movdaliasforms") + if !strings.Contains(ll, "load i64") || !strings.Contains(ll, "store i64") { + t.Fatalf("MOVD alias did not preserve MOVQ-width memory semantics:\n%s", ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "movd-alias-forms.ll", "movd-alias-forms.o", ll) +} + +func TestTranslateAMD64MOVDAliasRejectsFormsOutsideGo127Ymovq(t *testing.T) { + for _, instruction := range []string{ + "MOVD $1, X0", + "MOVD M0, X0", + "MOVD X0, X1, X2", + "MOVD.Z X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badmovd(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badmovd": {Name: "badmovd", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's MOVD/ymovq forms", instruction) + } + }) + } +} + +func TestTranslateAMD64LegacyPrefetchFamilyCompleteGo127Forms(t *testing.T) { + src := ` +TEXT prefetchforms(SB),NOSPLIT,$0-0 + PREFETCHNTA (AX) + PREFETCHT0 8(BX) + PREFETCHT1 source(SB) + PREFETCHT2 16(SP) + RET +` + ll := translateAMD64EcosystemCase(t, src, "prefetchforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "prefetch-forms.ll", "prefetch-forms.o", ll) +} + +func TestTranslateAMD64LegacyPrefetchFamilyRejectsNonMemoryForms(t *testing.T) { + for _, instruction := range []string{ + "PREFETCHNTA", + "PREFETCHT0 AX", + "PREFETCHT1 $1", + "PREFETCHT2 (AX), (BX)", + "PREFETCHT0.Z (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badprefetch(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badprefetch": {Name: "badprefetch", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yprefetch table", instruction) + } + }) + } +} + +func TestTranslateAMD64ArithmeticDisplacementExpression(t *testing.T) { + ll := translateAMD64EcosystemCase(t, ` +TEXT arithmeticdisplacement(SB),NOSPLIT,$32-0 + ADDQ 0+(1*16)(BP), R10 + MOVQ (-16+8)(SP), AX + RET +`, "arithmeticdisplacement") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "arithmetic-displacement.ll", "arithmetic-displacement.o", ll) +} + +func TestTranslateAMD64GoTLSPseudoRegisterForms(t *testing.T) { + const source = ` +TEXT ·tlsforms(SB),4,$0-0 + MOVQ TLS, CX + MOVQ 0(CX)(TLS*1), BX + MOVQ 8(TLS), DX + RET + ` + x86ScalarCompareTLSGoObject(t, source, "amd64", "linux", false) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func tlsforms()", "amd64", "x86_64-unknown-linux-gnu", api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("%s: TLS source without native contract returned error=%v, IR=%s", api, err, ir) + } + } +} + +func TestTranslateAMD64LegacyIntegerToFloatConversionFamilyCompleteGo127Forms(t *testing.T) { + ll := translateAMD64EcosystemCase(t, ` +TEXT integerfloatconversions(SB),NOSPLIT,$0-0 + CVTSL2SS AX, X0 + CVTSL2SS 4(BX), X1 + CVTSL2SD CX, X2 + CVTSL2SD 8(BX), X3 + CVTSQ2SS SI, X4 + CVTSQ2SS 16(BX), X5 + CVTSQ2SD DI, X6 + CVTSQ2SD 24(BX), X7 + CVTPL2PS X0, X1 + CVTPL2PS 32(BX), X2 + CVTPL2PD X4, X5 + CVTPL2PD 48(BX), X6 + RET +`, "integerfloatconversions") + for _, want := range []string{"@llvm.experimental.constrained.sitofp.f32.i32", "@llvm.experimental.constrained.sitofp.f64.i64", "cvtdq2ps", "cvtdq2pd"} { + if !strings.Contains(ll, want) { + t.Fatalf("missing %q in integer-to-float conversion IR:\n%s", want, ll) + } + } + if strings.Count(ll, "asm sideeffect") != 4 || strings.Contains(ll, " = sitofp <") { + t.Fatal("packed native conversions lost whole-opcode side effects") + } + ll = x86SignedIntegerFloatOptimize(t, ll) + if strings.Count(ll, "asm sideeffect") != 4 { + t.Fatal("O2 dropped a native packed conversion") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "integer-float-conversions.ll", "integer-float-conversions.o", ll) +} + +func TestTranslateAMD64LegacyIntegerToFloatConversionFamilyRejectsNonGoForms(t *testing.T) { + for _, instruction := range []string{ + "CVTSL2SS $1, X0", + "CVTSQ2SS X0, X1", + "CVTSQ2SD AX, Y0", + "CVTPL2PS AX, X0", + "CVTPL2PD X0, Y0", + "CVTPL2PS.Z X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badconversion(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badconversion": {Name: "badconversion", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's legacy integer-to-float tables", instruction) + } + }) + } +} + +func TestTranslateAMD64ScalarFloatBroadcastFamilyCompleteGo127Forms(t *testing.T) { + ll := translateAMD64EcosystemCase(t, ` +TEXT scalarfloatbroadcasts(SB),NOSPLIT,$0-0 + VBROADCASTSS X0, X1 + VBROADCASTSS 4(AX), Y1 + VBROADCASTSS X16, Z17 + VBROADCASTSS X16, K1, X17 + VBROADCASTSS.Z 8(AX), K2, Y18 + VBROADCASTSS 12(AX), K3, Z19 + VBROADCASTSD X0, Y1 + VBROADCASTSD 8(AX), Z1 + VBROADCASTSD X16, K1, Y17 + VBROADCASTSD.Z 16(AX), K2, Z18 + RET +`, "scalarfloatbroadcasts") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-float-broadcasts.ll", "scalar-float-broadcasts.o", ll) +} + +func TestTranslateAMD64ScalarFloatBroadcastFamilyRejectsNonGoForms(t *testing.T) { + for _, instruction := range []string{ + "VBROADCASTSS AX, X0", + "VBROADCASTSD X0, X1", + "VBROADCASTSS.Z X0, Y1", + "VBROADCASTSD X0, K0, Y1", + "VBROADCASTSS.BCST 4(AX), Y1", + "VBROADCASTSD X0, K1, K2, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badbroadcast(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badbroadcast": {Name: "badbroadcast", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar broadcast tables", instruction) + } + }) + } +} + +func TestTranslateAMD64DwordToQwordMultiplyFamilyCompleteGo127Forms(t *testing.T) { + ll := translateAMD64EcosystemCase(t, ` +TEXT dwordqwordmultiply(SB),NOSPLIT,$0-0 + PMULDQ X0, X1 + PMULDQ 16(AX), X2 + PMULULQ M0, M1 + PMULULQ 24(AX), M2 + PMULULQ X0, X1 + PMULULQ 32(AX), X2 + VPMULDQ X0, X1, X2 + VPMULDQ Y0, Y1, Y2 + VPMULDQ Z0, Z1, Z2 + VPMULDQ Z0, Z1, K1, Z2 + VPMULDQ.Z Z0, Z1, K2, Z2 + VPMULDQ.BCST 40(AX), Z1, Z2 + VPMULDQ.BCST.Z 48(AX), Z1, K3, Z2 + VPMULUDQ X0, X1, X2 + VPMULUDQ Y0, Y1, Y2 + VPMULUDQ Z0, Z1, Z2 + VPMULUDQ Z0, Z1, K1, Z2 + VPMULUDQ.Z Z0, Z1, K2, Z2 + VPMULUDQ.BCST 56(AX), Z1, Z2 + VPMULUDQ.BCST.Z 64(AX), Z1, K3, Z2 + RET +`, "dwordqwordmultiply") + for _, want := range []string{"sext <", "zext <", "mul <2 x i64>", "mul <4 x i64>", "mul <8 x i64>"} { + if !strings.Contains(ll, want) { + t.Fatalf("missing %q in dword-to-qword multiply IR:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "dword-qword-multiply.ll", "dword-qword-multiply.o", ll) +} + +func TestTranslateAMD64DwordToQwordMultiplyFamilyRejectsNonGoForms(t *testing.T) { + for _, instruction := range []string{ + "PMULDQ M0, M1", + "PMULULQ Y0, Y1", + "VPMULUDQ X0, Y1, X2", + "VPMULDQ Z0, Z1, K0, Z2", + "VPMULUDQ.Z Z0, Z1, Z2", + "VPMULDQ.BCST Z0, Z1, Z2", + "VPMULUDQ.Z.BCST 8(AX), Z1, K1, Z2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT badmultiply(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"badmultiply": {Name: "badmultiply", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's dword-to-qword multiply tables", instruction) + } + }) + } +} + +func TestTranslate386NewlyDiscoveredX86FamiliesCompleteGo127Forms(t *testing.T) { + // Keep the 386 matrix explicit: x86 opcode names are shared with amd64, but + // cmd/asm's operand tables and available registers differ in 32-bit mode. + src := ` +TEXT discoveredx86forms386(SB),NOSPLIT,$0-0 + MOVD (AX), X0 + MOVD X0, 8(AX) + MOVD X0, X1 + MOVD 16(AX), M0 + MOVD M0, 24(AX) + MOVD X0, M1 + CVTSL2SS AX, X0 + CVTSL2SS 4(BX), X1 + CVTSL2SD CX, X2 + CVTSL2SD 8(BX), X3 + CVTPL2PS X0, X1 + CVTPL2PS 16(BX), X2 + CVTPL2PD X4, X5 + CVTPL2PD 24(BX), X6 + VBROADCASTSS X0, X1 + VBROADCASTSS 4(AX), Y1 + VBROADCASTSS 8(AX), Z1 + VBROADCASTSD X0, Y2 + VBROADCASTSD 16(AX), Z2 + PMULDQ X0, X1 + PMULDQ 16(AX), X2 + PMULULQ X0, X1 + PMULULQ 24(AX), X2 + VPMULDQ X0, X1, X2 + VPMULDQ Y0, Y1, Y2 + VPMULDQ Z0, Z1, Z2 + VPMULUDQ X0, X1, X2 + VPMULUDQ Y0, Y1, Y2 + VPMULUDQ Z0, Z1, Z2 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "discoveredx86forms386": {Name: "discoveredx86forms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "discovered-x86-386-forms.ll", "discovered-x86-386-forms.o", ll) +} + +func TestTranslate386NewlyDiscoveredX86FamiliesRejectArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "MOVD AX, BX", + "MOVD M0, M1", + "MOVD M0, X0", + "CVTSQ2SS AX, X0", + "CVTSQ2SD 8(AX), X0", + "PMULULQ M0, M1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad386discovered(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "bad386discovered": {Name: "bad386discovered", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + }) + } +} + +func TestTranslateAMD64MOVQFromXRegisterToFrameResult(t *testing.T) { + src := ` +TEXT vectorret(SB),NOSPLIT,$0-8 + PXOR X0, X0 + MOVQ X0, vectorGlobal<>(SB) + MOVQ X0, ret+0(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"vectorret": { + Name: "vectorret", + Ret: I64, + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: I64, + Index: 0, + Field: -1, + }}}, + }}, + }); err != nil { + t.Fatal(err) + } +} + +func TestTranslateAMD64X87Arithmetic(t *testing.T) { + src := ` +TEXT x87amd64(SB),NOSPLIT,$0-0 + FLDZ + FLD1 + FADDD F1, F0 + FSUBD F1, F0 + FLDCW 0(SP) + RET +` + translateAMD64EcosystemCase(t, src, "x87amd64") +} + +func TestTranslateAMD64FuzzerBitTestAndVectorSubtract(t *testing.T) { + src := ` +TEXT fuzzops(SB),NOSPLIT,$0-0 + BTL $16, CX + JCS done + VPSUBB Y0, Y1, Y2 +done: + RET +` + translateAMD64EcosystemCase(t, src, "fuzzops") +} + +func TestTranslateX86DivisionAndFrameArithmetic(t *testing.T) { + amd64Source := ` +TEXT divmod(SB),NOSPLIT,$0-32 + MOVQ $0, DX + MOVQ a+0(FP), AX + DIVQ b+8(FP) + MOVQ AX, quo+16(FP) + MOVQ DX, rem+24(FP) + RET +` + translateX86FrameCase(t, amd64Source, "amd64", "x86_64-unknown-linux-gnu", "divmod") + + x86Source := ` +TEXT mulparts(SB),NOSPLIT,$0-32 + MOVL a+0(FP), AX + MULL b+8(FP) + MOVL AX, hi+16(FP) + ADDL DX, hi+16(FP) + RET +` + translateX86FrameCase(t, x86Source, "386", "i386-unknown-linux-gnu", "mulparts") +} + +func TestTranslateAMD64LegacyBranchHintAndSignedDivision(t *testing.T) { + src := ` +TEXT signeddivmod(SB),NOSPLIT,$0-32 + MOVQ a+0(FP), AX + MOVQ b+8(FP), CX + CMPQ CX, $-1 + JEQ $1, special + CQO + IDIVQ CX + JMP done +special: + NEGQ AX + MOVQ $0, DX +done: + MOVQ AX, quo+16(FP) + MOVQ DX, rem+24(FP) + RET +` + translateX86FrameCase(t, src, "amd64", "x86_64-unknown-linux-gnu", "signeddivmod") +} + +func TestTranslateAMD64LoopBranch(t *testing.T) { + src := ` +TEXT countedloop(SB),NOSPLIT,$0-0 + MOVQ $2, CX +loop: + LOOP loop + RET +` + translateAMD64EcosystemCase(t, src, "countedloop") +} + +func TestTranslateAMD64RawDataDirectives(t *testing.T) { + src := ` +TEXT rawencoding(SB),NOSPLIT,$0-0 + LONG $0xe77da1c4 + QUAD $0x4109048d47d18941 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawencoding": {Name: "rawencoding", Ret: Void}}}); err == nil || !strings.Contains(err.Error(), "decode raw x86 directive group") { + t.Fatalf("reachable opaque data directives must fail closed, got %v", err) + } +} + +func TestTranslateAMD64HighQuadwordInterleave(t *testing.T) { + src := ` +TEXT highquadword(SB),NOSPLIT,$0-0 + PUNPCKHQDQ X1, X0 + VPUNPCKHQDQ Y0, Y1, Y2 + RET +` + translateAMD64EcosystemCase(t, src, "highquadword") +} + +func TestTranslateAMD64AdditionalAVXUnpackAndBroadcastFamilies(t *testing.T) { + src := ` +TEXT avxunpackbroadcast(SB),NOSPLIT,$0-0 + VPUNPCKLQDQ X0, X1, X2 + VPUNPCKLQDQ Y0, Y1, Y2 + VPUNPCKLDQ X0, X1, X2 + VPUNPCKLDQ Y0, Y1, Y2 + VPUNPCKHDQ X0, X1, X2 + VPUNPCKHDQ Y0, Y1, Y2 + VPBROADCASTD (AX), X3 + VPBROADCASTD (AX), Y3 + VPSHUFD $0x4e, (AX), X4 + VPSHUFD $0x4e, X0, X4 + VPSHUFD $0x4e, (AX), Z4 + VPSLLD $12, X5, X13 + VPSRLD $7, (AX), X13 + VPSLLD $12, Y5, Y13 + VPSLLD $12, Z5, Z13 + RET +` + translateAMD64EcosystemCase(t, src, "avxunpackbroadcast") +} + +func TestTranslateAMD64DwordShuffleCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT dwordshuffleforms(SB),NOSPLIT,$0-0 + PSHUFD $1, X1, X2 + PSHUFD $1, (AX), X2 + PSHUFL $1, X1, X2 + VPSHUFD $1, X1, X2 + VPSHUFD $1, (AX), X2 + VPSHUFD $1, Y1, Y2 + VPSHUFD $1, (AX), Y2 + VPSHUFD $1, Z1, Z2 + VPSHUFD $1, (AX), Z2 + VPSHUFD $1, X1, K1, X2 + VPSHUFD.Z $1, (AX), K1, X2 + VPSHUFD $1, Y1, K1, Y2 + VPSHUFD.Z $1, (AX), K1, Y2 + VPSHUFD $1, Z1, K1, Z2 + VPSHUFD.Z $1, (AX), K1, Z2 + RET +` + translateAMD64EcosystemCase(t, src, "dwordshuffleforms") +} + +func TestTranslateAMD64CanonicalPackedByteShiftAliases(t *testing.T) { + src := ` +TEXT packedbyteshifts(SB),NOSPLIT,$0-0 + PSLLO $4, X0 + PSRLO $4, X0 + RET +` + translateAMD64EcosystemCase(t, src, "packedbyteshifts") +} + +func TestTranslateAMD64PackedUniformShiftCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's x86 assembler assigns all of these instructions the same + // _yvpslld operand-format table. Keep this test in lockstep with that + // family instead of adding only the form first observed in a dependency. + for _, op := range []string{ + "VPSLLD", "VPSLLQ", "VPSLLW", + "VPSRAD", "VPSRAW", + "VPSRLD", "VPSRLQ", "VPSRLW", + } { + var src strings.Builder + fmt.Fprintf(&src, "TEXT uniformshift%s(SB),NOSPLIT,$0-0\n", op) + for _, operands := range []string{ + "$3, X1, X2", "$-1, X1, X2", "$3, (AX), X2", + "$3, Y1, Y2", "$-1, Y1, Y2", "$3, (AX), Y2", + "$3, Z1, Z2", "$3, (AX), Z2", + "X0, X1, X2", "(AX), X1, X2", + "X0, Y1, Y2", "(AX), Y1, Y2", + "X0, Z1, Z2", "(AX), Z1, Z2", + "$3, X1, K1, X2", "$3, (AX), K1, X2", + "$3, Y1, K1, Y2", "$3, (AX), K1, Y2", + "$3, Z1, K1, Z2", "$3, (AX), K1, Z2", + "X0, X1, K1, X2", "(AX), X1, K1, X2", + "X0, Y1, K1, Y2", "(AX), Y1, K1, Y2", + "X0, Z1, K1, Z2", "(AX), Z1, K1, Z2", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, operands) + } + for _, operands := range []string{ + "$3, X1, K1, X2", "$3, (AX), K1, X2", + "$3, Y1, K1, Y2", "$3, (AX), K1, Y2", + "$3, Z1, K1, Z2", "$3, (AX), K1, Z2", + "X0, X1, K1, X2", "(AX), X1, K1, X2", + "X0, Y1, K1, Y2", "(AX), Y1, K1, Y2", + "X0, Z1, K1, Z2", "(AX), Z1, K1, Z2", + } { + fmt.Fprintf(&src, "\t%s.Z %s\n", op, operands) + } + src.WriteString("\tRET\n") + translateAMD64EcosystemCase(t, src.String(), "uniformshift"+op) + } +} + +func TestTranslateAMD64PackedIntegerCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns all eight equality/greater-than variants the shared + // _yvpcmpeqb table. VEX writes X/Y vectors; EVEX compares X/Y/Z lanes into + // K registers, optionally masked by K1-K7. D/Q also permit memory broadcast. + ops := []string{ + "VPCMPEQB", "VPCMPEQW", "VPCMPEQD", "VPCMPEQQ", + "VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ", + } + forms := []string{ + "X1, X2, X3", "(AX), X2, X3", + "Y1, Y2, Y3", "(AX), Y2, Y3", + "X1, X2, K3", "(AX), X2, K3", + "X1, X2, K1, K3", "(AX), X2, K1, K3", + "Y1, Y2, K3", "(AX), Y2, K3", + "Y1, Y2, K1, K3", "(AX), Y2, K1, K3", + "Z1, Z2, K3", "(AX), Z2, K3", + "Z1, Z2, K1, K3", "(AX), Z2, K1, K3", + } + var src strings.Builder + src.WriteString("TEXT packedintegercompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range ops { + for _, form := range forms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + for _, form := range []string{ + "(AX), X2, K3", "(AX), X2, K1, K3", + "(AX), Y2, K3", "(AX), Y2, K1, K3", + "(AX), Z2, K3", "(AX), Z2, K1, K3", + } { + fmt.Fprintf(&src, "\t%s.BCST %s\n", op, form) + } + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedintegercompareforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-integer-compare-forms.ll", "packed-integer-compare-forms.o", ll) +} + +func TestTranslateAMD64PackedIntegerCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPCMPEQB Z1, Z2, Z3", + "VPCMPEQW X1, Y2, Y3", + "VPCMPEQD X1, X2, K0, K3", + "VPCMPEQQ.Z X1, X2, K1, K3", + "VPCMPGTB.BCST (AX), X2, K3", + "VPCMPGTW.SAE X1, X2, K3", + "VPCMPGTD.BCST X1, X2, K3", + "VPCMPGTQ.BCST (AX), X2, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegercompare(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedintegercompare": {Name: "invalidpackedintegercompare", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvpcmpeqb table", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedAndNotCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 defines the whole AND-NOT family across three operand tables: + // PANDN's ymm table (MMX/XMM), VPANDN's _yvaddsubpd table (X/Y VEX), + // and VPANDND/Q's _yvblendmpd table (X/Y/Z EVEX with optional K1-K7 + // masking and zeroing). Only the D/Q EVEX variants permit memory broadcast. + src := ` +TEXT packedandnotforms(SB),NOSPLIT,$0-0 + PANDN M1, M2 + PANDN (AX), M2 + PANDN X1, X2 + PANDN (AX), X2 + VPANDN X1, X2, X3 + VPANDN (AX), X2, X3 + VPANDN Y1, Y2, Y3 + VPANDN (AX), Y2, Y3 + VPANDND X1, X2, X3 + VPANDND (AX), X2, X3 + VPANDND X1, X2, K1, X3 + VPANDND.Z (AX), X2, K1, X3 + VPANDND Y1, Y2, Y3 + VPANDND (AX), Y2, Y3 + VPANDND Y1, Y2, K1, Y3 + VPANDND.Z (AX), Y2, K1, Y3 + VPANDND Z1, Z2, Z3 + VPANDND (AX), Z2, Z3 + VPANDND Z1, Z2, K1, Z3 + VPANDND.Z (AX), Z2, K1, Z3 + VPANDND.BCST (AX), X2, X3 + VPANDND.BCST (AX), X2, K1, X3 + VPANDND.BCST.Z (AX), X2, K1, X3 + VPANDND.BCST (AX), Y2, Y3 + VPANDND.BCST (AX), Y2, K1, Y3 + VPANDND.BCST.Z (AX), Y2, K1, Y3 + VPANDND.BCST (AX), Z2, Z3 + VPANDND.BCST (AX), Z2, K1, Z3 + VPANDND.BCST.Z (AX), Z2, K1, Z3 + VPANDNQ X1, X2, X3 + VPANDNQ (AX), X2, X3 + VPANDNQ X1, X2, K1, X3 + VPANDNQ.Z (AX), X2, K1, X3 + VPANDNQ Y1, Y2, Y3 + VPANDNQ (AX), Y2, Y3 + VPANDNQ Y1, Y2, K1, Y3 + VPANDNQ.Z (AX), Y2, K1, Y3 + VPANDNQ Z1, Z2, Z3 + VPANDNQ (AX), Z2, Z3 + VPANDNQ Z1, Z2, K1, Z3 + VPANDNQ.Z (AX), Z2, K1, Z3 + VPANDNQ.BCST (AX), X2, X3 + VPANDNQ.BCST (AX), X2, K1, X3 + VPANDNQ.BCST.Z (AX), X2, K1, X3 + VPANDNQ.BCST (AX), Y2, Y3 + VPANDNQ.BCST (AX), Y2, K1, Y3 + VPANDNQ.BCST.Z (AX), Y2, K1, Y3 + VPANDNQ.BCST (AX), Z2, Z3 + VPANDNQ.BCST (AX), Z2, K1, Z3 + VPANDNQ.BCST.Z (AX), Z2, K1, Z3 + RET +` + ll := translateAMD64EcosystemCase(t, src, "packedandnotforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-and-not-forms.ll", "packed-and-not-forms.o", ll) +} + +func TestTranslateAMD64PackedAndNotRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PANDN Y1, Y2", + "VPANDN Z1, Z2, Z3", + "VPANDN.X Z1, Z2, Z3", + "VPANDND X1, Y2, Y3", + "VPANDND X1, X2, K0, X3", + "VPANDND.Z X1, X2, X3", + "VPANDND.SAE X1, X2, K1, X3", + "VPANDND.BCST X1, X2, X3", + "VPANDNQ.BCST (AX), X2, Y3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedandnot(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedandnot": {Name: "invalidpackedandnot", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PANDN/VPANDN operand tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedScalarBroadcastCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns VPBROADCASTB/W/D/Q the same _yvpbroadcastb table. + // VEX broadcasts X/m to X/Y; EVEX adds GP and X/m sources, X/Y/Z + // destinations, K1-K7 merge masking, and masked .Z zeroing. + forms := []string{ + "X1, X2", "(AX), X2", "table<>(SB), X2", "0, X2", + "X1, Y2", "(AX), Y2", "table<>(SB), Y2", "0x1234, Y2", + "AX, X2", "AX, Y2", "AX, Z2", + "X1, Z2", "(AX), Z2", + "AX, K1, X2", "X1, K1, X2", "(AX), K1, X2", + "AX, K1, Y2", "X1, K1, Y2", "(AX), K1, Y2", + "AX, K1, Z2", "X1, K1, Z2", "(AX), K1, Z2", + } + zeroForms := []string{ + "AX, K1, X2", "X1, K1, X2", "(AX), K1, X2", + "AX, K1, Y2", "X1, K1, Y2", "(AX), K1, Y2", + "AX, K1, Z2", "X1, K1, Z2", "(AX), K1, Z2", + } + var src strings.Builder + src.WriteString("TEXT packedscalarbroadcastforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + for _, form := range forms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + for _, form := range zeroForms { + fmt.Fprintf(&src, "\t%s.Z %s\n", op, form) + } + } + src.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", strings.Replace(src.String(), ",NOSPLIT", "", 1), true) + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedscalarbroadcastforms": {Name: "packedscalarbroadcastforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "packed-scalar-broadcast-forms-"+target.name+".ll", "packed-scalar-broadcast-forms-"+target.name+".o", ll) + }) + } +} + +func TestTranslateAMD64PackedScalarBroadcastRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPBROADCASTB $1, X2", + "VPBROADCASTW Y1, Y2", + "VPBROADCASTD X1, K0, X2", + "VPBROADCASTQ X1, AX", + "VPBROADCASTB.Z X1, X2", + "VPBROADCASTW.BCST (AX), X2", + "VPBROADCASTD.SAE X1, K1, X2", + "VPBROADCASTQ X1, K1, Y2, Z3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedscalarbroadcast(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedscalarbroadcast": {Name: "invalidpackedscalarbroadcast", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's _yvpbroadcastb table", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedIntegerAddCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy ymm table gives these instructions MMX and XMM forms. + for _, op := range []string{"PADDB", "PADDL", "PADDSB", "PADDSW", "PADDUSB", "PADDUSW", "PADDW"} { + src := fmt.Sprintf(` +TEXT legacy%sforms(SB),NOSPLIT,$0-0 + %s M1, M2 + %s (AX), M2 + %s X1, X2 + %s (AX), X2 + RET +`, op, op, op, op, op) + translateAMD64EcosystemCase(t, src, "legacy"+op+"forms") + } + // PADDQ is deliberately narrower in Go's legacy yxm table: XMM only. + translateAMD64EcosystemCase(t, ` +TEXT legacyPADDQforms(SB),NOSPLIT,$0-0 + PADDQ X1, X2 + PADDQ (AX), X2 + RET +`, "legacyPADDQforms") + + // Every AVX variant shares _yvandnpd: X/Y/Z register-or-memory inputs, + // optional K1-K7 merge/zero masks. D/Q additionally enable broadcast. + commonForms := []string{ + "X1, X2, X3", "(AX), X2, X3", + "X1, X2, K1, X3", "(AX), X2, K1, X3", + "Y1, Y2, Y3", "(AX), Y2, Y3", + "Y1, Y2, K1, Y3", "(AX), Y2, K1, Y3", + "Z1, Z2, Z3", "(AX), Z2, Z3", + "Z1, Z2, K1, Z3", "(AX), Z2, K1, Z3", + } + maskedForms := []string{ + "X1, X2, K1, X3", "(AX), X2, K1, X3", + "Y1, Y2, K1, Y3", "(AX), Y2, K1, Y3", + "Z1, Z2, K1, Z3", "(AX), Z2, K1, Z3", + } + var src strings.Builder + src.WriteString("TEXT packedintegeraddforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "VPADDB", "VPADDW", "VPADDD", "VPADDQ", + "VPADDSB", "VPADDSW", "VPADDUSB", "VPADDUSW", + } { + for _, form := range commonForms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + for _, form := range maskedForms { + fmt.Fprintf(&src, "\t%s.Z %s\n", op, form) + } + if op == "VPADDD" || op == "VPADDQ" { + for _, form := range []string{ + "(AX), X2, X3", "(AX), X2, K1, X3", + "(AX), Y2, Y3", "(AX), Y2, K1, Y3", + "(AX), Z2, Z3", "(AX), Z2, K1, Z3", + } { + fmt.Fprintf(&src, "\t%s.BCST %s\n", op, form) + } + for _, form := range []string{ + "(AX), X2, K1, X3", "(AX), Y2, K1, Y3", "(AX), Z2, K1, Z3", + } { + fmt.Fprintf(&src, "\t%s.BCST.Z %s\n", op, form) + } + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedintegeraddforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-integer-add-forms.ll", "packed-integer-add-forms.o", ll) +} + +func TestTranslateAMD64PackedIntegerAddRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PADDQ M1, M2", + "PADDB Y1, Y2", + "VPADDB X1, Y2, Y3", + "VPADDW X1, X2, K0, X3", + "VPADDD.Z X1, X2, X3", + "VPADDQ.BCST X1, X2, X3", + "VPADDSB.BCST (AX), X2, X3", + "VPADDSW.SAE X1, X2, K1, X3", + "VPADDUSB X1, X2, K1, X3, X4", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegeradd(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedintegeradd": {Name: "invalidpackedintegeradd", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-add tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedIntegerSubtractCompleteGoAssemblerForms(t *testing.T) { + // All eight legacy instructions use yxm: X/m128, X. Unlike the packed-add + // family, none of these legacy subtract spellings has an MMX row. + var src strings.Builder + src.WriteString("TEXT packedintegersubtractforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSUBB", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", "PSUBW"} { + fmt.Fprintf(&src, "\t%s X1, X15\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + + // All eight V spellings share _yvandnpd. Exercise every VEX/EVEX width, + // both register and memory members of each rm class, high EVEX registers, + // K merge/zero masks, and the D/Q-only broadcast rows. + commonForms := []string{ + "X1, X2, X3", "(AX), X2, X3", + "Y1, Y2, Y3", "(AX), Y2, Y3", + "Z1, Z2, Z3", "(AX), Z2, Z3", + "X20, X21, X22", "Y20, Y21, Y22", "Z20, Z21, Z22", + } + maskedForms := []string{ + "X20, X21, K1, X22", "(AX), X21, K1, X22", + "Y20, Y21, K1, Y22", "(AX), Y21, K1, Y22", + "Z20, Z21, K1, Z22", "(AX), Z21, K1, Z22", + } + for _, op := range []string{ + "VPSUBB", "VPSUBD", "VPSUBQ", "VPSUBSB", + "VPSUBSW", "VPSUBUSB", "VPSUBUSW", "VPSUBW", + } { + for _, form := range commonForms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + for _, form := range maskedForms { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + fmt.Fprintf(&src, "\t%s.Z %s\n", op, form) + } + if op == "VPSUBD" || op == "VPSUBQ" { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K1, %s22\n", op, width, width) + } + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "packedintegersubtractforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-integer-subtract-forms.ll", "packed-integer-subtract-forms.o", ll) +} + +func TestTranslate386PackedIntegerSubtractCompleteGoAssemblerForms(t *testing.T) { + // In 386 mode only the low eight registers have proved physical bindings. + // Go-accepted high-bank EVEX spellings require separate Context tests: + // frontend acceptance alone is not a mode32 physical register identity. + var src strings.Builder + src.WriteString("TEXT packedintegersubtractforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSUBB", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", "PSUBW"} { + fmt.Fprintf(&src, "\t%s X1, X7\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + for _, op := range []string{ + "VPSUBB", "VPSUBD", "VPSUBQ", "VPSUBSB", + "VPSUBSW", "VPSUBUSB", "VPSUBUSW", "VPSUBW", + } { + for _, form := range []string{ + "X1, X2, X7", "(AX), X2, X7", + "Y1, Y2, Y7", "(AX), Y2, Y7", + "Z1, Z2, Z7", "(AX), Z2, Z7", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + if op == "VPSUBD" || op == "VPSUBQ" { + fmt.Fprintf(&src, "\t%s.BCST (AX), X2, X7\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Y2, Y7\n", op) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z2, Z7\n", op) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedintegersubtractforms386": {Name: "packedintegersubtractforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-integer-subtract-386-forms.ll", "packed-integer-subtract-386-forms.o", ll) +} + +func TestTranslateAMD64PackedIntegerSubtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PSUBB M1, M2", + "PSUBQ Y1, Y2", + "VPSUBB X1, Y2, Y3", + "VPSUBW X1, X2, K0, X3", + "VPSUBD.Z X1, X2, X3", + "VPSUBQ.BCST X1, X2, X3", + "VPSUBSB.BCST (AX), X2, X3", + "VPSUBSW.SAE X1, X2, K1, X3", + "VPSUBUSB X1, X2, K1, X3, X4", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegersubtract(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedintegersubtract": {Name: "invalidpackedintegersubtract", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed-subtract tables", instruction) + } + }) + } +} + +func TestTranslate386PackedIntegerSubtractRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "PSUBB X1, X8", + "VPSUBQ Z8, Z2, Z3", + "VPSUBD X1, X2, K1, X3", + "VPSUBQ.Z X1, X2, K1, X3", + "VPSUBD.BCST.Z (AX), X2, K1, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedintegersubtract386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpackedintegersubtract386": {Name: "invalidpackedintegersubtract386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedSumAbsoluteDifferencesCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT packedsadforms(SB),NOSPLIT,$0-0 + PSADBW X1, X2 + PSADBW (AX), X2 + VPSADBW X1, X2, X3 + VPSADBW (AX), X2, X3 + VPSADBW Y1, Y2, Y3 + VPSADBW (AX), Y2, Y3 + VPSADBW Z1, Z2, Z3 + VPSADBW (AX), Z2, Z3 + RET +` + ll := translateAMD64EcosystemCase(t, src, "packedsadforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-sad-forms.ll", "packed-sad-forms.o", ll) +} + +func TestTranslateAMD64PackedSumAbsoluteDifferencesRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PSADBW Y1, Y2", + "VPSADBW X1, Y2, Y3", + "VPSADBW X1, X2, K1, X3", + "VPSADBW.Z X1, X2, X3", + "VPSADBW.BCST (AX), X2, X3", + "VPSADBW $1, X2, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedsad(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsad": {Name: "invalidpackedsad", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PSADBW/VPSADBW tables", instruction) + } + }) + } +} + +func TestTranslateAMD64VectorLaneExtractCompleteGoAssemblerForms(t *testing.T) { + // VEXTRACTF128/I128 share Go 1.27's _yvextractf128 table. + var src strings.Builder + src.WriteString("TEXT vectorlaneextractforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VEXTRACTF128", "VEXTRACTI128"} { + fmt.Fprintf(&src, "\t%s $1, Y1, X2\n", op) + fmt.Fprintf(&src, "\t%s $-1, Y1, (AX)\n", op) + } + // These four produce 128 bits from either Y or Z and share + // _yvextractf32x4, including masked register/memory destinations. + for _, op := range []string{"VEXTRACTF32X4", "VEXTRACTF64X2", "VEXTRACTI32X4", "VEXTRACTI64X2"} { + for _, form := range []string{ + "$1, Y1, X2", "$1, Y1, (AX)", + "$1, Y1, K1, X2", "$1, Y1, K1, (AX)", + "$3, Z1, X2", "$3, Z1, (AX)", + "$3, Z1, K1, X2", "$3, Z1, K1, (AX)", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + fmt.Fprintf(&src, "\t%s.Z $1, Y1, K1, X2\n", op) + fmt.Fprintf(&src, "\t%s.Z $3, Z1, K1, X2\n", op) + } + // These four produce 256 bits from Z and share _yvextractf32x8. + for _, op := range []string{"VEXTRACTF32X8", "VEXTRACTF64X4", "VEXTRACTI32X8", "VEXTRACTI64X4"} { + for _, form := range []string{ + "$1, Z1, Y2", "$1, Z1, (AX)", + "$1, Z1, K1, Y2", "$1, Z1, K1, (AX)", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, form) + } + fmt.Fprintf(&src, "\t%s.Z $1, Z1, K1, Y2\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "vectorlaneextractforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "vector-lane-extract-forms.ll", "vector-lane-extract-forms.o", ll) +} + +func TestTranslateAMD64VectorLaneExtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VEXTRACTI128 $1, Z1, X2", + "VEXTRACTF128 $1, Y1, K1, X2", + "VEXTRACTI32X4 $1, X1, X2", + "VEXTRACTF64X2 $1, Y1, Y2", + "VEXTRACTI32X8 $1, Y1, Y2", + "VEXTRACTF64X4 $1, Z1, K0, Y2", + "VEXTRACTF32X4.Z $1, Y1, X2", + "VEXTRACTI64X2.Z $1, Z1, K1, (AX)", + "VEXTRACTI64X4.SAE $1, Z1, Y2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidvectorlaneextract(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidvectorlaneextract": {Name: "invalidvectorlaneextract", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's vector-lane extract tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedScalarExtractCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yextr/yextrw tables accept an unsigned imm8, an + // X0-X15 source, and a GP-or-memory destination. The yextr table used by + // B/D/Q also (unusually) classifies M0-M7 as destinations; those names + // encode the correspondingly numbered GP registers. + var src strings.Builder + src.WriteString("TEXT packedscalarextractforms(SB),NOSPLIT,$0-0\n") + for _, spec := range []struct { + op string + maxIndex int + mmxDst bool + }{ + {op: "PEXTRB", maxIndex: 15, mmxDst: true}, + {op: "PEXTRW", maxIndex: 7}, + {op: "PEXTRD", maxIndex: 3, mmxDst: true}, + {op: "PEXTRQ", maxIndex: 1, mmxDst: true}, + } { + fmt.Fprintf(&src, "\t%s $%d, X15, AX\n", spec.op, spec.maxIndex) + fmt.Fprintf(&src, "\t%s $255, X15, (AX)\n", spec.op) + if spec.mmxDst { + fmt.Fprintf(&src, "\t%s $%d, X15, M2\n", spec.op, spec.maxIndex) + } + } + + // Go 1.27's _yvextractps/_yvpextrw tables add signed-imm8 VEX forms + // for X0-X15 and unsigned-imm8 EVEX forms for X16-X31. They do not add + // mask or instruction-suffix forms. + for _, spec := range []struct { + op string + maxIndex int + }{ + {op: "VPEXTRB", maxIndex: 15}, + {op: "VPEXTRW", maxIndex: 7}, + {op: "VPEXTRD", maxIndex: 3}, + {op: "VPEXTRQ", maxIndex: 1}, + } { + fmt.Fprintf(&src, "\t%s $%d, X1, R8\n", spec.op, spec.maxIndex) + fmt.Fprintf(&src, "\t%s $255, X1, (AX)\n", spec.op) + fmt.Fprintf(&src, "\t%s $-128, X1, R8\n", spec.op) + fmt.Fprintf(&src, "\t%s $%d, X31, R15\n", spec.op, spec.maxIndex) + fmt.Fprintf(&src, "\t%s $255, X31, (AX)\n", spec.op) + } + src.WriteString("\tRET\n") + + ll := translateAMD64EcosystemCase(t, src.String(), "packedscalarextractforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-scalar-extract-forms.ll", "packed-scalar-extract-forms.o", ll) +} + +func TestTranslateAMD64PackedScalarExtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PEXTRB $-1, X1, AX", + "PEXTRD $256, X1, AX", + "PEXTRW $1, X1, M2", + "PEXTRQ $1, X16, AX", + "PEXTRB $1, Y1, AX", + "VPEXTRB $-129, X1, AX", + "VPEXTRW $256, X1, AX", + "VPEXTRD $-1, X20, AX", + "VPEXTRQ $1, X1, M2", + "VPEXTRB $1, X1, K1, AX", + "VPEXTRW.Z $1, X1, AX", + "VPEXTRD.BCST $1, X1, (AX)", + "VPEXTRQ.SAE $1, X1, AX", + "VPEXTRB $1, X1, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedscalarextract(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedscalarextract": {Name: "invalidpackedscalarextract", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PEXTR/VPEXTR tables", instruction) + } + }) + } +} + +func TestTranslateAMD64MoveHighLowPackedSingleCompleteGoAssemblerForms(t *testing.T) { + // MOVHLPS/MOVLHPS use Go 1.27's yxr table (X0-X15, two operands). + // VMOVHLPS/VMOVLHPS share _yvmovhlps: three X operands, with both the + // X0-X15 VEX entry and the X0-X31 EVEX entry. + src := ` +TEXT movehighlowpackedsingleforms(SB),NOSPLIT,$0-0 + MOVHLPS X1, X2 + MOVLHPS X15, X0 + VMOVHLPS X1, X2, X3 + VMOVLHPS X15, X14, X13 + VMOVHLPS X20, X2, X3 + VMOVHLPS X1, X20, X3 + VMOVLHPS X1, X2, X31 + RET +` + ll := translateAMD64EcosystemCase(t, src, "movehighlowpackedsingleforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "move-high-low-packed-single-forms.ll", "move-high-low-packed-single-forms.o", ll) +} + +func TestTranslateAMD64MoveHighLowPackedSingleRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "MOVHLPS X16, X1", + "MOVLHPS X1, X16", + "MOVHLPS (AX), X1", + "MOVLHPS X1, X2, X3", + "VMOVHLPS X1, X2", + "VMOVLHPS (AX), X2, X3", + "VMOVHLPS Y1, Y2, Y3", + "VMOVLHPS X1, X2, K1, X3", + "VMOVHLPS.Z X1, X2, X3", + "VMOVLHPS.BCST X1, X2, X3", + "VMOVHLPS.SAE X1, X2, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidmovehighlowpackedsingle(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidmovehighlowpackedsingle": {Name: "invalidmovehighlowpackedsingle", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's MOVHLPS/MOVLHPS tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PopulationCountCompleteGoAssemblerForms(t *testing.T) { + // POPCNTW/L/Q all use Go 1.27's yml_rl table: a Yml GP-or-memory + // source followed by a Yrl GP destination. + src := ` +TEXT populationcountforms(SB),NOSPLIT,$0-0 + POPCNTW AX, BX + POPCNTW (AX), R8 + POPCNTW R15, R14 + POPCNTL AX, BX + POPCNTL (AX), R8 + POPCNTL R15, R14 + POPCNTQ AX, BX + POPCNTQ (AX), R8 + POPCNTQ R15, R14 + RET +` + ll := translateAMD64EcosystemCase(t, src, "populationcountforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "population-count-forms.ll", "population-count-forms.o", ll) +} + +func TestTranslate386PopulationCountCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT populationcount386forms(SB),NOSPLIT,$0-0 + POPCNTW AX, BX + POPCNTW (AX), CX + POPCNTL AX, BX + POPCNTL (AX), CX + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "populationcount386forms": {Name: "populationcount386forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "population-count-386-forms.ll", "population-count-386-forms.o", ll) + + invalid, err := Parse(ArchAMD64, "TEXT invalidpopulationcount386(SB),NOSPLIT,$0-0\n\tPOPCNTQ AX, BX\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(invalid, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidpopulationcount386": {Name: "invalidpopulationcount386", Ret: Void}, + }, + }); err == nil { + t.Fatal("Translate accepted POPCNTQ for GOARCH=386") + } +} + +func TestTranslateAMD64PopulationCountRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "POPCNTW AL, BX", + "POPCNTL AX, AH", + "POPCNTQ M1, AX", + "POPCNTW X1, AX", + "POPCNTL AX, (BX)", + "POPCNTQ $1, AX", + "POPCNTW.Z AX, BX", + "POPCNTL AX, BX, CX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpopulationcount(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpopulationcount": {Name: "invalidpopulationcount", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's POPCNT tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedMoveMaskCompleteGoAssemblerForms(t *testing.T) { + // PMOVMSKB's ymskb table has X and M source rows. VPMOVMSKB's + // _yvmovmskpd table has X and Y source rows. Every row writes Yrl and + // every vector register is limited to 0-15; neither table has EVEX rows. + src := ` +TEXT packedmovemaskforms(SB),NOSPLIT,$0-0 + PMOVMSKB X0, AX + PMOVMSKB X15, R8 + PMOVMSKB M0, BX + PMOVMSKB M7, R15 + VPMOVMSKB X0, AX + VPMOVMSKB X15, R8 + VPMOVMSKB Y0, BX + VPMOVMSKB Y15, R15 + RET +` + ll := translateAMD64EcosystemCase(t, src, "packedmovemaskforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-move-mask-forms.ll", "packed-move-mask-forms.o", ll) +} + +func TestTranslateAMD64PackedMoveMaskRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PMOVMSKB Y1, AX", + "PMOVMSKB X16, AX", + "PMOVMSKB (AX), BX", + "PMOVMSKB X1, M2", + "VPMOVMSKB M1, AX", + "VPMOVMSKB X16, AX", + "VPMOVMSKB Y16, AX", + "VPMOVMSKB X1, AL", + "VPMOVMSKB X1, X2", + "VPMOVMSKB X1, K1, AX", + "PMOVMSKB.Z X1, AX", + "VPMOVMSKB.BCST Y1, AX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedmovemask(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedmovemask": {Name: "invalidpackedmovemask", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMOVMSKB/VPMOVMSKB tables", instruction) + } + }) + } +} + +func TestTranslateAMD64BitwiseNotCompleteGoAssemblerForms(t *testing.T) { + // NOTB/W/L/Q all use Go 1.27's yscond table, whose sole row is Ymb. + // Besides memory and full GP spellings, Ymb admits low/high byte register + // spellings; for wider operations Go encodes those spellings by ModRM + // number (for example, NOTW AH encodes NOTW SP). + var src strings.Builder + src.WriteString("TEXT bitwisenotforms(SB),NOSPLIT,$0-0\n") + operands := []string{ + "AX", "BX", "CX", "DX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15", + "AL", "AH", "BL", "BH", "CL", "CH", "DL", "DH", + "BPB", "SIB", "DIB", "R8B", "R9B", "R10B", "R11B", "R12B", "R13B", "R14B", "R15B", + "(BX)", + } + for _, op := range []string{"NOTB", "NOTW", "NOTL", "NOTQ"} { + for _, operand := range operands { + fmt.Fprintf(&src, "\t%s %s\n", op, operand) + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "bitwisenotforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "bitwise-not-forms.ll", "bitwise-not-forms.o", ll) +} + +func TestTranslate386BitwiseNotCompleteGoAssemblerForms(t *testing.T) { + // In 386 mode Go classifies SP as Yrl32, which does not cover Ymb, but + // accepts BP/SI/DI by synthesizing an equivalent instruction sequence. + // Exercise every accepted GP spelling, both byte aliases, and memory. + var src strings.Builder + src.WriteString("TEXT bitwisenot386forms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"NOTB", "NOTW", "NOTL"} { + for _, operand := range []string{ + "AX", "BX", "CX", "DX", "BP", "SI", "DI", + "AL", "AH", "BL", "BH", "CL", "CH", "DL", "DH", + "(BX)", + } { + fmt.Fprintf(&src, "\t%s %s\n", op, operand) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "bitwisenot386forms": {Name: "bitwisenot386forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "bitwise-not-386-forms.ll", "bitwise-not-386-forms.o", ll) + + for _, instruction := range []string{"NOTB SP", "NOTW SP", "NOTL SP", "NOTQ AX", "NOTB R8"} { + t.Run("reject_"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + invalid, err := Parse(ArchAMD64, "TEXT invalidbitwisenot386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(invalid, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidbitwisenot386": {Name: "invalidbitwisenot386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + }) + } +} + +func TestTranslateAMD64BitwiseNotRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "NOTB M1", + "NOTW X1", + "NOTL K1", + "NOTQ $1", + "NOTB AX, BX", + "NOTW.Z AX", + "NOTL.BCST (AX)", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidbitwisenot(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidbitwisenot": {Name: "invalidbitwisenot", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's NOT tables", instruction) + } + }) + } +} + +func TestTranslateAMD64ParallelBitDepositExtractCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses one _yandnl row for the whole family: + // Yml source/mask, Yrl source value, and Yrl destination. This gives each + // L/Q instruction both register and memory first-operand forms. + var src strings.Builder + src.WriteString("TEXT parallelbitforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + fmt.Fprintf(&src, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&src, "\t%s (SI), R8, R15\n", op) + fmt.Fprintf(&src, "\t%s parallelBitMask(SB), R9, R10\n", op) + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "parallelbitforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "parallel-bit-forms.ll", "parallel-bit-forms.o", ll) +} + +func TestTranslate386ParallelBitDepositExtractCompleteGoAssemblerForms(t *testing.T) { + // The Go 1.27 assembler accepts the same four instruction names in 386 + // mode. Yml/Yrl admit all eight 386 GP registers, including SP in any + // register position; only the first operand can instead be memory. + var src strings.Builder + src.WriteString("TEXT parallelbitforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + fmt.Fprintf(&src, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&src, "\t%s (SI), BP, DI\n", op) + fmt.Fprintf(&src, "\t%s SP, AX, BX\n", op) + fmt.Fprintf(&src, "\t%s AX, SP, BX\n", op) + fmt.Fprintf(&src, "\t%s AX, BX, SP\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "parallelbitforms386": {Name: "parallelbitforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "parallel-bit-386-forms.ll", "parallel-bit-386-forms.o", ll) +} + +func TestTranslateAMD64ParallelBitDepositExtractRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PDEPL AX, BX", + "PDEPQ AX, (BX), CX", + "PEXTL AX, BX, (CX)", + "PEXTQ $1, AX, BX", + "PDEPL AL, BX, CX", + "PDEPQ AX, X0, BX", + "PEXTL AX, BX, X0", + "PEXTQ.Z AX, BX, CX", + "PDEPL.BCST (AX), BX, CX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidparallelbit(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidparallelbit": {Name: "invalidparallelbit", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PDEP/PEXT table", instruction) + } + }) + } +} + +func TestTranslate386ParallelBitDepositExtractRejectsHighRegisters(t *testing.T) { + for _, instruction := range []string{ + "PDEPL R8, AX, BX", + "PDEPQ AX, R8, BX", + "PEXTL AX, BX, R8", + } { + file, err := Parse(ArchAMD64, "TEXT invalidparallelbit386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidparallelbit386": {Name: "invalidparallelbit386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + } +} + +func TestTranslateAMD64BitTestFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives all BT/BTC/BTR/BTS W/L/Q instructions exactly two ybtl + // rows: signed imm8 + Yml and Yrl + Yml. + var src strings.Builder + src.WriteString("TEXT bittestforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"BT", "BTC", "BTR", "BTS"} { + for _, width := range []string{"W", "L", "Q"} { + op := stem + width + fmt.Fprintf(&src, "\t%s $-128, AX\n", op) + fmt.Fprintf(&src, "\t%s $127, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, R15\n", op) + fmt.Fprintf(&src, "\t%s R8, 8(SI)\n", op) + fmt.Fprintf(&src, "\t%s R9, bitTestGlobal(SB)\n", op) + } + } + src.WriteString("\tRET\n") + ll := translateAMD64EcosystemCase(t, src.String(), "bittestforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "bit-test-forms.ll", "bit-test-forms.o", ll) +} + +func TestTranslate386BitTestFamilyCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT bittestforms386(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"BT", "BTC", "BTR", "BTS"} { + for _, width := range []string{"W", "L"} { + op := stem + width + fmt.Fprintf(&src, "\t%s $-128, AX\n", op) + fmt.Fprintf(&src, "\t%s $127, (BX)\n", op) + fmt.Fprintf(&src, "\t%s CX, DI\n", op) + fmt.Fprintf(&src, "\t%s SP, BP\n", op) + fmt.Fprintf(&src, "\t%s AX, SP\n", op) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "bittestforms386": {Name: "bittestforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "bit-test-386-forms.ll", "bit-test-386-forms.o", ll) +} + +func TestTranslateAMD64BitTestFamilyRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "BTB $1, AX", + "BTRL $128, AX", + "BTSQ $-129, AX", + "BTCW AX", + "BTL (AX), BX", + "BTRQ AL, AX", + "BTSW AX, $1", + "BTQ AX, X0", + "BTCL.Z AX, BX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidbittest(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidbittest": {Name: "invalidbittest", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's ybtl table", instruction) + } + }) + } +} + +func TestTranslate386BitTestFamilyRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "BTQ $1, AX", + "BTCQ AX, BX", + "BTRQ $1, (BX)", + "BTSQ AX, SP", + "BTL R8, AX", + "BTRW AX, R8", + } { + file, err := Parse(ArchAMD64, "TEXT invalidbittest386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidbittest386": {Name: "invalidbittest386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for GOARCH=386", instruction) + } + } +} + +func TestTranslateAMD64ScalarAddSubCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalaraddsubforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ADD", "SUB"} { + fmt.Fprintf(&src, "\t%sB $-2147483648, AL\n", stem) + fmt.Fprintf(&src, "\t%sB $4294967295, bytearg+0(FP)\n", stem) + fmt.Fprintf(&src, "\t%sB BP, (BX)\n", stem) + fmt.Fprintf(&src, "\t%sB (BX), R15\n", stem) + fmt.Fprintf(&src, "\t%sB bytearg+0(FP), R14\n", stem) + fmt.Fprintf(&src, "\t%sB CL, bytearg+0(FP)\n", stem) + for _, form := range []struct { + width string + fpName string + offset int + }{ + {width: "W", fpName: "wordarg", offset: 8}, + {width: "L", fpName: "longarg", offset: 16}, + {width: "Q", fpName: "quadarg", offset: 24}, + } { + op := stem + form.width + fmt.Fprintf(&src, "\t%s $-2147483648, AX\n", op) + fmt.Fprintf(&src, "\t%s $4294967295, %s+%d(FP)\n", op, form.fpName, form.offset) + fmt.Fprintf(&src, "\t%s R8, (BX)\n", op) + fmt.Fprintf(&src, "\t%s (BX), R15\n", op) + fmt.Fprintf(&src, "\t%s %s+%d(FP), R14\n", op, form.fpName, form.offset) + fmt.Fprintf(&src, "\t%s CX, %s+%d(FP)\n", op, form.fpName, form.offset) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "scalaraddsubforms": { + Name: "scalaraddsubforms", + Args: []LLVMType{I8, I16, I32, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I8, Index: 0, Field: -1}, + {Offset: 8, Type: I16, Index: 1, Field: -1}, + {Offset: 16, Type: I32, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-add-sub-forms.ll", "scalar-add-sub-forms.o", ll) +} + +func TestTranslate386ScalarAddSubCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalaraddsubforms386(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"ADD", "SUB"} { + fmt.Fprintf(&src, "\t%sB $4294967296, BP\n", stem) + fmt.Fprintf(&src, "\t%sB $-2147483649, bytearg+0(FP)\n", stem) + fmt.Fprintf(&src, "\t%sB SI, (BX)\n", stem) + fmt.Fprintf(&src, "\t%sB (BX), DI\n", stem) + for _, form := range []struct { + width string + fpName string + offset int + }{ + {width: "W", fpName: "wordarg", offset: 4}, + {width: "L", fpName: "longarg", offset: 8}, + } { + op := stem + form.width + fmt.Fprintf(&src, "\t%s $4294967296, AX\n", op) + fmt.Fprintf(&src, "\t%s $1, SP\n", op) + fmt.Fprintf(&src, "\t%s $-2147483649, %s+%d(FP)\n", op, form.fpName, form.offset) + fmt.Fprintf(&src, "\t%s SI, (BX)\n", op) + fmt.Fprintf(&src, "\t%s (BX), DI\n", op) + } + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "scalaraddsubforms386": { + Name: "scalaraddsubforms386", + Args: []LLVMType{I8, I16, I32}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I8, Index: 0, Field: -1}, + {Offset: 4, Type: I16, Index: 1, Field: -1}, + {Offset: 8, Type: I32, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "scalar-add-sub-386-forms.ll", "scalar-add-sub-386-forms.o", ll) +} + +func TestTranslateAMD64ScalarAddSubRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "ADDB (AX), (BX)", + "SUBW (AX), (BX)", + "ADDL $-2147483649, AX", + "SUBQ $4294967296, AX", + "ADDQ.Z AX, BX", + "SUBQ X0, AX", + "ADDQ AX, X0", + "SUBQ AX, $1", + "ADDB SP, (AX)", + "SUBB (AX), SP", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidscalaraddsub(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidscalaraddsub": {Name: "invalidscalaraddsub", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's ADD/SUB tables", instruction) + } + }) + } +} + +func TestTranslate386ScalarAddSubRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "ADDQ AX, BX", + "SUBQ $1, (BX)", + "ADDL R8, AX", + "SUBW AX, R8", + "ADDB R8, (AX)", + "SUBB (AX), R8", + "ADDB SP, (AX)", + } { + file, err := Parse(ArchAMD64, "TEXT invalidscalaraddsub386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidscalaraddsub386": {Name: "invalidscalaraddsub386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + } +} + +func TestTranslateAMD64CompareExchangeOriginalSyntaxContextCatalogue(t *testing.T) { + src := ` +DATA cmpxchgdata+0(SB)/8, $0 +DATA cmpxchgdata+8(SB)/8, $0 +GLOBL cmpxchgdata(SB), NOPTR, $16 +TEXT cmpxchgforms(SB),NOSPLIT,$0-16 + CMPXCHGB AH, R11 + CMPXCHGB R11, SP + CMPXCHGB DL, (R12) + CMPXCHGB DL, 0(FS) + CMPXCHGB DL, cmpxchgdata(SB) + CMPXCHGB DL, value+0(FP) + CMPXCHGW DX, R11 + CMPXCHGW R11, (R12) + CMPXCHGW DX, 0(GS) + CMPXCHGW DX, cmpxchgdata(SB) + CMPXCHGW DX, value+0(FP) + CMPXCHGL DX, R11 + CMPXCHGL R11, (R12) + CMPXCHGL DX, 0(FS) + CMPXCHGL DX, cmpxchgdata(SB) + CMPXCHGL DX, value+0(FP) + CMPXCHGQ DX, R11 + CMPXCHGQ R11, (R12) + CMPXCHGQ DX, 0(GS) + CMPXCHGQ DX, cmpxchgdata(SB) + CMPXCHGQ DX, value+0(FP) + CMPXCHG8B R11 + CMPXCHG8B SP + CMPXCHG8B (R12) + CMPXCHG8B 0(FS) + CMPXCHG8B cmpxchgdata(SB) + CMPXCHG8B value+0(FP) + CMPXCHG16B R11 + CMPXCHG16B SP + CMPXCHG16B (R12) + CMPXCHG16B 0(GS) + CMPXCHG16B cmpxchgdata(SB) + CMPXCHG16B value+0(FP) + RET +` + x86CMPXCHGMixedOriginalCatalogue(t, "amd64", src) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "cmpxchgforms": { + Name: "cmpxchgforms", + Args: []LLVMType{LLVMType("i128")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: LLVMType("i128"), Index: 0, Field: -1}, + }}, + }, + }, + }) + // This unchanged syntax catalogue contains Go-accepted physical high-byte + // rewrites, non-memory pair #UD and an unbound 16B FP shadow. It is not a + // numerical native-positive. Closed memory forms have independent objects. + if !errors.Is(err, ErrProbeNeedsContext) || ll != "" { + t.Fatalf("mixed original native-identity gaps acquired a numerical object: %v", err) + } +} + +func TestTranslate386CompareExchangeOriginalSyntaxContextCatalogue(t *testing.T) { + src := ` +DATA cmpxchgdata386+0(SB)/8, $0 +GLOBL cmpxchgdata386(SB), NOPTR, $8 +TEXT cmpxchgforms386(SB),NOSPLIT,$0-8 + CMPXCHGB BP, (BX) + CMPXCHGB SI, DI + CMPXCHGB DL, 0(FS) + CMPXCHGB DL, cmpxchgdata386(SB) + CMPXCHGB DL, value+0(FP) + CMPXCHGW SI, DI + CMPXCHGW DX, SP + CMPXCHGW DX, (BX) + CMPXCHGW DX, 0(FS) + CMPXCHGW DX, cmpxchgdata386(SB) + CMPXCHGW DX, value+0(FP) + CMPXCHGL SI, DI + CMPXCHGL DX, SP + CMPXCHGL DX, (BX) + CMPXCHGL DX, 0(FS) + CMPXCHGL DX, cmpxchgdata386(SB) + CMPXCHGL DX, value+0(FP) + CMPXCHG8B AX + CMPXCHG8B BP + CMPXCHG8B SI + CMPXCHG8B (BX) + CMPXCHG8B 0(FS) + CMPXCHG8B cmpxchgdata386(SB) + CMPXCHG8B value+0(FP) + RET +` + x86CMPXCHGMixedOriginalCatalogue(t, "386", src) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "cmpxchgforms386": { + Name: "cmpxchgforms386", + Args: []LLVMType{I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + }}, + }, + }, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ll != "" { + t.Fatalf("mixed original byte expansion/pair #UD acquired a numerical object: %v", err) + } +} + +func TestTranslateAMD64CompareExchangeRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CMPXCHGB (BX), DL", + "CMPXCHGW (BX), DX", + "CMPXCHGL $1, DX", + "CMPXCHGQ DX, $1", + "CMPXCHGB X0, DL", + "CMPXCHGL DX, X0", + "CMPXCHG8B", + "CMPXCHG8B (BX), AX", + "CMPXCHG8B $1", + "CMPXCHG16B", + "CMPXCHG16B (BX), AX", + "CMPXCHG16B.Z (BX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidcmpxchg(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidcmpxchg": {Name: "invalidcmpxchg", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's CMPXCHG tables", instruction) + } + }) + } +} + +func TestTranslate386CompareExchangeRejectsArchitectureRestrictedForms(t *testing.T) { + for _, instruction := range []string{ + "CMPXCHGQ DX, (BX)", + "CMPXCHG16B (BX)", + "CMPXCHGB R8, (BX)", + "CMPXCHGW R8, (BX)", + "CMPXCHGL R8, (BX)", + "CMPXCHGB SP, (BX)", + "CMPXCHGB DL, SP", + "CMPXCHG8B SP", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalidcmpxchg386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidcmpxchg386": {Name: "invalidcmpxchg386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which Go 1.27 rejects for 386", instruction) + } + }) + } +} + +func TestTranslateAMD64TrailingZeroCountCompleteGoAssemblerForms(t *testing.T) { + // TZCNTW/L/Q all use Go 1.27's ycrc32l table: Yml source, Yrl + // destination. The width is selected only by the opcode prefix. + src := ` +TEXT trailingzerocountforms(SB),NOSPLIT,$0-0 + TZCNTW AX, BX + TZCNTW (AX), R8 + TZCNTW R15, R14 + TZCNTL AX, BX + TZCNTL (AX), R8 + TZCNTL R15, R14 + TZCNTQ AX, BX + TZCNTQ (AX), R8 + TZCNTQ R15, R14 + RET +` + ll := translateAMD64EcosystemCase(t, src, "trailingzerocountforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "trailing-zero-count-forms.ll", "trailing-zero-count-forms.o", ll) +} + +func TestTranslate386TrailingZeroCountCompleteGoAssemblerForms(t *testing.T) { + src := ` +TEXT trailingzerocount386forms(SB),NOSPLIT,$0-0 + TZCNTW AX, BX + TZCNTW (AX), CX + TZCNTL AX, BX + TZCNTL (AX), CX + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "trailingzerocount386forms": {Name: "trailingzerocount386forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "trailing-zero-count-386-forms.ll", "trailing-zero-count-386-forms.o", ll) + + invalid, err := Parse(ArchAMD64, "TEXT invalidtrailingzerocount386(SB),NOSPLIT,$0-0\n\tTZCNTQ AX, BX\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(invalid, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidtrailingzerocount386": {Name: "invalidtrailingzerocount386", Ret: Void}, + }, + }); err == nil { + t.Fatal("Translate accepted TZCNTQ for GOARCH=386") + } +} + +func TestTranslateAMD64TrailingZeroCountRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "TZCNTW AL, BX", + "TZCNTL AX, AH", + "TZCNTQ M1, AX", + "TZCNTW X1, AX", + "TZCNTL AX, (BX)", + "TZCNTQ $1, AX", + "TZCNTW.Z AX, BX", + "TZCNTL AX, BX, CX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidtrailingzerocount(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidtrailingzerocount": {Name: "invalidtrailingzerocount", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's TZCNT tables", instruction) + } + }) + } +} + +func TestTranslateAMD64FloatingCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxcmpi table accepts X/m, X0-X15, signed imm8. + // The packed AVX _yvcmppd table adds X/Y vector-result VEX forms and + // X/Y/Z K-result EVEX forms. EVEX memory sources may use .BCST; only + // the Z-register source form may use .SAE. The scalar _yvcmpsd table + // has X vector-result VEX forms and X K-result EVEX forms; only the + // EVEX register-source form may use .SAE. + var src strings.Builder + src.WriteString("TEXT floatingcompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"CMPPD", "CMPPS", "CMPSD", "CMPSS"} { + fmt.Fprintf(&src, "\t%s X1, X15, $-128\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15, $127\n", op) + } + for _, op := range []string{"VCMPPD", "VCMPPS"} { + fmt.Fprintf(&src, "\t%s $255, X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $255, Y1, Y2, Y3\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), Y2, Y3\n", op) + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s $255, %s20, %s21, K0\n", op, width, width) + fmt.Fprintf(&src, "\t%s $0, (AX), %s21, K1, K7\n", op, width) + fmt.Fprintf(&src, "\t%s.BCST $255, (AX), %s21, K7\n", op, width) + } + fmt.Fprintf(&src, "\t%s.SAE $255, Z20, Z21, K1, K7\n", op) + } + for _, op := range []string{"VCMPSD", "VCMPSS"} { + fmt.Fprintf(&src, "\t%s $255, X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), X2, X3\n", op) + fmt.Fprintf(&src, "\t%s $255, X20, X21, K0\n", op) + fmt.Fprintf(&src, "\t%s $0, (AX), X21, K1, K7\n", op) + fmt.Fprintf(&src, "\t%s.SAE $255, X20, X21, K1, K7\n", op) + } + src.WriteString("\tRET\n") + + ll := translateAMD64EcosystemCase(t, src.String(), "floatingcompareforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "floating-compare-forms.ll", "floating-compare-forms.o", ll) +} + +func TestTranslate386FloatingCompareCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT floatingcompareforms386(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"CMPPD", "CMPPS", "CMPSD", "CMPSS"} { + fmt.Fprintf(&src, "\t%s X1, X7, $-128\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7, $127\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "floatingcompareforms386": {Name: "floatingcompareforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "floating-compare-forms-386.ll", "floating-compare-forms-386.o", ll) +} + +func TestTranslateAMD64FloatingCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "CMPPD X1, X2, $-129", + "CMPPS X1, X2, $128", + "CMPSD X16, X2, $0", + "CMPSS X1, X16, $0", + "CMPPD Y1, X2, $0", + "CMPPS.BCST (AX), X2, $0", + "VCMPPD $-1, X1, X2, X3", + "VCMPPS $256, X1, X2, X3", + "VCMPSD $0, Y1, X2, X3", + "VCMPSS $0, X1, Y2, X3", + "VCMPPD $0, X1, X2, Y3", + "VCMPPS $0, Y1, Y2, X3", + "VCMPPD $0, X20, X21, X22", + "VCMPSD $0, X20, X21, X22", + "VCMPSS $0, X1, X2, K0, K1", + "VCMPPD $0, X1, X2, K0, K1", + "VCMPPS.Z $0, Z1, Z2, K1, K2", + "VCMPPD.BCST $0, Z1, Z2, K1", + "VCMPPS.BCST $0, (AX), X2, X3", + "VCMPSD.BCST $0, (AX), X2, K1", + "VCMPSS.BCST $0, (AX), X2, K1", + "VCMPPD.SAE $0, X1, X2, K1", + "VCMPPS.SAE $0, Y1, Y2, K1", + "VCMPPD.SAE $0, (AX), Z2, K1", + "VCMPSD.SAE $0, (AX), X2, K1", + "VCMPSS.SAE $0, X1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidfloatingcompare(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidfloatingcompare": {Name: "invalidfloatingcompare", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's CMP/VCMP floating-point tables", instruction) + } + }) + } +} + +func TestTranslate386FloatingCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "CMPPD X8, X1, $0", + "CMPSD X1, X8, $0", + "VCMPPD $0, X1, X2, X3", + "VCMPSD $0, X1, X2, X3", + } { + src := "TEXT invalidfloatingcompare386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidfloatingcompare386": {Name: "invalidfloatingcompare386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for GOARCH=386", instruction) + } + } +} + +func TestTranslateAMD64PackedSaturatingNarrowCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 names the legacy signed dword-to-word instruction PACKSSLW. + // PACKSSLW/PACKSSWB/PACKUSWB use ymm (MMX/m64 and X/m128 forms), while + // PACKUSDW uses yxm_q4 (X/m128 only). All four V instructions share + // _yvandnpd: X/Y VEX plus X/Y/Z EVEX, optional K1-K7 and .Z. Only the + // dword-to-word forms enable EVEX 32-bit memory broadcast. + var src strings.Builder + src.WriteString("TEXT packedsaturatingnarrowforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PACKSSLW", "PACKSSWB", "PACKUSWB"} { + fmt.Fprintf(&src, "\t%s M1, M7\n", op) + fmt.Fprintf(&src, "\t%s (AX), M7\n", op) + fmt.Fprintf(&src, "\t%s X1, X15\n", op) + fmt.Fprintf(&src, "\t%s (AX), X15\n", op) + } + src.WriteString("\tPACKUSDW X1, X15\n\tPACKUSDW (AX), X15\n") + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + fmt.Fprintf(&src, "\t%s X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2, X3\n", op) + fmt.Fprintf(&src, "\t%s Y1, Y2, Y3\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y2, Y3\n", op) + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s20, %s21, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z %s20, %s21, K7, %s22\n", op, width, width, width) + } + } + for _, op := range []string{"VPACKSSDW", "VPACKUSDW"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K7, %s22\n", op, width, width) + } + } + src.WriteString("\tRET\n") + + ll := translateAMD64EcosystemCase(t, src.String(), "packedsaturatingnarrowforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-saturating-narrow-forms.ll", "packed-saturating-narrow-forms.o", ll) +} + +func TestTranslate386PackedSaturatingNarrowCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedsaturatingnarrowforms386(SB),4,$0-0\n") + for _, op := range []string{"PACKSSLW", "PACKSSWB", "PACKUSWB"} { + fmt.Fprintf(&src, "\t%s X1, X7\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + } + src.WriteString("\tPACKUSDW X1, X7\n\tPACKUSDW (AX), X7\n") + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, bank := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s1,%s2,%s3\n", op, bank, bank, bank) + fmt.Fprintf(&src, "\t%s (AX),%s2,%s3\n", op, bank, bank) + if op == "VPACKSSDW" || op == "VPACKUSDW" { + fmt.Fprintf(&src, "\t%s.BCST (AX),%s2,%s3\n", op, bank, bank) + } + } + } + src.WriteString("\tRET\n") + goAssemblerOracleOutput(t, "386", src.String(), "", true) + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedsaturatingnarrowforms386": {Name: "packedsaturatingnarrowforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-saturating-narrow-forms-386.ll", "packed-saturating-narrow-forms-386.o", ll) +} + +func TestTranslateAMD64PackedSaturatingNarrowRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PACKSSLW M1, X2", + "PACKSSWB X1, M2", + "PACKUSDW M1, M2", + "PACKUSWB X16, X1", + "PACKSSLW X1, X16", + "PACKSSWB.Z X1, X2", + "PACKUSWB X1, X2, X3", + "VPACKSSDW X1, X2", + "VPACKSSWB X1, Y2, Y3", + "VPACKUSDW Y1, Y2, X3", + "VPACKUSWB Z1, Z2, Y3", + "VPACKSSDW X1, X2, K0, X3", + "VPACKSSWB.Z X1, X2, X3", + "VPACKUSDW.BCST X1, X2, X3", + "VPACKUSWB.BCST (AX), X2, X3", + "VPACKSSWB.BCST.Z (AX), X2, K1, X3", + "VPACKSSDW.SAE X1, X2, X3", + "VPACKUSWB X1, X2, K1", + "VPACKUSDW X1, (AX), X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidpackedsaturatingnarrow(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsaturatingnarrow": {Name: "invalidpackedsaturatingnarrow", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PACK/VPACK tables", instruction) + } + }) + } +} + +func TestTranslate386PackedSaturatingNarrowRejectsVectorForms(t *testing.T) { + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, row := range []string{op + " X1,X2,K1,X3", op + ".Z Z1,Z2,K1,Z3", op + " Z1,Z2,Z8"} { + t.Run(row, func(t *testing.T) { + source := "TEXT invalidpackedsaturatingnarrow386(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{TargetTriple: "i386-unknown-linux-gnu", Goarch: "386", + Sigs: map[string]FuncSig{"invalidpackedsaturatingnarrow386": {Name: "invalidpackedsaturatingnarrow386", Ret: Void}}, + }); err == nil { + t.Fatal("Go-rejected mode32 pack syntax acquired successful lowering") + } + }) + } + } +} + +func TestTranslateAMD64VariableDwordPermuteCompleteGoAssemblerForms(t *testing.T) { + // VPERMD and VPERMPS share Go 1.27's _yvpermd table: Y-only VEX, + // Y/Z EVEX, optional K1-K7 and .Z, plus 32-bit memory broadcast. + var src strings.Builder + src.WriteString("TEXT variabledwordpermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMD", "VPERMPS"} { + fmt.Fprintf(&src, "\t%s Y1, Y2, Y3\n", op) + fmt.Fprintf(&src, "\t%s (AX), Y2, Y3\n", op) + for _, width := range []string{"Y", "Z"} { + fmt.Fprintf(&src, "\t%s %s20, %s21, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s21, K1, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z %s20, %s21, K7, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s.BCST (AX), %s21, %s22\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K7, %s22\n", op, width, width) + } + } + src.WriteString("\tRET\n") + + ll := translateAMD64EcosystemCase(t, src.String(), "variabledwordpermuteforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "variable-dword-permute-forms.ll", "variable-dword-permute-forms.o", ll) +} + +func TestTranslateAMD64VariableDwordPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMD X1, X2, X3", + "VPERMPS Y1, Z2, Z3", + "VPERMD Z1, Z2, Y3", + "VPERMPS Y1, (AX), Y3", + "VPERMD Y1, Y2, K0, Y3", + "VPERMPS.Z Y1, Y2, Y3", + "VPERMD.BCST Y1, Y2, Y3", + "VPERMPS.BCST.Z (AX), Z2, Z3", + "VPERMD.SAE Y1, Y2, Y3", + "VPERMPS Y1, Y2, K1", + "VPERMD Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidvariabledwordpermute(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidvariabledwordpermute": {Name: "invalidvariabledwordpermute", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's VPERMD/VPERMPS table", instruction) + } + }) + } +} + +func TestTranslate386VariableDwordPermuteCompleteGoAssemblerForms(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT variabledwordpermuteforms386(SB),$0-0\n") + for _, op := range []string{"VPERMD", "VPERMPS"} { + fmt.Fprintf(&source, "\t%s Y1, Y2, Y3\n", op) + fmt.Fprintf(&source, "\t%s (AX), Y2, Y3\n", op) + fmt.Fprintf(&source, "\t%s Y31, Y30, Y29\n", op) + fmt.Fprintf(&source, "\t%s Z7, Z6, Z5\n", op) + fmt.Fprintf(&source, "\t%s.BCST (AX), Y30, Y29\n", op) + fmt.Fprintf(&source, "\t%s.BCST (AX), Z6, Z5\n", op) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "linux", triple: "i386-unknown-linux-gnu"}, + {name: "windows", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "386", + Sigs: map[string]FuncSig{ + "variabledwordpermuteforms386": {Name: "variabledwordpermuteforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "variable-dword-permute-forms-386-"+target.name+".ll", "variable-dword-permute-forms-386-"+target.name+".o", ll) + }) + } +} + +func TestTranslate386VariableDwordPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMD Z8, Z0, Z1", + "VPERMPS Z0, Z8, Z1", + "VPERMD Z0, Z1, Z8", + "VPERMD Y1, Y2, K1, Y3", + "VPERMPS.Z Y1, Y2, K1, Y3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's 386 VPERMD/VPERMPS table", instruction) + } + }) + } +} + +func TestTranslateAMD64ConditionalSetCompleteGoAssemblerForms(t *testing.T) { + conditions := []string{"CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS"} + var src strings.Builder + src.WriteString("TEXT conditionalsetforms(SB),NOSPLIT,$0-1\n") + for _, condition := range conditions { + op := "SET" + condition + fmt.Fprintf(&src, "\t%s R15\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s BPB\n", op) + fmt.Fprintf(&src, "\t%s R15B\n", op) + fmt.Fprintf(&src, "\t%s (AX)\n", op) + fmt.Fprintf(&src, "\t%s conditionalSetGlobal(SB)\n", op) + fmt.Fprintf(&src, "\t%s ret+0(FP)\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "conditionalsetforms": { + Name: "conditionalsetforms", + Ret: I8, + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: I8, + Index: 0, + Field: -1, + }}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "conditional-set-forms.ll", "conditional-set-forms.o", ll) +} + +func TestTranslate386ConditionalSetCompleteGoAssemblerForms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT conditionalsetforms386(SB),NOSPLIT,$0-1\n") + for _, condition := range []string{"CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS"} { + op := "SET" + condition + fmt.Fprintf(&src, "\t%s DX\n", op) + fmt.Fprintf(&src, "\t%s AH\n", op) + fmt.Fprintf(&src, "\t%s BP\n", op) + fmt.Fprintf(&src, "\t%s SI\n", op) + fmt.Fprintf(&src, "\t%s DI\n", op) + fmt.Fprintf(&src, "\t%s (AX)\n", op) + fmt.Fprintf(&src, "\t%s ret+0(FP)\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "conditionalsetforms386": { + Name: "conditionalsetforms386", + Ret: I8, + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: I8, + Index: 0, + Field: -1, + }}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "conditional-set-forms-386.ll", "conditional-set-forms-386.o", ll) +} + +func TestTranslateAMD64ConditionalSetRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "SETNE X1", + "SETLE M1", + "SETEQ K1", + "SETCC $1", + "SETGT", + "SETGE AX, BX", + "SETHI.Z AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidconditionalset(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidconditionalset": {Name: "invalidconditionalset", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's SETcc tables", instruction) + } + }) + } +} + +func TestTranslate386ConditionalSetRejectsArchitectureRestrictedRegisters(t *testing.T) { + for _, register := range []string{"SP", "R8"} { + file, err := Parse(ArchAMD64, "TEXT invalidconditionalset386(SB),NOSPLIT,$0-0\n\tSETNE "+register+"\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidconditionalset386": {Name: "invalidconditionalset386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted SETNE %s for GOARCH=386", register) + } + } +} + +func TestTranslateAMD64VectorScalarMoveCompleteGoAssemblerForms(t *testing.T) { + // VMOVSD and VMOVSS share Go 1.27's _yvmovsd table. It contains VEX + // scalar register-to-memory, memory-to-register, and three-register merge + // forms plus their EVEX high-register and optional K-mask counterparts. + // Zeroing is valid only for masked register destinations. + var src strings.Builder + src.WriteString("TEXT vectorscalarmoveforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VMOVSD", "VMOVSS"} { + fmt.Fprintf(&src, "\t%s X1, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), X1\n", op) + fmt.Fprintf(&src, "\t%s X1, X2, X3\n", op) + fmt.Fprintf(&src, "\t%s X20, (AX)\n", op) + fmt.Fprintf(&src, "\t%s X20, K1, (AX)\n", op) + fmt.Fprintf(&src, "\t%s (AX), X20\n", op) + fmt.Fprintf(&src, "\t%s (AX), K1, X20\n", op) + fmt.Fprintf(&src, "\t%s.Z (AX), K7, X20\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, X22\n", op) + fmt.Fprintf(&src, "\t%s X20, X21, K1, X22\n", op) + fmt.Fprintf(&src, "\t%s.Z X20, X21, K7, X22\n", op) + } + src.WriteString("\tRET\n") + + ll := translateAMD64EcosystemCase(t, src.String(), "vectorscalarmoveforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "vector-scalar-move-forms.ll", "vector-scalar-move-forms.o", ll) +} + +func TestTranslateAMD64VectorScalarMoveFrameSources(t *testing.T) { + src := ` +TEXT vmovsdframe(SB),NOSPLIT,$0-8 + VMOVSD value+0(FP), X0 + RET +TEXT vmovssframe(SB),NOSPLIT,$0-4 + VMOVSS value+0(FP), X0 + RET +TEXT vmovsdframeresult(SB),NOSPLIT,$0-8 + VMOVSD X0, value+0(FP) + RET +TEXT vmovssframeresult(SB),NOSPLIT,$0-4 + VMOVSS X0, value+0(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "vmovsdframe": { + Name: "vmovsdframe", + Args: []LLVMType{LLVMType("double")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, + Type: LLVMType("double"), + Index: 0, + Field: -1, + }}}, + }, + "vmovssframe": { + Name: "vmovssframe", + Args: []LLVMType{LLVMType("float")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, + Type: LLVMType("float"), + Index: 0, + Field: -1, + }}}, + }, + "vmovsdframeresult": { + Name: "vmovsdframeresult", + Ret: LLVMType("double"), + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: LLVMType("double"), + Index: 0, + Field: -1, + }}}, + }, + "vmovssframeresult": { + Name: "vmovssframeresult", + Ret: LLVMType("float"), + Frame: FrameLayout{Results: []FrameSlot{{ + Offset: 0, + Type: LLVMType("float"), + Index: 0, + Field: -1, + }}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "vector-scalar-move-frame.ll", "vector-scalar-move-frame.o", ll) +} + +func TestTranslateAMD64VectorScalarMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVSD X1, X2", + "VMOVSS (AX), (BX)", + "VMOVSD Y1, X2, X3", + "VMOVSS X1, Y2, X3", + "VMOVSD X1, X2, Y3", + "VMOVSS X1, X2, (AX)", + "VMOVSD (AX), K0, X1", + "VMOVSS X1, K0, (AX)", + "VMOVSD X1, X2, K0, X3", + "VMOVSS.Z (AX), X1", + "VMOVSD.Z X1, K1, (AX)", + "VMOVSS.Z X1, X2, X3", + "VMOVSD.BCST (AX), X1", + "VMOVSS.SAE X1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidvectorscalarmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", strings.ReplaceAll(src, "NOSPLIT", "4"), false) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidvectorscalarmove": {Name: "invalidvectorscalarmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's VMOVSD/VMOVSS table", instruction) + } + }) + } +} + +func TestTranslate386VectorScalarMoveRejectsForms(t *testing.T) { + for _, op := range []string{"VMOVSD", "VMOVSS"} { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),4,$0-0\n"+op+" X0, X1, K1, X2\nRET\n", false) + file, err := Parse(ArchAMD64, "TEXT invalidvectorscalarmove386(SB),NOSPLIT,$0-0\n\t"+op+" X0, X1, K1, X2\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalidvectorscalarmove386": {Name: "invalidvectorscalarmove386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %s for GOARCH=386", op) + } + } +} + +func TestTranslateX86VectorScalarFlagCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives VCOMISD/VCOMISS/VUCOMISD/VUCOMISS the shared + // _yvcomisd table. It has exactly two rows: VEX X/m,X and EVEX X/m,X. + // EVEX permits X0-X31 and .SAE for register sources; it does not permit + // masks, zeroing, broadcast, or vector widths other than X. Go's 386 + // assembler exposes the same two rows, including the EVEX register class. + var src strings.Builder + src.WriteString("TEXT vectorscalarflagcompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VCOMISD", "VCOMISS", "VUCOMISD", "VUCOMISS"} { + fmt.Fprintf(&src, "\t%s X1, X2\n", op) + fmt.Fprintf(&src, "\t%s (AX), X2\n", op) + fmt.Fprintf(&src, "\t%s X20, X2\n", op) + fmt.Fprintf(&src, "\t%s X1, X21\n", op) + fmt.Fprintf(&src, "\t%s (AX), X21\n", op) + fmt.Fprintf(&src, "\t%s.SAE X1, X2\n", op) + fmt.Fprintf(&src, "\t%s.SAE X20, X21\n", op) + } + src.WriteString("\tRET\n") + // Named Go frontend compatibility includes high X registers even on + // 386; raw architectural 32-bit decoding has stricter register limits. + for _, arch := range []string{"386", "amd64"} { + assembleX87ControlBytes(t, arch, strings.ReplaceAll(src.String(), "NOSPLIT", "4")) + } + + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-macosx"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "vectorscalarflagcompareforms": {Name: "vectorscalarflagcompareforms", Ret: Void}, + }, + }) + if target.goarch == "386" { + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "physical mode32 high-X") { + t.Fatalf("source-accepted high banks lack a physical 386 register binding: %v", err) + } + // The separate complete AF object matrix retains all legal + // physical X0-X7 formats on both 386 target platforms. + return + } + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "vector-scalar-flag-compare-"+target.goarch+".ll", "vector-scalar-flag-compare-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86VectorScalarFlagCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + for _, instruction := range []string{ + "VCOMISD", + "VCOMISS X1", + "VUCOMISD X1, X2, X3", + "VUCOMISS Y1, X2", + "VCOMISD X1, Y2", + "VCOMISS Z1, X2", + "VUCOMISD K1, X2", + "VUCOMISS X1, (AX)", + "VCOMISD $1, X2", + "VCOMISS.SAE (AX), X2", + "VUCOMISD.BCST (AX), X2", + "VUCOMISS.Z X1, X2", + "VCOMISD X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalidvectorscalarflagcompare(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalidvectorscalarflagcompare": {Name: "invalidvectorscalarflagcompare", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s, which is absent from Go 1.27's _yvcomisd table", instruction, target.goarch) + } + }) + } + }) + } +} + +func TestTranslateAMD64AdditionalPackedIntegerFamilies(t *testing.T) { + src := ` +TEXT packedintegerfamilies(SB),NOSPLIT,$0-0 + PADDB X0, X1 + PCMPEQW X0, X1 + PMULLW X0, X1 + PABSD X0, X1 + PSUBUSB X0, X1 + PTEST X0, X1 + RET +` + translateAMD64EcosystemCase(t, src, "packedintegerfamilies") +} + +func TestTranslateAMD64AdditionalConditionalMoveWidths(t *testing.T) { + src := ` +TEXT conditionalmoves(SB),NOSPLIT,$0-0 + CMPQ AX, BX + CMOVQLE CX, DX + CMOVLGT CX, DX + CMOVWCC CX, DX + RET +` + translateAMD64EcosystemCase(t, src, "conditionalmoves") +} + +func TestTranslateAMD64LegacyScalarAndPackedFloatingPointFamilies(t *testing.T) { + src := ` +TEXT floatfamilies(SB),NOSPLIT,$0-0 + MOVSS (AX), X0 + SQRTSS X0, X1 + ADDSS X0, X1 + SUBSS X0, X1 + MULSS X0, X1 + DIVSS X0, X1 + MINSS X0, X1 + MAXSS X0, X1 + MOVSS X1, (BX) + MOVUPD (AX), X2 + MOVAPD X2, (BX) + MOVUPD X2, 16(BX) + XORPD X2, X3 + ANDPS X2, X3 + ADDPS X2, X3 + SUBPS X2, X3 + MULPS X2, X3 + DIVPS X2, X3 + MINPS X2, X3 + MAXPS X2, X3 + SQRTPS X2, X3 + ADDPD X2, X3 + SUBPD X2, X3 + MULPD X2, X3 + DIVPD X2, X3 + MINPD X2, X3 + MAXPD X2, X3 + SQRTPD X2, X3 + MOVLHPS X2, X3 + MOVHLPS X2, X3 + MOVLPD (AX), X3 + MOVLPD X3, (BX) + MOVHPD (AX), X3 + MOVHPD X3, (BX) + SHUFPD $1, X2, X3 + UNPCKLPS X2, X3 + UNPCKHPD X2, X3 + MOVDDUP X2, X3 + HADDPD X2, X3 + HADDPD (AX), X3 + HADDPS X2, X3 + HADDPS (AX), X3 + CVTSS2SD X2, X3 + CVTSS2SD (AX), X3 + CVTSD2SS X2, X3 + CVTSD2SS (AX), X3 + UCOMISS X2, X3 + UCOMISD X2, X3 + RET +` + translateAMD64EcosystemCase(t, src, "floatfamilies") +} + +func TestTranslateAMD64DuplicateMoveFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxm table accepts X/m -> X for MOVDDUP, + // MOVSHDUP, and MOVSLDUP. Its shared _yvmovddup table accepts + // X/m -> X, Y/m -> Y, and Z/m -> Z for all three V-prefixed forms, + // with optional K masking (and .Z zero masking) on EVEX forms. + src := ` +TEXT duplicatemoveforms(SB),NOSPLIT,$0-8 + MOVDDUP X1, X2 + MOVDDUP (AX), X2 + MOVDDUP source(SB), X2 + MOVDDUP alpha+0(FP), X2 + MOVSHDUP X1, X2 + MOVSHDUP (AX), X2 + MOVSHDUP source(SB), X2 + MOVSLDUP X1, X2 + MOVSLDUP (AX), X2 + MOVSLDUP source(SB), X2 + VMOVDDUP X1, X2 + VMOVDDUP (AX), X2 + VMOVDDUP Y1, Y2 + VMOVDDUP (AX), Y2 + VMOVDDUP X16, X17 + VMOVDDUP X16, K1, X17 + VMOVDDUP.Z (AX), K1, X17 + VMOVDDUP Y16, Y17 + VMOVDDUP Y16, K1, Y17 + VMOVDDUP.Z (AX), K1, Y17 + VMOVDDUP Z1, Z2 + VMOVDDUP Z1, K1, Z2 + VMOVDDUP.Z source(SB), K1, Z2 + VMOVSHDUP X1, X2 + VMOVSHDUP (AX), X2 + VMOVSHDUP Y1, Y2 + VMOVSHDUP (AX), Y2 + VMOVSHDUP X16, X17 + VMOVSHDUP X16, K1, X17 + VMOVSHDUP.Z (AX), K1, X17 + VMOVSHDUP Y16, Y17 + VMOVSHDUP Y16, K1, Y17 + VMOVSHDUP.Z (AX), K1, Y17 + VMOVSHDUP Z1, Z2 + VMOVSHDUP Z1, K1, Z2 + VMOVSHDUP.Z source(SB), K1, Z2 + VMOVSLDUP X1, X2 + VMOVSLDUP (AX), X2 + VMOVSLDUP Y1, Y2 + VMOVSLDUP (AX), Y2 + VMOVSLDUP X16, X17 + VMOVSLDUP X16, K1, X17 + VMOVSLDUP.Z (AX), K1, X17 + VMOVSLDUP Y16, Y17 + VMOVSLDUP Y16, K1, Y17 + VMOVSLDUP.Z (AX), K1, Y17 + VMOVSLDUP Z1, Z2 + VMOVSLDUP Z1, K1, Z2 + VMOVSLDUP.Z source(SB), K1, Z2 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"duplicatemoveforms": { + Name: "duplicatemoveforms", + Args: []LLVMType{LLVMType("double")}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, + Type: LLVMType("double"), + Index: 0, + Field: -1, + }}}, + }}, + }) + if err != nil { + t.Fatal(err) + } +} + +func TestTranslateAMD64DuplicateMoveFamilyRejectsFormsOutsideGoOptabs(t *testing.T) { + for _, instruction := range []string{ + "MOVSHDUP Y1, Y2", + "MOVSLDUP.Z X1, X2", + "VMOVSHDUP X1, Y2", + "VMOVSLDUP.Z X1, X2", + "VMOVDDUP.Z.Z X1, K1, X2", + "VMOVDDUP X1, K0, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidduplicatemove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidduplicatemove": {Name: "invalidduplicatemove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's yxm/_yvmovddup tables", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedDwordMultiplyFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses yxm_q4 for PMULLD (X/m, X) and _yvandnpd for + // VPMULLD. The latter has VEX X/Y forms plus EVEX X/Y/Z forms with + // optional K1-K7 masking, .Z zero masking, and memory-only .BCST. + src := ` +TEXT packeddwordmultiplyforms(SB),NOSPLIT,$0-0 + PMULLD X1, X2 + PMULLD (AX), X2 + PMULLD source(SB), X2 + VPMULLD X1, X2, X3 + VPMULLD (AX), X2, X3 + VPMULLD Y1, Y2, Y3 + VPMULLD (AX), Y2, Y3 + VPMULLD X16, X17, X18 + VPMULLD X16, X17, K1, X18 + VPMULLD.Z (AX), X17, K1, X18 + VPMULLD Y16, Y17, Y18 + VPMULLD Y16, Y17, K1, Y18 + VPMULLD.Z (AX), Y17, K1, Y18 + VPMULLD Z1, Z2, Z3 + VPMULLD Z1, Z2, K1, Z3 + VPMULLD.Z source(SB), Z2, K1, Z3 + VPMULLD.BCST (AX), X17, X18 + VPMULLD.BCST (AX), Y17, K1, Y18 + VPMULLD.BCST.Z source(SB), Z2, K1, Z3 + RET +` + translateAMD64EcosystemCase(t, src, "packeddwordmultiplyforms") +} + +func TestTranslateAMD64PackedDwordMultiplyFamilyRejectsFormsOutsideGoOptabs(t *testing.T) { + for _, instruction := range []string{ + "PMULLD Y1, Y2", + "VPMULLD X1, Y2, Y3", + "VPMULLD X1, X2, K0, X3", + "VPMULLD.Z X1, X2, X3", + "VPMULLD.BCST X1, X2, K1, X3", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackeddwordmultiply(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackeddwordmultiply": {Name: "invalidpackeddwordmultiply", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMULLD/VPMULLD tables", instruction) + } + }) + } +} + +func TestTranslateAMD64MXCSRStateFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's ysvrs_mo/ysvrs_om and _yvldmxcsr tables give all four + // instructions exactly one memory operand. Exercise general, SP, SB, and + // named FP memory; VEX changes only encoding, not the operand shape. + src := ` +TEXT mxcsrstateforms(SB),NOSPLIT,$0-8 + LDMXCSR (AX) + LDMXCSR -8(SP) + LDMXCSR control(SB) + LDMXCSR input+0(FP) + VLDMXCSR (AX) + VLDMXCSR control(SB) + STMXCSR 4(BX) + STMXCSR -8(SP) + STMXCSR control(SB) + STMXCSR output+4(FP) + VSTMXCSR (AX) + VSTMXCSR output+4(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "mxcsrstateforms": { + Name: "mxcsrstateforms", + Args: []LLVMType{I32}, + Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 4, Type: I32, Index: 0, Field: -1}}, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "%local_stack = alloca") { + t.Fatal("amd64 SP-relative MXCSR memory did not receive byte-addressable local-stack backing") + } + for _, intrinsic := range []string{"@llvm.x86.sse.ldmxcsr", "@llvm.x86.sse.stmxcsr"} { + if !strings.Contains(ll, intrinsic) { + t.Fatalf("MXCSR lowering omitted %s", intrinsic) + } + } +} + +func TestTranslateAMD64MXCSRStateFamilyRejectsNonMemoryForms(t *testing.T) { + for _, instruction := range []string{ + "LDMXCSR AX", + "VLDMXCSR $1", + "STMXCSR X1", + "VSTMXCSR.Z (AX)", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidmxcsrstate(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidmxcsrstate": {Name: "invalidmxcsrstate", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, but Go's MXCSR tables require one memory operand", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedFloatToDwordFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's yxcvm1 table gives both instructions X/m -> X and X/m -> M. + // An X destination consumes four float32 lanes; an MMX destination consumes + // two lanes (and therefore only 64 bits for a memory source). + src := ` +TEXT packedfloattodwordforms(SB),NOSPLIT,$0-0 + CVTPS2PL X1, X2 + CVTPS2PL (AX), X2 + CVTPS2PL source(SB), X2 + CVTTPS2PL X1, X2 + CVTTPS2PL (AX), X2 + CVTTPS2PL source(SB), X2 + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedfloattodwordforms": {Name: "packedfloattodwordforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, intrinsic := range []string{"asm sideeffect \"cvtps2dq", "asm sideeffect \"cvttps2dq"} { + if !strings.Contains(ll, intrinsic) { + t.Fatalf("packed float-to-dword lowering omitted %s", intrinsic) + } + } +} + +func TestTranslateAMD64PackedFloatToDwordFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + for _, instruction := range []string{ + "CVTPS2PL Y1, X2", + "CVTPS2PL X1, Y2", + "CVTPS2PL $1, X2", + "CVTTPS2PL.Z X1, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedfloattodword(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedfloattodword": {Name: "invalidpackedfloattodword", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's yxcvm1 table", instruction) + } + }) + } +} + +func TestTranslateAMD64ConditionalMoveFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 has 48 entries sharing yml_rl: W/L/Q width crossed with all + // 16 x86 condition codes. Yml accepts a same-width GP register or memory; + // Yrl requires a GP register destination. Exercise both source categories + // and both ordinary and extended registers for every instruction. + conditions := []string{"CC", "CS", "EQ", "GE", "GT", "HI", "LE", "LS", "LT", "MI", "NE", "OC", "OS", "PC", "PL", "PS"} + var src strings.Builder + src.WriteString("TEXT conditionalmoveforms(SB),NOSPLIT,$0-8\n") + for _, width := range []string{"W", "L", "Q"} { + for _, condition := range conditions { + op := "CMOV" + width + condition + fmt.Fprintf(&src, "\t%s CX, DX\n", op) + fmt.Fprintf(&src, "\t%s (R10), R11\n", op) + } + } + src.WriteString("\tCMOVQPS source(SB), R10\n") + src.WriteString("\tCMOVQPS input+0(FP), R9\n") + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"conditionalmoveforms": { + Name: "conditionalmoveforms", + Args: []LLVMType{I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{ + Offset: 0, Type: I64, Index: 0, Field: -1, + }}}, + }}, + }); err != nil { + t.Fatal(err) + } +} + +func TestTranslateAMD64ConditionalMoveFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + for _, instruction := range []string{ + "CMOVQPS $1, AX", + "CMOVQPS AX, (BX)", + "CMOVQPS X1, AX", + "CMOVQPS.Z AX, BX", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidconditionalmove(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidconditionalmove": {Name: "invalidconditionalmove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, but Go 1.27's yml_rl requires register/memory source and register destination", instruction) + } + }) + } +} + +func TestTranslateAMD64PackedSingleToDoubleFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxm entry is X/m64 -> X. The VCVTPS2PD + // _yvcvtph2ps table adds X/m64 -> X, X/m128 -> Y, and Y/m256 -> Z, + // with EVEX masks/zeroing, memory broadcast, and SAE on the Z form. + src := ` +TEXT packedsingletodoubleforms(SB),NOSPLIT,$0-0 + CVTPS2PD X1, X2 + CVTPS2PD (AX), X2 + CVTPS2PD source(SB), X2 + VCVTPS2PD X1, X2 + VCVTPS2PD (AX), X2 + VCVTPS2PD X16, X17 + VCVTPS2PD X16, K1, X17 + VCVTPS2PD.Z (AX), K7, X17 + VCVTPS2PD.BCST (AX), X17 + VCVTPS2PD.BCST.Z source(SB), K1, X17 + VCVTPS2PD X1, Y2 + VCVTPS2PD (AX), Y2 + VCVTPS2PD X16, Y17 + VCVTPS2PD X16, K1, Y17 + VCVTPS2PD.Z (AX), K7, Y17 + VCVTPS2PD.BCST (AX), Y17 + VCVTPS2PD.BCST.Z source(SB), K1, Y17 + VCVTPS2PD Y1, Z2 + VCVTPS2PD (AX), Z2 + VCVTPS2PD Y16, Z17 + VCVTPS2PD Y16, K1, Z17 + VCVTPS2PD.Z (AX), K7, Z17 + VCVTPS2PD.BCST (AX), Z17 + VCVTPS2PD.BCST.Z source(SB), K1, Z17 + VCVTPS2PD.SAE Y16, Z17 + VCVTPS2PD.SAE.Z Y16, K1, Z17 + RET +` + ll := translateAMD64EcosystemCase(t, src, "packedsingletodoubleforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-single-to-double.ll", "packed-single-to-double.o", ll) +} + +func TestTranslateAMD64PackedSingleToDoubleFamilyRejectsFormsOutsideGoOptabs(t *testing.T) { + for _, instruction := range []string{ + "CVTPS2PD Y1, X2", + "CVTPS2PD.Z X1, X2", + "VCVTPS2PD Y1, Y2", + "VCVTPS2PD X1, Z2", + "VCVTPS2PD Z1, Z2", + "VCVTPS2PD X1, K0, X2", + "VCVTPS2PD.Z X1, X2", + "VCVTPS2PD.BCST X1, X2", + "VCVTPS2PD.SAE (AX), Z2", + "VCVTPS2PD.SAE X1, X2", + } { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + src := "TEXT invalidpackedsingletodouble(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "invalidpackedsingletodouble": {Name: "invalidpackedsingletodouble", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's CVTPS2PD/VCVTPS2PD tables", instruction) + } + }) + } +} + +func TestTranslateAMD64ComplexFrameBitwiseMoveForms(t *testing.T) { + // MOVSD and MOVUPS retain their ordinary yxmov X/m<->X formats when the + // memory operand names a complex64 or complex128 value in the Go FP frame. + src := ` +TEXT complexframemoves(SB),NOSPLIT,$0-48 + MOVSD input64+0(FP), X0 + MOVSD X0, result64+24(FP) + MOVUPS input128+8(FP), X1 + MOVUPS X1, result128+32(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "complexframemoves": { + Name: "complexframemoves", + Args: []LLVMType{"{ float, float }", "{ double, double }"}, + Ret: LLVMType("{ float, float, double, double }"), + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: LLVMType("float"), Index: 0, Field: 0}, + {Offset: 4, Type: LLVMType("float"), Index: 0, Field: 1}, + {Offset: 8, Type: LLVMType("double"), Index: 1, Field: 0}, + {Offset: 16, Type: LLVMType("double"), Index: 1, Field: 1}, + }, + Results: []FrameSlot{ + {Offset: 24, Type: LLVMType("float"), Index: 0, Field: -1}, + {Offset: 28, Type: LLVMType("float"), Index: 1, Field: -1}, + {Offset: 32, Type: LLVMType("double"), Index: 2, Field: -1}, + {Offset: 40, Type: LLVMType("double"), Index: 3, Field: -1}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "complex-frame-moves.ll", "complex-frame-moves.o", ll) +} + +func TestTranslateAMD64WritableFPParameterSlots(t *testing.T) { + // Go's assembler gives named FP parameter slots mutable frame semantics. + // Real assembly such as Gonum's GEMV kernels advances a slice base pointer + // by writing it back to base+off(FP), then reads the updated value later. + src := ` +TEXT writablefpparam(SB),NOSPLIT,$0-16 +body: + MOVQ input+0(FP), AX + ADDQ $8, AX + MOVQ AX, input+0(FP) + MOVQ input+0(FP), BX + MOVQ BX, result+8(FP) + RET +` + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "writablefpparam": { + Name: "writablefpparam", + Args: []LLVMType{Ptr}, + Ret: Ptr, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: Ptr, Index: 0, Field: -1}}, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "%fp_arg_0 = alloca ptr") { + t.Fatalf("writable FP parameter did not receive mutable shadow storage:\n%s", ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "writable-fp-param.ll", "writable-fp-param.o", ll) +} + +func TestTranslateAMD64ConditionalBranchAliasFamilyCompleteGoAssemblerForms(t *testing.T) { + // Keep this list aligned with the x86 aliases installed by + // cmd/asm/internal/arch.archX86 in Go 1.27. Every conditional branch uses + // yjcond: target, $0+target, and $1+target. + aliases := []string{ + "JA", "JAE", "JB", "JBE", "JC", "JCC", "JCS", + "JE", "JEQ", "JG", "JGE", "JGT", "JHI", "JHS", + "JL", "JLE", "JLO", "JLS", "JLT", "JMI", "JNA", + "JNAE", "JNB", "JNBE", "JNC", "JNE", "JNG", "JNGE", + "JNL", "JNLE", "JNO", "JNP", "JNS", "JNZ", "JO", + "JOC", "JOS", "JP", "JPC", "JPE", "JPL", "JPO", + "JPS", "JS", "JZ", + } + var src strings.Builder + src.WriteString("TEXT conditionalbranchaliases(SB),NOSPLIT,$0-0\n") + src.WriteString("\tUCOMISD X0, X1\n") + for i, alias := range aliases { + for hint := -1; hint <= 1; hint++ { + label := fmt.Sprintf("branch_alias_%d_%d", i, hint+1) + if hint < 0 { + fmt.Fprintf(&src, "\t%s %s\n", alias, label) + } else { + fmt.Fprintf(&src, "\t%s $%d, %s\n", alias, hint, label) + } + fmt.Fprintf(&src, "%s:\n", label) + } + } + src.WriteString("\tRET\n") + translateAMD64EcosystemCase(t, src.String(), "conditionalbranchaliases") +} + +func TestTranslateX86MSRAccessFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 assigns both RDMSR and WRMSR the operand-free ynone table on + // amd64 and 386. Their inputs/outputs are the architectural CX and DX:AX + // register tuple. + src := ` +TEXT msraccessforms(SB),NOSPLIT,$0-0 + MOVL $0x10, CX + RDMSR + WRMSR + RET +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "msraccessforms": {Name: "msraccessforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`asm sideeffect "rdmsr"`, `asm sideeffect "wrmsr"`} { + if !strings.Contains(ll, want) { + t.Fatalf("MSR access lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "msr-access.ll", "msr-access.o", ll) + }) + } +} + +func TestTranslateX86MSRAccessFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + for _, instruction := range []string{"RDMSR AX", "WRMSR CX", "RDMSR.P", "WRMSR.Z"} { + name := target.goarch + "_" + strings.NewReplacer(" ", "_", ".", "_").Replace(instruction) + t.Run(name, func(t *testing.T) { + src := "TEXT invalidmsraccess(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalidmsraccess": {Name: "invalidmsraccess", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s, but Go 1.27 uses ynone", instruction, target.goarch) + } + }) + } + } +} + +func TestTranslateX86DescriptorTableFamilyCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 groups the system descriptor instructions into these complete + // operand tables: LGDT/LIDT and SGDT/SIDT take memory; LLDT/LTR/LMSW take + // GP-register or memory inputs; SLDT*/SMSW*/STR* write GP registers or + // memory. Q forms exist only on amd64. + common := ` +TEXT descriptorforms(SB),NOSPLIT,$32-0 + LGDT (BX) + LIDT 8(SP) + SGDT (CX) + SIDT 16(SP) + LLDT AX + LLDT (BX) + LTR DI + LTR 8(SP) + LMSW CX + LMSW (DX) + SLDTW AX + SLDTW (BX) + SLDTL DI + SLDTL 8(SP) + SMSWW CX + SMSWW (DX) + SMSWL BP + SMSWL 16(SP) + STRW SI + STRW (DI) + STRL BX + STRL 24(SP) + STRL SP +` + amd64Only := ` + SLDTQ R11 + SLDTQ (R12) + SMSWQ R13 + SMSWQ (R14) + STRQ R15 + STRQ (R10) +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-macosx"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + src := common + if target.goarch == "amd64" { + src += amd64Only + } + src += "\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "descriptorforms": {Name: "descriptorforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `asm sideeffect "lgdt`, `asm sideeffect "lidt`, + `asm sideeffect "sgdt`, `asm sideeffect "sidt`, + `asm sideeffect "lldt`, `asm sideeffect "ltr`, `asm sideeffect "lmsw`, + `asm sideeffect "sldt`, `asm sideeffect "smsw`, `asm sideeffect "str`, + } { + if !strings.Contains(ll, want) { + t.Fatalf("descriptor-table lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "descriptor-table.ll", "descriptor-table.o", ll) + }) + } +} + +func TestTranslateX86DescriptorTableFamilyRejectsFormsOutsideGoOptab(t *testing.T) { + tests := []struct { + name string + goarch string + triple string + instruction string + }{ + {name: "lgdt-register", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LGDT AX"}, + {name: "sgdt-register", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SGDT R11"}, + {name: "lidt-immediate", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LIDT $1"}, + {name: "sidt-two-operands", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SIDT (AX), (BX)"}, + {name: "lldt-immediate", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LLDT $1"}, + {name: "ltr-vector", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LTR X0"}, + {name: "lmsw-two-operands", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LMSW AX, BX"}, + {name: "sldtw-immediate", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SLDTW $1"}, + {name: "smswl-vector", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SMSWL X0"}, + {name: "strq-two-operands", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "STRQ AX, BX"}, + {name: "lgdt-suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "LGDT.P (AX)"}, + {name: "sldtw-suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SLDTW.Z AX"}, + {name: "386-sldtq", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "SLDTQ AX"}, + {name: "386-smswq", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "SMSWQ AX"}, + {name: "386-strq", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "STRQ AX"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invaliddescriptor(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "invaliddescriptor": {Name: "invaliddescriptor", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside the Go 1.27 optab", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86MachineRegisterMoveFamilyCompleteGoAssemblerForms(t *testing.T) { + // This is the complete special-register portion of Go 1.27's ymovtab, + // excluding the independently handled segment and TLS moves. MOVQ special + // forms are accepted in 386 mode too; CR8 is the sole 64-bit-only register. + common := ` +TEXT machineregistermoves(SB),NOSPLIT,$32-0 + MOVL CR0, AX + MOVL BX, CR0 + MOVL CR2, CX + MOVL DX, CR2 + MOVL CR3, BP + MOVL SI, CR3 + MOVL CR4, DI + MOVL AX, CR4 + MOVQ CR0, BX + MOVQ CX, CR0 + MOVQ CR2, DX + MOVQ BP, CR2 + MOVQ CR3, SI + MOVQ DI, CR3 + MOVQ CR4, AX + MOVQ BX, CR4 + MOVL DR0, CX + MOVL DX, DR0 + MOVL DR6, BP + MOVL SI, DR6 + MOVL DR7, DI + MOVL AX, DR7 + MOVQ DR0, BX + MOVQ CX, DR0 + MOVQ DR2, DX + MOVQ BP, DR2 + MOVQ DR3, SI + MOVQ DI, DR3 + MOVQ DR6, AX + MOVQ BX, DR6 + MOVQ DR7, CX + MOVQ DX, DR7 + MOVL TR6, AX + MOVL TR6, 0(SP) + MOVL BX, TR6 + MOVL 4(SP), TR6 + MOVL TR7, CX + MOVL TR7, 8(SP) + MOVL DX, TR7 + MOVL 12(SP), TR7 + MOVL 0(SP), GDTR + MOVL GDTR, 16(SP) + MOVL 0(SP), IDTR + MOVL IDTR, 16(SP) + MOVQ 0(SP), GDTR + MOVQ GDTR, 16(SP) + MOVQ 0(SP), IDTR + MOVQ IDTR, 16(SP) + MOVW AX, LDTR + MOVW 0(SP), LDTR + MOVW LDTR, BX + MOVW LDTR, 0(SP) + MOVW CX, MSW + MOVW 4(SP), MSW + MOVW MSW, DX + MOVW MSW, 4(SP) + MOVW BP, TASK + MOVW 8(SP), TASK + MOVW TASK, SI + MOVW TASK, 8(SP) + MOVL CR0, SP + MOVQ CR0, SP +` + amd64Only := ` + MOVL CR8, R11 + MOVL R12, CR8 + MOVQ CR8, R13 + MOVQ R14, CR8 +` + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-macosx"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + src := common + if target.goarch == "amd64" { + src += amd64Only + } + src += "\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "machineregistermoves": {Name: "machineregistermoves", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"%cr0", "%dr0", ".byte 0x0f, 0x24", ".byte 0x0f, 0x26"} { + if !strings.Contains(ll, want) { + t.Fatalf("machine-register lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "machine-register.ll", "machine-register.o", ll) + }) + } +} + +func TestTranslateX86MachineRegisterMoveFamilyRejectsFormsOutsideGoMovtab(t *testing.T) { + tests := []struct { + name string + goarch string + triple string + instruction string + }{ + {name: "cr1", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ CR1, AX"}, + {name: "cr5", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVL AX, CR5"}, + {name: "cr-memory", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ CR0, (AX)"}, + {name: "386-cr8-l", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVL CR8, AX"}, + {name: "386-cr8-q", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVQ AX, CR8"}, + {name: "movl-dr2", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVL DR2, AX"}, + {name: "movq-dr1", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ AX, DR1"}, + {name: "dr-memory", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ (AX), DR0"}, + {name: "tr5", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVL TR5, AX"}, + {name: "tr-q", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ TR6, AX"}, + {name: "gdtr-register", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ AX, GDTR"}, + {name: "gdtr-width", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVW (AX), GDTR"}, + {name: "ldtr-width", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVL AX, LDTR"}, + {name: "machine-suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVQ.P CR0, AX"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invalidmachineregistermove(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "invalidmachineregistermove": {Name: "invalidmachineregistermove", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's ymovtab", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86ScalarExtensionMoveCompleteGo127Forms(t *testing.T) { + amd64Ops := []string{ + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", "MOVBQSX", "MOVBQZX", + "MOVWLSX", "MOVWLZX", "MOVWQSX", "MOVWQZX", "MOVLQSX", "MOVLQZX", + "MOVSWW", "MOVZWW", + } + ops386 := []string{ + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", + "MOVWLSX", "MOVWLZX", "MOVLQZX", "MOVSWW", "MOVZWW", + } + for _, target := range []struct { + name string + goarch string + triple string + ops []string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", ops: amd64Ops}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", ops: amd64Ops}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", ops: amd64Ops}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", ops: ops386}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", ops: ops386}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT scalar_extension_forms(SB),NOSPLIT,$0-1\n") + for _, op := range target.ops { + source := "AX" + if strings.HasPrefix(op, "MOVB") { + source = "AL" + if target.goarch == "386" { + source = "BP" // Go rewrites this Yrl32 byte source through BX. + } + } + fmt.Fprintf(&src, "\t%s %s, CX\n", op, source) + fmt.Fprintf(&src, "\t%s 8(BX), DX\n", op) + } + src.WriteString("\tMOVBWZX x+0(FP), SP\n\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "scalar_extension_forms": { + Name: "scalar_extension_forms", Args: []LLVMType{I8}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scalar-extension-"+target.name+".ll", "scalar-extension-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ScalarExtensionMoveRejectsFormsOutsideGo127Tables(t *testing.T) { + tests := []struct { + name string + goarch string + triple string + instruction string + }{ + {name: "missing-source", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBWZX AX"}, + {name: "too-many", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBLZX AL, AX, BX"}, + {name: "immediate-source", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBQZX $1, AX"}, + {name: "vector-source", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBWZX X1, AX"}, + {name: "byte-source-for-word-class", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVWLZX AL, AX"}, + {name: "memory-destination", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVBLSX AL, (AX)"}, + {name: "byte-destination", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVWQZX AX, AL"}, + {name: "suffix", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "MOVLQZX.Z AX, BX"}, + {name: "386-byte-to-qword-sign", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBQSX AL, AX"}, + {name: "386-byte-to-qword-zero", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBQZX AL, AX"}, + {name: "386-word-to-qword-sign", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVWQSX AX, BX"}, + {name: "386-word-to-qword-zero", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVWQZX AX, BX"}, + {name: "386-long-to-qword-sign", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVLQSX AX, BX"}, + {name: "386-r8-source", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVWLZX R8, AX"}, + {name: "386-r8-destination", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBLZX AL, R8"}, + {name: "386-sp-byte-source", goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "MOVBWZX SP, AX"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invalid_scalar_extension(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "invalid_scalar_extension": {Name: "invalid_scalar_extension", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's ymb_rl/yml_rl tables", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86MaskLogicalCompleteGo127Forms(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT mask_logical_forms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"KAND", "KANDN", "KOR", "KXNOR", "KXOR"} { + for _, width := range []string{"B", "W", "D", "Q"} { + fmt.Fprintf(&src, "\t%s%s K1, K2, K3\n", stem, width) + } + } + for _, stem := range []string{"KNOT", "KTEST", "KORTEST"} { + for _, width := range []string{"B", "W", "D", "Q"} { + fmt.Fprintf(&src, "\t%s%s K1, K2\n", stem, width) + } + } + src.WriteString("\tRET\n") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "mask_logical_forms": {Name: "mask_logical_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mask-logical-"+target.name+".ll", "mask-logical-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86MaskLogicalRejectsFormsOutsideGo127Tables(t *testing.T) { + for _, instruction := range []string{ + "KXNORW K1, K2", + "KXNORW K1, K2, K3, K4", + "KANDW K1, K2, AX", + "KANDNW K1, (AX), K3", + "KORQ $1, K2, K3", + "KXORB K1, K2, K8", + "KNOTD K1", + "KNOTQ K1, K2, K3", + "KNOTW AX, K2", + "KTESTB K1, (AX)", + "KTESTW K1, K2, K3", + "KORTESTD K1", + "KORTESTQ.Z K1, K2", + "KXORQ.Z K1, K2, K3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalid_mask_logical(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalid_mask_logical": {Name: "invalid_mask_logical", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's _ykaddb/_yknotb tables", instruction, target.goarch) + } + } + }) + } +} + +func TestTranslateX86MaskMoveCompleteGo127Forms(t *testing.T) { + // Go 1.27 assigns KMOVB/W/D/Q the same _ykmovb table. Its four rows are: + // K -> memory, K -> GP, K-or-memory -> K, and GP -> K. Ykm therefore also + // includes the K -> K form. Exercise every row plus each Plan 9 memory + // spelling accepted as Ym, on both x86 architectures and all CI object + // formats. + var src strings.Builder + src.WriteString("TEXT mask_move_forms(SB),NOSPLIT,$0-16\n") + for _, op := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + fmt.Fprintf(&src, "\t%s K1, 16(BX)\n", op) + fmt.Fprintf(&src, "\t%s K1, AX\n", op) + fmt.Fprintf(&src, "\t%s K1, K2\n", op) + fmt.Fprintf(&src, "\t%s K0, K7\n", op) + fmt.Fprintf(&src, "\t%s 16(BX), K2\n", op) + fmt.Fprintf(&src, "\t%s AX, K2\n", op) + fmt.Fprintf(&src, "\t%s K5, SP\n", op) + fmt.Fprintf(&src, "\t%s SP, K5\n", op) + fmt.Fprintf(&src, "\t%s x+0(FP), K3\n", op) + fmt.Fprintf(&src, "\t%s K3, ret+8(FP)\n", op) + fmt.Fprintf(&src, "\t%s mask_move_source(SB), K4\n", op) + fmt.Fprintf(&src, "\t%s K4, mask_move_destination(SB)\n", op) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + targetSource := src.String() + if target.goarch == "amd64" { + var highRegisters strings.Builder + for _, op := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + fmt.Fprintf(&highRegisters, "\t%s K6, R15\n", op) + fmt.Fprintf(&highRegisters, "\t%s R15, K6\n", op) + } + targetSource += highRegisters.String() + } + targetSource += "\tRET\n" + file, err := Parse(ArchAMD64, targetSource) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "mask_move_forms": { + Name: "mask_move_forms", Args: []LLVMType{I64}, Ret: I64, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mask-move-"+target.name+".ll", "mask-move-"+target.name+".o", ll) + }) + } +} + +func TestTranslate386MaskMoveRejectsAMD64OnlyYrlRegisters(t *testing.T) { + for _, instruction := range []string{"KMOVB K1, R8", "KMOVW R8, K1", "KMOVD K1, R15", "KMOVQ R15, K1"} { + src := "TEXT invalid_mask_move_386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalid_mask_move_386": {Name: "invalid_mask_move_386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for 386 outside Go 1.27's Yrl class", instruction) + } + } +} + +func TestTranslateX86MaskMoveRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "KMOVB K1", + "KMOVW K1, K2, K3", + "KMOVD $1, K1", + "KMOVQ K1, $1", + "KMOVB 8(AX), BX", + "KMOVW AX, 8(BX)", + "KMOVD AX, BX", + "KMOVQ 8(AX), 16(BX)", + "KMOVB X1, K2", + "KMOVW K1, X2", + "KMOVD K8, AX", + "KMOVQ AX, K8", + "KMOVQ.Z K1, K2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT invalid_mask_move(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalid_mask_move": {Name: "invalid_mask_move", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's _ykmovb table", instruction, target.goarch) + } + } + }) + } +} + +func TestTranslateX86GatherCompleteGo127Forms(t *testing.T) { + // Go 1.27 has eight data-gather instructions over three operand tables, + // plus eight EVEX gather-prefetch instructions over two tables. Generate + // every row of all five tables for each opcode. The 386 assembler accepts + // EVEX gather with X/Y/Z0..7 VSIB indices; amd64 additionally accepts the + // high EVEX register classes. + dataGroups := []struct { + ops []string + table string + }{ + {ops: []string{"VGATHERDPS", "VGATHERQPD", "VPGATHERDD", "VPGATHERQQ"}, table: "dps"}, + {ops: []string{"VGATHERDPD", "VPGATHERDQ"}, table: "dpd"}, + {ops: []string{"VGATHERQPS", "VPGATHERQD"}, table: "qps"}, + } + prefetchGroups := []struct { + ops []string + indexWidth string + }{ + {ops: []string{"VGATHERPF0DPD", "VGATHERPF1DPD"}, indexWidth: "Y"}, + {ops: []string{ + "VGATHERPF0DPS", "VGATHERPF0QPD", "VGATHERPF0QPS", + "VGATHERPF1DPS", "VGATHERPF1QPD", "VGATHERPF1QPS", + }, indexWidth: "Z"}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT gather_forms(SB),NOSPLIT,$0-0\n") + for _, group := range dataGroups { + for _, op := range group.ops { + switch group.table { + case "dps": + fmt.Fprintf(&src, "\t%s X1, (BX)(X2*1), X3\n", op) + fmt.Fprintf(&src, "\t%s Y1, 4(BX)(Y2*2), Y3\n", op) + fmt.Fprintf(&src, "\t%s 8(X2*4), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 12(BX)(Y2*8), K1, Y3\n", op) + fmt.Fprintf(&src, "\t%s 16(BX)(Z2*4), K1, Z3\n", op) + case "dpd": + fmt.Fprintf(&src, "\t%s X1, (BX)(X2*1), X3\n", op) + fmt.Fprintf(&src, "\t%s Y1, 8(BX)(X2*2), Y3\n", op) + fmt.Fprintf(&src, "\t%s 16(X2*4), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 24(BX)(X2*8), K1, Y3\n", op) + fmt.Fprintf(&src, "\t%s 32(BX)(Y2*4), K1, Z3\n", op) + case "qps": + fmt.Fprintf(&src, "\t%s X1, (BX)(X2*1), X3\n", op) + fmt.Fprintf(&src, "\t%s X1, 4(BX)(Y2*2), X3\n", op) + fmt.Fprintf(&src, "\t%s 8(X2*4), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 12(BX)(Y2*8), K1, X3\n", op) + fmt.Fprintf(&src, "\t%s 16(BX)(Z2*4), K1, Y3\n", op) + } + // Unlike VSIB indices, Go 1.27 permits high EVEX X/Y + // destination registers even in 386 mode. + switch group.table { + case "dps": + fmt.Fprintf(&src, "\t%s (BX)(X2*4), K2, X20\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y2*4), K2, Y20\n", op) + case "dpd": + fmt.Fprintf(&src, "\t%s (BX)(X2*4), K2, X20\n", op) + fmt.Fprintf(&src, "\t%s (BX)(X2*4), K2, Y20\n", op) + case "qps": + fmt.Fprintf(&src, "\t%s (BX)(X2*8), K2, X20\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Z2*8), K2, Y20\n", op) + } + if target.goarch == "amd64" { + switch group.table { + case "dps": + fmt.Fprintf(&src, "\t%s (BX)(X20*4), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y20*4), K2, Y21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Z20*4), K2, Z21\n", op) + case "dpd": + fmt.Fprintf(&src, "\t%s (BX)(X20*4), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(X20*4), K2, Y21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y20*4), K2, Z21\n", op) + case "qps": + fmt.Fprintf(&src, "\t%s (BX)(X20*8), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Y20*8), K2, X21\n", op) + fmt.Fprintf(&src, "\t%s (BX)(Z20*8), K2, Y21\n", op) + } + } + } + } + for _, group := range prefetchGroups { + for _, op := range group.ops { + fmt.Fprintf(&src, "\t%s K1, 40(BX)(%s2*4)\n", op, group.indexWidth) + if target.goarch == "amd64" { + fmt.Fprintf(&src, "\t%s K7, 48(%s20*8)\n", op, group.indexWidth) + } + } + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "gather_forms": {Name: "gather_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "gather-"+target.name+".ll", "gather-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86GatherRejectsFormsOutsideGo127Tables(t *testing.T) { + tests := []struct { + name string + instruction string + }{ + {name: "missing-mask", instruction: "VPGATHERDD (AX)(X2*4), X3"}, + {name: "too-many", instruction: "VGATHERDPS (AX)(X2*4), K1, K2, X3"}, + {name: "plain-memory", instruction: "VPGATHERDD (AX), K1, X3"}, + {name: "gp-index", instruction: "VPGATHERDD (AX)(CX*4), K1, X3"}, + {name: "bad-scale", instruction: "VPGATHERDD (AX)(X2*3), K1, X3"}, + {name: "k0-mask", instruction: "VPGATHERDD (AX)(X2*4), K0, X3"}, + {name: "vector-mask-in-evex", instruction: "VPGATHERDD (AX)(X2*4), X1, X3"}, + {name: "k-mask-in-vex", instruction: "VPGATHERDD K1, (AX)(X2*4), X3"}, + {name: "dps-wrong-index", instruction: "VPGATHERDD (AX)(Y2*4), K1, X3"}, + {name: "dps-wrong-destination", instruction: "VPGATHERDD (AX)(X2*4), K1, Y3"}, + {name: "dpd-wrong-index", instruction: "VPGATHERDQ (AX)(Y2*4), K1, Y3"}, + {name: "dpd-wrong-destination", instruction: "VPGATHERDQ (AX)(Y2*4), K1, X3"}, + {name: "qps-wrong-index", instruction: "VPGATHERQD (AX)(Y2*8), K1, Y3"}, + {name: "qps-wrong-destination", instruction: "VPGATHERQD (AX)(Z2*8), K1, Z3"}, + {name: "vex-mask-index-alias", instruction: "VPGATHERDD X2, (AX)(X2*4), X3"}, + {name: "vex-mask-dest-alias", instruction: "VPGATHERDD X3, (AX)(X2*4), X3"}, + {name: "vex-index-dest-alias", instruction: "VPGATHERDD X1, (AX)(X3*4), X3"}, + {name: "evex-index-dest-alias", instruction: "VPGATHERDD (AX)(X3*4), K1, X3"}, + {name: "zero-suffix", instruction: "VPGATHERDD.Z (AX)(X2*4), K1, X3"}, + {name: "broadcast-suffix", instruction: "VPGATHERQQ.BCST (AX)(X2*8), K1, X3"}, + {name: "prefetch-k0", instruction: "VGATHERPF0DPD K0, (AX)(Y2*8)"}, + {name: "prefetch-wrong-index", instruction: "VGATHERPF0DPD K1, (AX)(Z2*8)"}, + {name: "prefetch-extra-dest", instruction: "VGATHERPF1DPS K1, (AX)(Z2*4), Z3"}, + {name: "prefetch-suffix", instruction: "VGATHERPF1QPS.Z K1, (AX)(Z2*8)"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + src := "TEXT invalid_gather(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + if _, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "invalid_gather": {Name: "invalid_gather", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q for %s outside Go 1.27's gather tables", test.instruction, target.goarch) + } + } + }) + } +} + +func TestTranslate386GatherRejectsHighVSIBAndZRegisters(t *testing.T) { + for _, instruction := range []string{ + "VPGATHERDD (AX)(X8*4), K1, X3", + "VPGATHERDQ (AX)(Y8*4), K1, Z3", + "VPGATHERQD (AX)(Z8*8), K1, Y3", + "VPGATHERDD (AX)(Z2*4), K1, Z8", + "VGATHERPF0DPD K1, (AX)(Y8*8)", + "VGATHERPF1DPS K1, (AX)(Z8*4)", + } { + src := "TEXT invalid_gather_386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "invalid_gather_386": {Name: "invalid_gather_386", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's 386 VSIB/register classes", instruction) + } + } +} + +func TestTranslateAMD64TernaryLogicCompleteGo127Forms(t *testing.T) { + // VPTERNLOGD/Q share Go 1.27's six-row _yvalignd table: X/Y/Z, + // each with an unmasked and K1-K7 masked row. Their EVEX encodings add + // scalar D/Q broadcast and zeroing. Exercise every row/suffix combination, + // high EVEX registers, and every value in the unsigned-imm8 truth table. + var src strings.Builder + src.WriteString("TEXT ternary_logic_forms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPTERNLOGD", "VPTERNLOGQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s $0, %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s $255, 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&src, "\t%s $0xd8, %s5, %s6, K1, %s7\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s.Z $0xaa, 16(BX), %s8, K2, %s9\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST $0xcc, 24(BX), %s10, %s11\n", op, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z $0xf0, 32(BX), %s12, K3, %s13\n", op, width, width) + } + fmt.Fprintf(&src, "\t%s $0x96, X29, X30, X31\n", op) + fmt.Fprintf(&src, "\t%s $0x96, Y29, Y30, Y31\n", op) + fmt.Fprintf(&src, "\t%s $0x96, Z29, Z30, Z31\n", op) + } + for immediate := 0; immediate <= 255; immediate++ { + fmt.Fprintf(&src, "\tVPTERNLOGD $%d, X0, X1, X2\n", immediate) + fmt.Fprintf(&src, "\tVPTERNLOGQ $%d, X3, X4, X5\n", immediate) + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "ternary_logic_forms": {Name: "ternary_logic_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "ternary-logic-"+target.name+".ll", "ternary-logic-"+target.name+".o", ll) + }) + } +} + +func TestAMD64TernaryLogicTruthTableOrderingExhaustive(t *testing.T) { + for immediate := 0; immediate <= 255; immediate++ { + for truthIndex := 0; truthIndex < 8; truthIndex++ { + a := truthIndex&4 != 0 + b := truthIndex&2 != 0 + third := truthIndex&1 != 0 + want := immediate&(1< +#include +#include +#include +#include +#include +#include +#include +%s +%s +typedef uint64_t (*Program)(const uint8_t*); +typedef struct { + Program translated, original; + unsigned width, destination, sign, dead; +} Kernel; +static const Kernel kernels[]={%s}; +static const uint64_t seed=UINT64_C(0x87654321fedcba98); + +static uint64_t reference(Kernel k, const uint8_t *p) { + if(k.dead) return seed; + uint64_t value=0; + for(unsigned b=0;b(SB)", "1234"} { + source := fmt.Sprintf("TEXT f(SB),4,$0-0\n%s %s,AX\nRET\n", row.op, address) + goAssemblerOracleOutput(t, arch, source, "", true) + control := ins + control.Args = []Operand{{Kind: OpSym, Sym: address}, {Kind: OpReg, Reg: AX}} + control.Raw = row.op + " " + address + ",AX" + if _, known, err := x86ScalarExtensionProperties(control.Op, control, arch); !known || err != nil { + t.Fatal("valid SB/absolute source lost its existing grammar", address, err) + } + } + }) + } + } +} diff --git a/amd64_fixed_local_aes.go b/amd64_fixed_local_aes.go new file mode 100644 index 000000000..b80f02012 --- /dev/null +++ b/amd64_fixed_local_aes.go @@ -0,0 +1,31 @@ +package plan9asm + +func fixedLocalAESByteStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + f, known, err := x86AESProperties(ins.Op, *ins, "amd64") + if !known { + return false, nil + } + if err != nil || f.byteWidth != 16 { + return true, goAMD64FrameContext("fixed local AES shape is not closed") + } + width := uint64(65535) + source := state.vectors[x86CallerVectorRoot(f.sourceOperand.Reg)] & width + if f.memory >= 0 { + if _, _, err := fixedLocalReadMemory(ins, f.memory, state, proof, residue, 16, 1); err != nil { + return true, err + } + source = width + } + input := uint64(0) + if f.state >= 0 { + input = state.vectors[x86CallerVectorRoot(ins.Args[f.state].Reg)] & width + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + defined := x86AESDefinedBytes(f, source, input) + if !f.spec.vector { + state.vectors[destination] = state.vectors[destination]&^width | defined + } else { + state.vectors[destination] = ^width | defined + } + return true, nil +} diff --git a/amd64_fixed_local_auto.go b/amd64_fixed_local_auto.go new file mode 100644 index 000000000..b25612c43 --- /dev/null +++ b/amd64_fixed_local_auto.go @@ -0,0 +1,127 @@ +package plan9asm + +import ( + "math" + "reflect" + "strings" +) + +// NAME_AUTO is a source address role, not an input/storage permission. Go +// obj6 adds deltasp-bpsize to its explicit offset. This bounded observer has +// no CALL/PUSH/POP/ADJSP producer, so the lexical bias is exactly TEXT's local +// size; separately proved SP relocation still contributes its native offset. +// Replay the original operand because parsing constant expressions alone +// loses the difference between tmp-8(SP) and physical -8(SP). +func fixedLocalMemoryAddress(ins Instr, index int, frame int64) (MemRef, error) { + fail := func() (MemRef, error) { + return MemRef{}, goAMD64FrameContext("fixed local original memory address role is not closed") + } + if index < 0 || index >= len(ins.Args) || ins.Args[index].Kind != OpMem { + return fail() + } + memory := ins.Args[index].Mem + if memory.Base != SP { + if memory.OffRaw != "" { + return fail() + } + return memory, nil + } + op, rest := splitOpcode(ins.Raw) + literals := splitTopLevelCSV(rest) + if !strings.EqualFold(op, string(ins.Op)) || len(literals) != len(ins.Args) { + return fail() + } + parsed, err := parseOperandForArch(ArchAMD64, literals[index]) + if err != nil || !reflect.DeepEqual(parsed, ins.Args[index]) { + return fail() + } + literal := x86ResultAUTOLiteral.FindStringSubmatch(strings.TrimSpace(literals[index])) + if literal == nil { + // Indexing/segment/symbol roles are independently excluded by the + // native observer; none may be relabeled as a named local offset. + if memory.OffRaw != "" { + return fail() + } + return memory, nil + } + prefix := strings.TrimSpace(literal[1]) + offset, named := parseNamedStackConstantOffset(prefix) + if _, bare := parseIdent(prefix); bare { + offset, named = 0, true + } + if !named { + if memory.OffRaw != "" { + return fail() + } + return memory, nil // Numeric, unnamed (SP) remains post-prologue SP. + } + if offset != memory.Off || frame <= 0 || offset > 0 && frame > math.MaxInt64-offset { + return fail() + } + memory.Off += frame + memory.OffRaw = "" + return memory, nil +} + +// Descriptive bare NAME_AUTO is offset zero, not an undefined preprocessor +// expression. This syntax query does not grant a native memory certificate. +func x86ConditionalMoveOffsetKnown(memory MemRef) bool { + if memory.OffRaw == "" { + return true + } + if memory.Base != SP || memory.Sym != "" { + return false + } + if offset, named := parseNamedStackConstantOffset(memory.OffRaw); named { + return offset == memory.Off + } + _, bare := parseIdent(memory.OffRaw) + return bare && memory.Off == 0 +} + +func fixedLocalConditionalMoveStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + effect, known, err := x86TypedConditionalMoveByteEffect(ins.Op, *ins, "amd64") + if !known { + return false, nil + } + fail := func() (bool, error) { + return true, goAMD64FrameContext("fixed local CMOV flags or potential numerical source bytes are not defined") + } + if err != nil || !state.flags || len(effect.operands) != 2 { + return fail() + } + source, destination := effect.operands[0], effect.operands[1] + if !fixedLocalNumericRead(destination.operand, destination.width, state) || x86SourceGRegister(destination.operand.Reg) { + return fail() + } + literal := false + if source.operand.Kind == OpMem || source.operand.Kind == OpSym { + var err error + _, literal, err = fixedLocalReadMemory(ins, source.index, state, proof, residue, source.width, 1) + if err != nil { + return true, err + } + } else if !fixedLocalNumericRead(source.operand, source.width, state) { + return fail() // FP/caller slots and private pointer bits are not imports. + } + root, _ := x86CallerGP(destination.operand.Reg) + booleans := fixedLocalBooleanRead(destination.operand, destination.width, state) + if source.operand.Kind == OpReg { + booleans &= fixedLocalBooleanRead(source.operand, source.width, state) + } else if literal { + booleans = 0 // Numerical literal bytes are not a boolean proof. + } else { + // The source read was proved above; its actual private offset is + // retained in this same residue's instruction-bound certificate. + access := ins.x86FixedLocalMemory[source.index] + booleans &= fixedLocalMemoryBooleans(state, access.offset[residue], source.width) + } + fixedLocalWriteBooleans(state, destination.operand.Reg, destination.width, booleans) + mask := x86CallerByteMask(destination.operand.Reg, destination.width) + if destination.width == 4 { + mask = 255 // False CMOVL still zero-defines the upper half. + } + state.known[root] |= mask + delete(state.constants, root) // Flags are unchanged, but selected values are not constants. + return true, nil +} diff --git a/amd64_fixed_local_binding.go b/amd64_fixed_local_binding.go new file mode 100644 index 000000000..529b609bd --- /dev/null +++ b/amd64_fixed_local_binding.go @@ -0,0 +1,381 @@ +package plan9asm + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "path/filepath" + "reflect" + "strings" +) + +// The public value transports only a private live observer binding. Neither +// a declaration, a frame size nor a JSON round trip can construct this proof. +type x86FixedLocalBinding struct { + identity *x86FixedLocalBinding + source *GoAMD64FrameSource + context context.Context + file [32]byte + assembly string + target string + inputs map[string][32]byte + signatures map[string]FuncSig + symbols map[string]string + declarations map[string]goAMD64FrameDeclaration + numericalReads *GoX86NumericalReadBinding + literalData map[string]x86FixedLocalData +} + +// BindFixedLocal binds a complete original typed ABI0 file to its actual Go +// object observation. It does not support dynamic incoming frames, pointees, +// imported declarations, native entries or arbitrary local-memory effects. +// The returned Options are independent copies; their private signatures may +// also be transported by GoModuleOptions.ManualSig. The capture context must +// remain live through translation. Close or any failed Verify revokes them. +func (source *GoAMD64FrameSource) BindFixedLocal(ctx context.Context, pkg GoPackage, file *File, options Options) (Options, error) { + if err := source.Verify(ctx); err != nil { + return Options{}, err + } + if file == nil || options.ResolveSym == nil { + return Options{}, goAMD64FrameContext("fixed local binding needs complete source and symbol resolution") + } + originalOptions := options + numerical := options.NumericalReads + if numerical != nil { + if err := verifyNumericalReadInputs(file, options); err != nil { + return Options{}, err + } + source.mu.Lock() + joined := source.numericalBinding == numerical + source.mu.Unlock() + if !joined || ctx != numerical.context || pkg.Types != numerical.originalTypes || + !numericalSameImports(pkg.Imports, numerical.originalImports) { + return Options{}, goAMD64FrameContext("fixed local source has a foreign numerical read owner") + } + } + // Never execute a caller callback while holding the observer lock. Freeze + // its answers before the locked full-source/declaration comparison; that + // comparison rejects any mutation made while resolving these names. + resolved := make(map[string]string, len(file.Funcs)) + for _, fn := range file.Funcs { + resolved[fn.Sym] = options.ResolveSym(fn.Sym) + } + for _, global := range file.Globl { + if name, valid := fixedLocalDataSymbol(global.Sym); valid { + resolved[name] = options.ResolveSym(name) + } + } + options.ResolveSym = func(symbol string) string { return resolved[symbol] } + if numerical != nil { + // The caller has been checked; the locked observation consumes only + // sealed answers, without re-entering numerical/owner callbacks. + options.ResolveSym = numerical.resolve + } + source.mu.Lock() + bound, err := source.bindFixedLocalLocked(ctx, pkg, file, options) + source.mu.Unlock() + if err != nil { + return Options{}, err + } + if err := source.Verify(ctx); err != nil { + return Options{}, err + } + if numerical != nil { + if err := verifyNumericalReadInputs(file, originalOptions); err != nil { + return Options{}, err + } + // Bind a fresh numerical token to the independently established local + // capability. The frame keeps its original token; no token is mutated. + bound.NumericalReads = nil + return numerical.source.BindNumericalReads(ctx, pkg, file, bound) + } + return bound, nil +} + +func (source *GoAMD64FrameSource) bindFixedLocalLocked(ctx context.Context, pkg GoPackage, file *File, options Options) (Options, error) { + if !source.ready || ctx == nil || ctx.Err() != nil || file == nil || file.Arch != ArchAMD64 || + options.Goarch != "amd64" || pkg.Types == nil || pkg.Path != source.selection.ImportPath || pkg.Types.Path() != pkg.Path { + return Options{}, goAMD64FrameContext("fixed local binding has no matching live package/target") + } + targets := map[string]string{ + "x86_64-unknown-linux-gnu": "linux/amd64", + "x86_64-apple-darwin": "darwin/amd64", + "x86_64-apple-macosx": "darwin/amd64", + "x86_64-pc-windows-msvc": "windows/amd64", + } + if targets[options.TargetTriple] != source.request.Target || options.WASMABI != WASMABIDirect || + len(file.UnlinkedPrelude)+len(options.X86TailGroups) != 0 { + return Options{}, goAMD64FrameContext("fixed local binding target or complete source roles disagree") + } + declarations, err := source.typedDeclarations(pkg, false) + if err != nil { + return Options{}, err + } + original, err := goAMD64FrameRead(filepath.Join(source.request.Directory, source.request.AssemblyFile)) + if err != nil { + return Options{}, err + } + expanded, err := source.preprocess(source.request.AssemblyFile, original) + if err != nil { + return Options{}, err + } + actual, err := ParseWithDefines(ArchAMD64, expanded, nil) + if err != nil { + return Options{}, err + } + fingerprint, err := x86CallerSourceFingerprint(actual) + if err != nil { + return Options{}, err + } + provided, err := x86CallerSourceFingerprint(file) + if err != nil || provided != fingerprint || len(options.Sigs) != len(actual.Funcs) { + return Options{}, goAMD64FrameContext("fixed local binding requires the exact full original parsed file") + } + resolve := options.ResolveSym + if resolve == nil { + return Options{}, goAMD64FrameContext("fixed local binding needs actual package symbol resolution") + } + binding := &x86FixedLocalBinding{ + source: source, + context: ctx, + file: fingerprint, + assembly: expanded, + target: options.TargetTriple, + inputs: make(map[string][32]byte, len(source.inputs)), + signatures: make(map[string]FuncSig), + symbols: make(map[string]string), + declarations: make(map[string]goAMD64FrameDeclaration), + numericalReads: options.NumericalReads, + } + binding.identity = binding + literalData, dataSymbols, err := source.fixedLocalData(actual, expanded, resolve) + if err != nil { + return Options{}, err + } + binding.literalData = literalData + for symbol, name := range dataSymbols { + binding.symbols[symbol] = name + } + if options.NumericalReads != nil { + if err := joinNumericalReadCertificates(actual, options); err != nil { + return Options{}, err + } + } + for name, sum := range source.inputs { + binding.inputs[name] = sum + } + options.Sigs, err = cloneFixedLocalSigs(options.Sigs) + if err != nil { + return Options{}, err + } + actualSignatures, err := cloneFixedLocalSigs(options.Sigs) + if err != nil { + return Options{}, err + } + for _, fn := range actual.Funcs { + name := resolve(fn.Sym) + observed, found := source.functions[name] + declaration, declared := declarations[name] + expectedName := pkg.Path + "." + strings.TrimPrefix(fn.Sym, "·") + sig, supplied := options.Sigs[name] + if !found || !supplied || name != expectedName || len(observed.code) == 0 || + observed.locals != fn.FrameSize+8 || observed.args != fn.ArgSize || + !reflect.DeepEqual(sig, observed.signature) || !declared || + !source.matchesFixedLocalDeclaration(name, declaration) { + return Options{}, goAMD64FrameContext("fixed local actual declaration/object/symbol mismatch: %s", fn.Sym) + } + proofSig := sig + proofSig.x86FixedLocal = binding + if _, err := proveX86FixedLocal(fn, proofSig); err != nil { + return Options{}, err + } + binding.signatures[name] = actualSignatures[name] + // Preserve the independently re-typechecked language identity, not + // merely the collapsed LLVM signature. Neither observation-only + // composite roles nor stale private metadata can replace this witness. + declaration.signature = actualSignatures[name] + binding.declarations[name] = declaration + binding.symbols[fn.Sym] = name + sig.x86FixedLocal = binding + options.Sigs[name] = sig + } + return options, nil +} + +// After validating mutable public input, consume only independent immutable +// copies. A stateful resolver or a mutation of the caller's Options map cannot +// change the declaration or function name between normalization and emission. +func freezeX86FixedLocalOptions(options Options, bindings []*x86FixedLocalBinding) (Options, error) { + if len(bindings) == 0 { + return options, nil + } + symbols := make(map[string]string) + options.Sigs = make(map[string]FuncSig) + for _, binding := range bindings { + for symbol, name := range binding.symbols { + symbols[symbol] = name + } + signatures, err := cloneFixedLocalSigs(binding.signatures) + if err != nil { + return Options{}, err + } + for name, sig := range signatures { + sig.x86FixedLocal = binding + options.Sigs[name] = sig + } + } + options.ResolveSym = func(symbol string) string { + if name, found := symbols[symbol]; found { + return name + } + if options.NumericalReads != nil { + return options.NumericalReads.resolve(symbol) + } + return symbol // No source contract admits external calls or addresses. + } + return options, nil +} + +func freezeX86FixedLocalFile(file *File, bindings []*x86FixedLocalBinding) (*File, error) { + if len(bindings) == 0 { + return file, nil + } + if len(bindings) != 1 { + return nil, goAMD64FrameContext("fixed local complete file has multiple source owners") + } + binding := bindings[0] + fingerprint, err := x86CallerSourceFingerprint(file) + if err != nil || fingerprint != binding.file { + return nil, goAMD64FrameContext("fixed local resolver changed the original source during validation") + } + // Parsing the guarded observer's immutable expanded bytes gives the + // consumer independent Func/Instr/Operand slices with no caller aliases + // or pre-existing private certificates. + actual, err := ParseWithDefines(ArchAMD64, binding.assembly, nil) + if err != nil { + return nil, err + } + fingerprint, err = x86CallerSourceFingerprint(actual) + if err != nil || fingerprint != binding.file { + return nil, goAMD64FrameContext("fixed local sealed source snapshot changed") + } + return actual, nil +} + +func cloneFixedLocalSigs(input map[string]FuncSig) (map[string]FuncSig, error) { + // The source signatures are JSON-encodable public ABI metadata. Reusing + // neither caller slices nor nested FrameSlot fields prevents mutation leaks. + encoded, err := json.Marshal(input) + if err != nil { + return nil, goAMD64FrameContext("fixed local declaration cannot be independently copied: %v", err) + } + var result map[string]FuncSig + if err := json.Unmarshal(encoded, &result); err != nil { + return nil, goAMD64FrameContext("fixed local declaration copy cannot be decoded: %v", err) + } + return result, nil +} + +func verifyX86FixedLocalBindings(file *File, options Options) ([]*x86FixedLocalBinding, error) { + var bindings []*x86FixedLocalBinding + seen := make(map[*x86FixedLocalBinding]bool) + for _, sig := range options.Sigs { + binding := sig.x86FixedLocal + if binding == nil || seen[binding] { + continue + } + seen[binding] = true + if file == nil || file.Arch != ArchAMD64 || options.Goarch != "amd64" || options.TargetTriple != binding.target || + options.ResolveSym == nil || options.WASMABI != WASMABIDirect || len(options.X86TailGroups) != 0 { + return nil, goAMD64FrameContext("fixed local consumer target or resolver changed") + } + if err := binding.verify(); err != nil { + return nil, err + } + if binding.numericalReads != nil && (options.NumericalReads == nil || + options.NumericalReads.source != binding.numericalReads.source || + options.NumericalReads.context != binding.context) { + return nil, goAMD64FrameContext("fixed local numerical consumer has a foreign source/context") + } + fingerprint, err := x86CallerSourceFingerprint(file) + if err != nil || fingerprint != binding.file || len(options.Sigs) != len(binding.signatures) { + return nil, goAMD64FrameContext("fixed local consumer original/normalized source changed") + } + for _, fn := range file.Funcs { + name := options.ResolveSym(fn.Sym) + actual, found := options.Sigs[name] + expected, declared := binding.signatures[name] + actual.x86FixedLocal = nil + if !found || !declared || name != binding.symbols[fn.Sym] || !reflect.DeepEqual(actual, expected) || + options.Sigs[name].x86FixedLocal != binding { + return nil, goAMD64FrameContext("fixed local consumer declaration or unique symbol binding changed") + } + } + for _, global := range file.Globl { + name, private := fixedLocalDataSymbol(global.Sym) + if !private || options.ResolveSym(name) != binding.symbols[name] { + return nil, goAMD64FrameContext("fixed local consumer literal symbol resolution changed") + } + } + bindings = append(bindings, binding) + } + return bindings, nil +} + +func (binding *x86FixedLocalBinding) verify() error { + if binding == nil || binding.identity != binding || binding.source == nil { + return goAMD64FrameContext("missing or expired fixed local observer binding") + } + if err := binding.source.Verify(binding.context); err != nil { + return err + } + binding.source.mu.Lock() + defer binding.source.mu.Unlock() + if !binding.source.ready || !reflect.DeepEqual(binding.inputs, binding.source.inputs) { + return goAMD64FrameContext("fixed local observer identity changed or was revoked") + } + if len(binding.declarations) != len(binding.signatures) { + return goAMD64FrameContext("fixed local typed language witnesses are incomplete") + } + for name, signature := range binding.signatures { + declaration, found := binding.declarations[name] + if !found || !reflect.DeepEqual(signature, declaration.signature) || + !binding.source.matchesFixedLocalDeclaration(name, declaration) { + return goAMD64FrameContext("fixed local typed language/object identity changed: %s", name) + } + } + return nil +} + +// Called only while source.mu is held. The expected declaration comes from +// an independent original-source typecheck at Bind, then a private deep copy +// retained by that binding. A factory observation alone grants no permission. +func (source *GoAMD64FrameSource) matchesFixedLocalDeclaration(name string, expected goAMD64FrameDeclaration) bool { + observed, found := source.functions[name] + original, declared := source.declarations[name] + return found && declared && fixedLocalTransportRole(expected.transport) && + observed.transport == expected.transport && original.transport == expected.transport && + expected.languageType != "" && observed.languageType == expected.languageType && + original.languageType == expected.languageType && + reflect.DeepEqual(observed.signature, expected.signature) && reflect.DeepEqual(original.signature, expected.signature) +} + +func finishX86FixedLocalBindings(bindings []*x86FixedLocalBinding) error { + for _, binding := range bindings { + if err := binding.verify(); err != nil { + return fmt.Errorf("fixed local post-consumption verification: %w", err) + } + } + return nil +} + +// Use a digest of the complete public instruction shape; private certificates +// are not serialized and cannot self-authenticate a modified instruction. +func fixedLocalInstructionDigest(ins Instr) ([32]byte, error) { + encoded, err := json.Marshal(ins) + if err != nil { + return [32]byte{}, goAMD64FrameContext("fixed local instruction cannot be bound: %v", err) + } + return sha256.Sum256(encoded), nil +} diff --git a/amd64_fixed_local_candidate.go b/amd64_fixed_local_candidate.go new file mode 100644 index 000000000..65a40f6be --- /dev/null +++ b/amd64_fixed_local_candidate.go @@ -0,0 +1,45 @@ +package plan9asm + +// GoAMD64FixedLocalFrameBindingNeeded is a source-role query, not a frame or +// memory certificate. The complete actual-object factory and fixed-local +// proof must independently succeed before any instruction gains backing. +func GoAMD64FixedLocalFrameBindingNeeded(file *File, goarch string) bool { + if file == nil || file.Arch != ArchAMD64 || goarch != "amd64" { + return false + } + for _, fn := range file.Funcs { + if fn.FrameSize <= 0 { + continue + } + for _, ins := range fn.Instrs { + memory, eligible := -1, false + if form, known, err := fixedLocalConversionProperties(ins); known { + memory, eligible = form.memory, err == nil && !form.mmx + } else if form, known, err := amd64PackedVectorMoveProperties(ins, goarch); known { + memory, eligible = form.memory, err == nil && form.spec.aligned && !form.undefinedOpcode() + } else if form, known, err := x86LegacyPackedMoveProperties(ins.Op, ins, goarch); known { + memory, eligible = form.memory, err == nil && form.aligned + } else if form, known, err := x86AESProperties(ins.Op, ins, goarch); known { + memory, eligible = form.memory, err == nil && form.byteWidth == 16 + } else if form, known, err := x86PackedWrappingAddProperties(ins, goarch); known { + memory, eligible = form.memory, err == nil && form.legacy && !form.mmx && form.byteWidth == 16 + } else if form, known, err := x86WordMultiplyAddProperties(ins, goarch); known { + memory, eligible = form.memory, err == nil && form.legacy && !form.mmx && form.byteWidth == 16 + } else if form, known, err := x86FloatCompareProperties(ins, goarch); known { + memory, eligible = form.memory, err == nil && form.legacy && !form.scalar && form.byteWidth == 16 + } else if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, goarch); known { + memory, eligible = 0, err == nil && form.source.Kind == OpMem + } + if eligible && memory >= 0 && memory < len(ins.Args) { + source := ins.Args[memory] + // Static SB and incoming native GP addresses keep their existing + // contracts. Virtual FP is not original local native storage. + if source.Kind == OpMem && (source.Mem.Base == SP || source.Mem.Base == BP || + source.Mem.Index == SP || source.Mem.Index == BP) { + return true + } + } + } + } + return false +} diff --git a/amd64_fixed_local_candidate_test.go b/amd64_fixed_local_candidate_test.go new file mode 100644 index 000000000..c9113913f --- /dev/null +++ b/amd64_fixed_local_candidate_test.go @@ -0,0 +1,49 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalFrameCandidateQueryCannotSupplyPermission(t *testing.T) { + for _, operation := range []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDD 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + } { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + source, sig, pkg := x86PrivateAlignedFrameAuditFixture(t, operation) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") || GoAMD64FixedLocalFrameBindingNeeded(file, "386") { + t.Fatal("candidate query lost its exact native family/target role") + } + x86PrivateAlignedFrameAuditAPIs(t, source, sig, pkg, func(t *testing.T, err error) { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("query supplied a forged native permission: %v", err) + } + }) + }) + } + for _, source := range []string{ + "TEXT ·Value(SB),4,$0-0\nPADDL 16(SP),X0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL X1,X0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nVPADDD X1,X2,X0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL (AX),M0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL X1\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL constant<>(SB),X0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL (AX),X0\nRET\n", + "TEXT ·Value(SB),4,$160-0\nPADDL (AX)(CX*2),X0\nRET\n", + "TEXT ·Value(SB),4,$160-16\nPADDL input+0(FP),X0\nRET\n", + } { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Fatalf("query widened unrelated source grammar: %s", source) + } + } +} diff --git a/amd64_fixed_local_conversion.go b/amd64_fixed_local_conversion.go new file mode 100644 index 000000000..8e2293eea --- /dev/null +++ b/amd64_fixed_local_conversion.go @@ -0,0 +1,73 @@ +package plan9asm + +// This normalizes the existing complete conversion grammars for the private +// frame observer. It grants neither caller storage nor a physical MMX role. +type x86FixedLocalConversion struct { + inputBytes, outputBytes, activeBytes int + memoryBytes, memory, destination int + mask int + legacy, zeroing, mmx bool +} + +func fixedLocalConversionProperties(ins Instr) (x86FixedLocalConversion, bool, error) { + if form, known, err := x86SignedPackedFloatProperties(ins, "amd64"); known { + return x86FixedLocalConversion{ + inputBytes: form.inputBytes, outputBytes: form.byteWidth, + activeBytes: form.byteWidth, memoryBytes: form.memoryBytes, + memory: form.memory, destination: form.destination, mask: form.mask, + legacy: form.spec.legacy, zeroing: form.zeroing, mmx: form.mmx, + }, true, err + } + form, known, err := x86PackedConversionProperties(ins, "amd64") + return x86FixedLocalConversion{ + inputBytes: form.inputBytes, outputBytes: form.shape.destinationBytes, + activeBytes: form.shape.lanes * form.spec.destinationBits / 8, + memoryBytes: form.memoryBytes, memory: form.memory, + destination: form.destination, mask: form.mask, + legacy: form.spec.legacy, zeroing: form.properties.zeroing, mmx: form.mmx, + }, known, err +} + +func fixedLocalConversionStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + form, known, err := fixedLocalConversionProperties(*ins) + if !known { + return false, nil + } + if err != nil || form.mmx { + return true, goAMD64FrameContext("fixed local conversion lacks an ordinary physical vector shape") + } + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return true, goAMD64FrameContext("fixed local conversion has an undefined mask") + } + if form.memory >= 0 { + // Legacy m128 retains its alignment requirement. CVTDQ2PD m64 + // and VEX/EVEX have no corresponding 16-byte #GP requirement. + alignment := int64(1) + if form.legacy && form.memoryBytes == 16 { + alignment = 16 + } + if _, _, err := fixedLocalReadMemory(ins, form.memory, state, proof, residue, + int64(form.memoryBytes), alignment); err != nil { + return true, err + } + } else { + input := state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] + required := x86CallerVectorByteMask(int64(form.inputBytes)) + if input&required != required { + return true, goAMD64FrameContext("fixed local conversion reads undefined numerical bytes") + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + active := x86CallerVectorByteMask(int64(form.activeBytes)) + if form.mask >= 0 && !form.zeroing && state.vectors[destination]&active != active { + return true, goAMD64FrameContext("fixed local conversion merges undefined destination bytes") + } + // Narrowing explicitly clears its carrier padding. Legacy preserves + // wider aliases; VEX/EVEX clears them. Integer FLAGS remain unchanged. + if form.legacy { + state.vectors[destination] |= x86CallerVectorByteMask(int64(form.outputBytes)) + } else { + state.vectors[destination] = ^uint64(0) + } + return true, nil +} diff --git a/amd64_fixed_local_conversion_runtime_linux_amd64_test.go b/amd64_fixed_local_conversion_runtime_linux_amd64_test.go new file mode 100644 index 000000000..bd78605d0 --- /dev/null +++ b/amd64_fixed_local_conversion_runtime_linux_amd64_test.go @@ -0,0 +1,7 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNative(t *testing.T) { + fixedLocalConversionRuntime(t, x86PackedDoubleDwordNativeTools(t), true) +} diff --git a/amd64_fixed_local_conversion_runtime_unix_test.go b/amd64_fixed_local_conversion_runtime_unix_test.go new file mode 100644 index 000000000..360cf0888 --- /dev/null +++ b/amd64_fixed_local_conversion_runtime_unix_test.go @@ -0,0 +1,265 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "runtime" + "strings" + "testing" +) + +type fixedLocalConversionRuntimeCase struct { + name, destination string + width, offset int + dead bool +} + +func fixedLocalConversionRuntimeCases(all bool) []fixedLocalConversionRuntimeCase { + rows := x86PackedNativeAddressRows() + rows = append(rows, + struct{ name, destination string }{"CVTPL2PS", "X1"}, + struct{ name, destination string }{"CVTPL2PD", "X1"}, + struct{ name, destination string }{"VCVTDQ2PS", "Y1"}, + struct{ name, destination string }{"VCVTDQ2PD", "Y1"}) + var cases []fixedLocalConversionRuntimeCase + for _, row := range rows { + if !all && strings.HasPrefix(row.name, "V") { + continue + } + width := 16 + if strings.HasPrefix(row.destination, "Y") { + width = 32 + } + for _, dead := range []bool{false, true} { + for _, offset := range []int{120, 128, 128 + width - 8, 128 + width} { + cases = append(cases, fixedLocalConversionRuntimeCase{ + name: row.name, destination: row.destination, + width: width, offset: offset, dead: dead, + }) + } + } + } + return cases +} + +func fixedLocalConversionRuntimeProgram(cases []fixedLocalConversionRuntimeCase, triple string) (string, string, string, string) { + var assembly, declarations, goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, row := range cases { + name := fmt.Sprintf("Value%d", index) + operation := row.name + " 64(BP)," + row.destination + if !row.dead { + move := "VMOVDQU" + if !strings.HasPrefix(row.name, "V") { + move = "MOVOU" + } + operation += "\n" + move + " " + row.destination + ",128(BP)" + } + body := fixedLocalPackedAssembly(operation) + body = strings.Replace(body, "$256-16", "$512-16", 1) + body = strings.Replace(body, "MOVQ 128(BP),AX", fmt.Sprintf("MOVQ %d(BP),AX", row.offset), 1) + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + canary := row.dead || row.offset < 128 || row.offset >= 128+row.width + fmt.Fprintf(&goRows, "{%s, %v},\n", name, canary) + canaryValue := 0 + if canary { + canaryValue = 1 + } + fmt.Fprintf(&cRows, "{value%d, %d},\n", index, canaryValue) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t) __asm__(\"%sexample.invalid/framefixture.%s\");\n", index, prefix, name) + } + // The caller controls and snapshots MXCSR outside the observed frame. + // These helpers belong only to the independently compiled Go executable. + original := `package main +import "fmt" +func readCSR() uint32 +func writeCSR(csr uint32) +` + declarations.String() + ` +func main() { + rows := []struct { call func(uint64) uint64; canary bool }{ +` + goRows.String() + ` + } + values := []uint64{0,1,^uint64(0),0x8000000080000000,0x3fc000003fc00000,0x3ff8000000000000,0x7fc123457fc12345,0x7ff8123456789abc} + for n,row := range rows { + for v,value := range values { + for mode:=0; mode<32; mode++ { + csr := uint32(0x1f80) | uint32(mode&3)<<13 | uint32((mode>>2)&1)<<6 | uint32((mode>>3)&1)<<15 + if mode&16 != 0 { csr |= 63 } + writeCSR(csr) + got := row.call(value) + status := readCSR() + writeCSR(0x1f80) + if row.canary && got != value { panic("original private-frame canary changed") } + fmt.Printf("%d %d %d %016x %08x\n", n,v,mode,got,status) + } + } + } +} +` + driver := `#include +#include +#include +` + cDeclarations.String() + ` +int main(void) { + const struct { uint64_t (*call)(uint64_t); int canary; } rows[] = { +` + cRows.String() + ` + }; + const uint64_t values[] = {0,1,UINT64_MAX,UINT64_C(0x8000000080000000),UINT64_C(0x3fc000003fc00000),UINT64_C(0x3ff8000000000000),UINT64_C(0x7fc123457fc12345),UINT64_C(0x7ff8123456789abc)}; + for (unsigned n=0; n>2)&1)<<6) | (((mode>>3)&1)<<15); + if (mode&16) csr |= 63; + _mm_setcsr(csr); + uint64_t got = rows[n].call(values[v]); + unsigned status = _mm_getcsr(); + _mm_setcsr(0x1f80); + if (rows[n].canary && got != values[v]) return 2; + printf("%u %u %u %016llx %08x\n",n,v,mode,(unsigned long long)got,status); + } + } + } + return 0; +} +` + return assembly.String(), declarations.String(), original, driver +} + +func fixedLocalConversionRuntime(t *testing.T, execution x86FloatCompareExecution, all bool) { + t.Helper() + cases := fixedLocalConversionRuntimeCases(all) + for start := 0; start < len(cases); start += 48 { + end := start + 48 + if end > len(cases) { + end = len(cases) + } + t.Run(fmt.Sprintf("batch%d", start/48), func(t *testing.T) { + assembly, declarations, original, driver := fixedLocalConversionRuntimeProgram(cases[start:end], execution.triple) + helpers := "TEXT ·readCSR(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\nTEXT ·writeCSR(SB),4,$0-4\nLDMXCSR csr+0(FP)\nRET\n" + want := string(x86ScalarFloatIntegerOriginalOutput(t, execution, assembly+helpers, original)) + if strings.Count(want, "\n") != (end-start)*8*32 { + t.Fatal("original Go numerical/status denominator changed") + } + pkg, request, file, options := x86CMOVFixedLocalFixture(t, + assembly, declarations, runtime.GOOS+"/amd64", execution.triple) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, optimize := range []bool{false, true} { + input, level := ir, "-O0" + if optimize { + input, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + got := string(execution.compileAndRunLLVM(t, "private-conversion", input, driver, level)) + if got != want { + t.Fatal("original Go and LLVM private-frame value/MXCSR/canary results differ") + } + } + }) + }) + } +} + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionSSE(t *testing.T) { + fixedLocalConversionRuntime(t, x86FloatCompareRuntimeTools(t, x86FloatCompareSSE), false) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("Rosetta SSE diagnostic, not native AVX512 execution") + } +} + +func TestGoAMD64FixedLocalConversionRuntimeOwnership(t *testing.T) { + for _, row := range []struct { + all bool + count int + }{{false, 48}, {true, 336}} { + cases := fixedLocalConversionRuntimeCases(row.all) + if len(cases) != row.count { + t.Fatalf("runtime cases all=%v: %d, want %d", row.all, len(cases), row.count) + } + names := make(map[string]int) + for _, entry := range cases { + names[entry.name]++ + } + if len(names)*8 != row.count { + t.Fatal("runtime inventory lost distinct family members", names) + } + for name, count := range names { + if count != 8 { + t.Fatalf("%s lost a live/dead or canary axis: %d", name, count) + } + } + assembly, _, _, _ := fixedLocalConversionRuntimeProgram(cases, "x86_64-unknown-linux-gnu") + requireX86GoAssemblerResult(t, "amd64", assembly, true) + } + const filename = "amd64_fixed_local_conversion_runtime_linux_amd64_test.go" + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "386"}, {"linux", "arm"}, {"linux", "arm64"}, + {"js", "wasm"}, {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + context := build.Default + context.GOOS, context.GOARCH = target.os, target.arch + selected, err := context.MatchFile(".", filename) + if err != nil || selected != (target.os == "linux" && target.arch == "amd64") { + t.Fatal("required native execution owner selection", target, selected, err) + } + } + syntax, err := parser.ParseFile(token.NewFileSet(), filename, nil, 0) + if err != nil { + t.Fatal(err) + } + owners := 0 + for _, declaration := range syntax.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok { + continue + } + owners++ + if function.Name.Name != "TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalConversionNative" || function.Body == nil || len(function.Body.List) != 1 { + t.Fatal("native owner missing or conditional") + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native runtime must execute directly") + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || len(call.Args) != 3 || !fixedLocalConversionIdent(call.Fun, "fixedLocalConversionRuntime") || + !fixedLocalConversionIdent(call.Args[0], "t") || !fixedLocalConversionIdent(call.Args[2], "true") { + t.Fatal("native runtime lost its complete-family axis") + } + tools, ok := call.Args[1].(*ast.CallExpr) + if !ok || len(tools.Args) != 1 || !fixedLocalConversionIdent(tools.Fun, "x86PackedDoubleDwordNativeTools") || + !fixedLocalConversionIdent(tools.Args[0], "t") { + t.Fatal("native runtime bypassed actual CPU/OS/opcode checks") + } + } + if owners != 1 { + t.Fatal("missing or ambiguous native owner", owners) + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatal("required native runtime root not selected by CI", err) + } +} + +func fixedLocalConversionIdent(expression ast.Expr, name string) bool { + identifier, ok := expression.(*ast.Ident) + return ok && identifier.Name == name +} diff --git a/amd64_fixed_local_conversion_test.go b/amd64_fixed_local_conversion_test.go new file mode 100644 index 000000000..571561904 --- /dev/null +++ b/amd64_fixed_local_conversion_test.go @@ -0,0 +1,305 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalConversionOperations() []string { + var operations []string + seen := make(map[string]bool) + for _, row := range append(x86PackedConversionOracleRows("amd64"), x86SignedPackedFloatRows("amd64")...) { + // Keep the complete independent register/mask/rounding inventory and + // one private address representative. Physical MMX stays unbound. + if strings.Contains(row, "M0") || strings.Contains(row, "M7") { + continue + } + _, operands, _ := strings.Cut(row, " ") + source, _, _ := strings.Cut(operands, ",") + if source == "(AX)" || source == "3(AX)" { + row = strings.Replace(row, source, "64(BP)", 1) + } else if strings.Contains(row, "(") || strings.Contains(row, " 256,") { + continue + } + if !seen[row] { + seen[row] = true + operations = append(operations, row) + } + } + return operations +} + +func fixedLocalConversionOperation(row string) string { + // Every source and merge byte is independently numerical, including + // legacy upper aliases. Conversion output is then observed by a store. + prefix := "KMOVQ AX,K1\nKMOVQ AX,K3\nKMOVQ AX,K7\n" + for _, index := range []int{0, 7, 15, 20, 21, 31} { + prefix += fmt.Sprintf("VMOVDQU64 64(BP),Z%d\n", index) + } + parts := strings.Split(row, ",") + destination := strings.TrimSpace(parts[len(parts)-1]) + return prefix + row + "\nVMOVDQU64 Z" + destination[1:] + ",128(BP)" +} + +func TestGoAMD64FixedLocalConversionCompleteFamilyProof(t *testing.T) { + var original strings.Builder + names := make(map[string]bool) + for index, row := range fixedLocalConversionOperations() { + t.Run(row, func(t *testing.T) { + op := strings.Fields(row)[0] + name, _, _ := strings.Cut(op, ".") + names[name] = true + operation := fixedLocalConversionOperation(row) + pkg, _, file, options := fixedLocalPackedFixture(t, operation) + assembly := fixedLocalPackedAssembly(operation) + original.WriteString(strings.Replace(assembly, "·Value", fmt.Sprintf("·Value%d", index), 1)) + if strings.Contains(row, "(BP)") && !GoAMD64FixedLocalFrameBindingNeeded(file, "amd64") { + t.Error("native conversion memory did not request a complete original frame binding") + } + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); err != nil { + t.Fatalf("initialized private conversion rejected: %v", err) + } + }) + } + if len(names) != 42 { + t.Fatalf("complete independent conversion namespace = %d, want 42", len(names)) + } + requireX86GoAssemblerResult(t, "amd64", original.String(), true) +} + +func TestGoAMD64FixedLocalConversionWholeSourceNegatives(t *testing.T) { + operation := "VCVTDQ2PD 64(BP),Y0\nVMOVDQU Y0,128(BP)" + pkg, _, _, options := fixedLocalPackedFixture(t, operation) + original := fixedLocalPackedAssembly(operation) + for name, source := range map[string]string{ + "input hole": strings.Replace(original, "MOVQ AX,64(BP)", "MOVB AX,64(BP)", 1), + "outside frame": strings.Replace(original, "VCVTDQ2PD 64(BP)", "VCVTDQ2PD 256(BP)", 1), + "incoming pointer": strings.Replace(original, "MOVQ SP,BP", "MOVQ AX,BP", 1), + "unknown mask": strings.Replace(original, "64(BP),Y0", "64(BP),K1,Y0", 1), + "unknown merge": strings.Replace(original, operation, "KMOVQ AX,K1\nVCVTDQ2PD 64(BP),K1,Y0\nVMOVDQU Y0,128(BP)", 1), + "zero mask missing storage": strings.Replace(original, operation, "MOVQ $0,DX\nKMOVQ DX,K1\nVCVTDQ2PD.Z 256(BP),K1,Y0\nVMOVDQU Y0,128(BP)", 1), + "unknown source vector": strings.Replace(original, "VCVTDQ2PD 64(BP)", "VCVTDQ2PD X0", 1), + "legacy m128 alignment": strings.Replace(original, operation, "CVTPL2PS 65(BP),X0\nMOVOU X0,128(BP)", 1), + "MMX input": strings.Replace(original, operation, "CVTPL2PD M0,X0\nMOVOU X0,128(BP)", 1), + "MMX output": strings.Replace(original, operation, "CVTPS2PL X0,M0", 1), + "private escape": strings.Replace(original, "MOVQ AX,ret+8(FP)", "MOVQ BP,ret+8(FP)", 1), + } { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unclosed native conversion accepted: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalConversionSparseByteEffects(t *testing.T) { + // Independent representatives of every width, alias and mask rule. The + // complete family test separately binds all encoder forms to these rules. + for _, row := range []struct { + instruction string + input, active int + legacy, mask, zero bool + }{ + {"CVTPL2PD X0,X1", 8, 16, true, false, false}, + {"CVTPS2PL X0,X1", 16, 16, true, false, false}, + {"CVTPD2PL X0,X1", 16, 8, true, false, false}, + {"VCVTDQ2PD X0,Y1", 16, 32, false, false, false}, + {"VCVTPD2DQY Y0,X1", 32, 16, false, false, false}, + {"VCVTPD2DQ Z0,Y1", 64, 32, false, false, false}, + {"VCVTPS2QQ X0,Y1", 16, 32, false, false, false}, + {"VCVTQQ2PSY Y0,X1", 32, 16, false, false, false}, + {"VCVTPS2DQ.Z X0,K1,X1", 16, 16, false, true, true}, + {"VCVTPS2DQ X0,K1,X1", 16, 16, false, true, false}, + {"VCVTPD2DQX X0,K1,X1", 16, 8, false, true, false}, + } { + t.Run(row.instruction, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT value(SB),4,$0-0\n"+row.instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + source := x86CallerVectorRoot(ins.Args[0].Reg) + destination := x86CallerVectorRoot(ins.Args[len(ins.Args)-1].Reg) + fresh := func() x86FixedLocalState { + return x86FixedLocalState{ + vectors: map[Reg]uint64{source: ^uint64(0), destination: ^uint64(0)}, + masks: map[Reg]bool{"K1": true}, + flags: true, + } + } + check := func(state *x86FixedLocalState, accepted bool) { + t.Helper() + known, err := fixedLocalConversionStep(&ins, state, &x86FixedLocalProof{}, 0) + if !known || accepted && err != nil || !accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("conversion effect accepted=%t: known=%t, %v", accepted, known, err) + } + } + for missing := 0; missing < 64; missing++ { + state := fresh() + state.vectors[source] &^= uint64(1) << uint(missing) + check(&state, missing >= row.input) + } + for missing := 0; missing < 64; missing++ { + state := fresh() + state.vectors[destination] &^= uint64(1) << uint(missing) + accepted := !row.mask || row.zero || missing >= row.active + check(&state, accepted) + if !accepted { + continue + } + want := ^uint64(0) + if row.legacy && missing >= 16 { + want &^= uint64(1) << uint(missing) + } + if state.vectors[destination] != want || !state.flags { + t.Fatalf("byte %d: alias/zero padding/FLAGS changed: %+v", missing, state) + } + } + if row.mask { + state := fresh() + state.masks["K1"] = false + check(&state, false) + } + }) + } +} + +func TestGoAMD64FixedLocalConversionExactMemoryTuples(t *testing.T) { + for _, row := range []struct { + instruction string + width int + aligned bool + }{ + {"CVTPL2PD (BP),X1", 8, false}, + {"CVTPL2PS (BP),X1", 16, true}, + {"CVTPS2PL (BP),X1", 16, true}, + {"CVTPD2PL (BP),X1", 16, true}, + {"VCVTDQ2PD (BP),X1", 8, false}, + {"VCVTDQ2PD (BP),Y1", 16, false}, + {"VCVTDQ2PD (BP),Z1", 32, false}, + {"VCVTDQ2PS (BP),Z1", 64, false}, + {"VCVTDQ2PD.BCST.Z (BP),K1,Z1", 4, false}, + {"VCVTPD2DQX (BP),X1", 16, false}, + {"VCVTPD2DQY (BP),X1", 32, false}, + {"VCVTPD2DQ (BP),Y1", 64, false}, + {"VCVTPS2QQ.BCST.Z (BP),K1,Z1", 4, false}, + {"VCVTUQQ2PS.BCST.Z (BP),K1,Y1", 8, false}, + } { + t.Run(row.instruction, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT value(SB),4,$128-0\n"+row.instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + destination := x86CallerVectorRoot(original.Args[len(original.Args)-1].Reg) + check := func(missing, offset int, accepted bool) { + t.Helper() + ins := original + state := x86FixedLocalState{ + pointers: map[Reg]int64{BP: int64(offset)}, + vectors: map[Reg]uint64{destination: ^uint64(0)}, + masks: map[Reg]bool{"K1": true}, + bytes: make([]bool, 128), + } + for index := range state.bytes { + state.bytes[index] = index != missing + } + proof := &x86FixedLocalProof{frame: 128} + known, err := fixedLocalConversionStep(&ins, &state, proof, 0) + if !known || accepted && err != nil || !accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("byte=%d offset=%d accepted=%t: known=%t, %v", missing, offset, accepted, known, err) + } + if accepted { + access := ins.x86FixedLocalMemory[0] + if access.owner != proof || access.width != int64(row.width) || access.write || state.flags { + t.Fatal("wrong native memory tuple, write permission or invented FLAGS", access) + } + } + if missing >= 0 && state.bytes[missing] { + t.Fatal("conversion manufactured initialized memory") + } + } + for missing := 0; missing < 128; missing++ { + check(missing, 0, missing >= row.width) + } + check(-1, 1, !row.aligned) + check(-1, 129-row.width, false) + }) + } +} + +func TestGoAMD64FixedLocalConversionActualObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + rows := fixedLocalConversionOperations() + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + seen := 0 + for start := 0; start < len(rows); start += 64 { + end := start + 64 + if end > len(rows) { + end = len(rows) + } + seen += end - start + t.Run(fmt.Sprintf("%s/batch%d", target.name, start/64), func(t *testing.T) { + var assembly, declarations strings.Builder + for index := start; index < end; index++ { + name := fmt.Sprintf("Value%d", index) + body := fixedLocalPackedAssembly(fixedLocalConversionOperation(rows[index])) + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + } + pkg, request, file, options := x86CMOVFixedLocalFixture(t, + assembly.String(), declarations.String(), target.name, target.triple) + source := requireGoAMD64FrameCapture(t, pkg, request) + // Match the actual producer's complete CPP output, including + // spaces before opcode suffix tokens. Keep the original file + // unchanged; its raw text is part of the binding fingerprint. + expanded, err := source.preprocess(request.AssemblyFile, []byte(assembly.String())) + if err != nil { + t.Fatal(err) + } + file, err = Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "fixed_local_native") || !strings.Contains(ir, "asm sideeffect") { + t.Fatal("conversion did not consume native private backing") + } + for _, optimize := range []bool{false, true} { + input := ir + if optimize { + input = x86MulDivRoundOptimize(t, ir) + } + compileLLVMToObject(t, llc, target.triple, "frame-conversion.ll", "frame-conversion.o", input) + } + }) + }) + } + if seen != len(rows) { + t.Fatalf("%s covered %d/%d conversion forms", target.name, seen, len(rows)) + } + } + t.Logf("complete object plan: %d forms, three actual-Go targets, three LLVM APIs, O0/O2", len(rows)) +} diff --git a/amd64_fixed_local_data.go b/amd64_fixed_local_data.go new file mode 100644 index 000000000..49543567d --- /dev/null +++ b/amd64_fixed_local_data.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "bytes" + "path/filepath" + "strings" +) + +type x86FixedLocalData struct { + size, alignment int64 +} + +func fixedLocalDataSymbol(symbol string) (string, bool) { + name := strings.TrimPrefix(symbol, "·") + if !strings.HasSuffix(name, "<>") || len(name) <= 2 || strings.ContainsAny(name, "·/.") { + return "", false + } + return "·" + name, true +} + +// Called under the original source lock after its whole parsed file has been +// matched. Private literal definitions cannot be replaced by a Go variable, +// linker winner, mutable global, address initializer or caller-supplied extent. +func (source *GoAMD64FrameSource) fixedLocalData(file *File, expanded string, resolve func(string) string) (map[string]x86FixedLocalData, map[string]string, error) { + if len(file.Data)+len(file.Globl) == 0 { + return nil, nil, nil + } + object, err := source.read(filepath.Join(source.owned, source.request.AssemblyFile+".o")) + if err != nil { + return nil, nil, err + } + observed, err := source.frameDataObject(expanded, object) + if err != nil { + return nil, nil, err + } + if len(observed.Layouts) != len(file.Globl) { + return nil, nil, goAMD64FrameContext("fixed local DATA lacks its whole original definition inventory") + } + data := make(map[string]x86FixedLocalData) + symbols := make(map[string]string) + var total int64 + for index, global := range file.Globl { + layout := observed.Layouts[index] + definition := layout.Definition + name, private := fixedLocalDataSymbol(global.Sym) + expected := source.selection.ImportPath + "." + strings.TrimPrefix(name, "·") + if _, duplicate := symbols[name]; duplicate { + return nil, nil, goAMD64FrameContext("fixed local DATA definitions collapse to the same translated identity") + } + total += int64(definition.Size) + if !private || resolve(name) != expected || !layout.Static || layout.ABI != 65535 || + layout.Flags != 8 && layout.Flags != 24 || definition.Kind != 3 || definition.Flags != 0 || + definition.Flags2 != 0 || len(definition.Relocations) != 0 || global.SizeRaw != "" || + definition.Size == 0 || definition.Size > 64<<10 || total > 1<<20 || len(definition.Data) > int(definition.Size) { + return nil, nil, goAMD64FrameContext("fixed local DATA requires bounded private relocation-free readonly storage") + } + // Compare the translator's complete initializer with independent actual + // Go bytes, including sparse zero gaps and trailing allocation. A GLOBL + // that shortens preceding DATA must not acquire a larger LLVM extent. + image := make([]byte, definition.Size) + size := global.Size + for _, fragment := range file.Data { + if fragment.Sym != global.Sym { + continue + } + payload, err := dataStmtPayload(fragment) + if err != nil || fragment.Addr != "" || fragment.Off < 0 || fragment.Off > int64(len(image))-int64(len(payload)) { + return nil, nil, goAMD64FrameContext("fixed local DATA initializer has an unbound source byte extent") + } + copy(image[fragment.Off:], payload) + if end := fragment.Off + int64(len(payload)); end > size { + size = end + } + } + if size != int64(definition.Size) || !bytes.Equal(image[:len(definition.Data)], definition.Data) || + !bytes.Equal(image[len(definition.Data):], make([]byte, len(image)-len(definition.Data))) { + return nil, nil, goAMD64FrameContext("fixed local translated DATA differs from the original Go initialized extent") + } + data[global.Sym] = x86FixedLocalData{size, goAssemblyDataAlignment(ArchAMD64, size)} + symbols[name] = expected + } + return data, symbols, nil +} + +// Readonly DATA never receives a frame offset or a native-frame certificate. +// The only consumer is an already registered numerical memory operand. The +// complete original file and immutable symbol resolution own its LLVM global. +func fixedLocalReadMemory(ins *Instr, index int, state *x86FixedLocalState, proof *x86FixedLocalProof, + residue int, width, alignment int64) (offset int64, literal bool, err error) { + if index >= 0 && index < len(ins.Args) && ins.Args[index].Kind == OpSym { + if proof.binding == nil || width <= 0 || alignment <= 0 { + return 0, false, goAMD64FrameContext("fixed local DATA read has no original literal owner") + } + name, offset, valid := parseSBRef(ins.Args[index].Sym) + data, found := proof.binding.literalData[name] + if !valid || !found || offset < 0 || width > data.size || offset > data.size-width || + alignment > data.alignment || offset%alignment != 0 { + return 0, false, goAMD64FrameContext("fixed local DATA read exceeds original identity, extent or alignment") + } + return offset, true, nil + } + offset, err = fixedLocalMemory(ins, index, state, proof, residue, width, alignment, false, false) + return offset, false, err +} diff --git a/amd64_fixed_local_data_go127_test.go b/amd64_fixed_local_data_go127_test.go new file mode 100644 index 000000000..224450308 --- /dev/null +++ b/amd64_fixed_local_data_go127_test.go @@ -0,0 +1,107 @@ +//go:build go1.27 + +package plan9asm + +import ( + "go/types" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +const fixedLocalLiteralData = "DATA literal<>+0(SB)/8,$0x0102030405060708\n" + + "DATA literal<>+8(SB)/8,$0x1112131415161718\nGLOBL literal<>(SB),8,$32\n" + +func fixedLocalDataFixture(t *testing.T, operation, data string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options, string) { + t.Helper() + pkg, request, _, options := fixedLocalFrameFixture(t, operation) + name := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + assembly := data + string(original) + if err := os.WriteFile(name, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return pkg, request, file, options, assembly +} + +func TestGoAMD64FixedLocalReadonlyDataThreeTargetObjects(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, file, options, assembly := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", fixedLocalLiteralData) + request.Target, options.TargetTriple = target.name, target.triple + x86LEAActualTargetGoObject(t, "amd64", target.triple, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal("original private readonly DATA cannot join the fixed-local numeric proof", err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, func(t *testing.T, ir string, err error) { + if err != nil || !strings.Contains(ir, "internal constant [32 x i8]") || !strings.Contains(ir, "fixed_local_native") { + t.Fatal("missing joined private DATA/frame translation", err) + } + compileLLVMToObject(t, llc, target.triple, "data-frame.ll", "data-frame.o", ir) + compileLLVMToObject(t, llc, target.triple, "data-frame-opt.ll", "data-frame-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func fixedLocalDataAPIs(t *testing.T, pkg GoPackage, file *File, assembly string, options Options, check func(*testing.T, string, error)) { + t.Helper() + t.Run("Translate", func(t *testing.T) { + ir, err := Translate(file, options) + check(t, ir, err) + }) + t.Run("Module", func(t *testing.T) { + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + ir := "" + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + check(t, ir, err) + }) + t.Run("GoModule", func(t *testing.T) { + goos := "linux" + if strings.Contains(options.TargetTriple, "apple") { + goos = "darwin" + } else if strings.Contains(options.TargetTriple, "windows") { + goos = "windows" + } + translation, err := TranslateGoModule(pkg, []byte(assembly), GoModuleOptions{ + GOOS: goos, GOARCH: "amd64", TargetTriple: options.TargetTriple, + Sizes: types.SizesFor("gc", "amd64"), ResolveSym: options.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { sig, found := options.Sigs[name]; return sig, found }, + }) + ir := "" + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + ir = translation.Module.String() + } + check(t, ir, err) + }) +} diff --git a/amd64_fixed_local_data_guards_go127_test.go b/amd64_fixed_local_data_guards_go127_test.go new file mode 100644 index 000000000..7b4b6c023 --- /dev/null +++ b/amd64_fixed_local_data_guards_go127_test.go @@ -0,0 +1,106 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalReadonlyDataConsumerIdentity(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + pkg, request, file, options, assembly := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", fixedLocalLiteralData) + fixedLocalDataAPIs(t, pkg, file, assembly, options, fixedLocalRequireContext) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + for _, mutation := range []string{"resolver", "JSON", "byte", "extent", "flags", "missing DATA", "new DATA"} { + t.Run(mutation, func(t *testing.T) { + changed, input := bound, assembly + switch mutation { + case "resolver": + changed.ResolveSym = func(symbol string) string { + if symbol == "·literal<>" { + return "foreign.literal<>" + } + return options.ResolveSym(symbol) + } + case "JSON": + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + case "byte": + input = strings.Replace(input, "0x0102030405060708", "0x0102030405060709", 1) + case "extent": + input = strings.Replace(input, ",8,$32", ",8,$64", 1) + case "flags": + input = strings.Replace(input, ",8,$32", ",16,$32", 1) + case "missing DATA": + input = strings.TrimPrefix(input, fixedLocalLiteralData) + case "new DATA": + input = "GLOBL extra<>(SB),8,$8\n" + input + } + parsed, err := Parse(ArchAMD64, input) + if err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, parsed, input, changed, fixedLocalRequireContext) + }) + } + // A callback's later answer cannot replace the already verified symbol. + t.Run("stateful", func(t *testing.T) { + calls := 0 + changed := bound + changed.ResolveSym = func(symbol string) string { + if symbol == "·literal<>" { + calls++ + if calls > 1 { + return "foreign.literal<>" + } + } + return options.ResolveSym(symbol) + } + // Each API gets a fresh callback counter. + fixedLocalDataAPIs(t, pkg, file, assembly, changed, func(t *testing.T, ir string, err error) { + if err != nil || strings.Contains(ir, "foreign.literal") || calls != 1 { + t.Fatal("mutable resolver escaped sealed literal identity", calls, err) + } + calls = 0 + }) + }) + path := filepath.Join(request.Directory, request.AssemblyFile) + changed := strings.Replace(assembly, "0x0102030405060708", "0x0102030405060709", 1) + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, fixedLocalRequireContext) + if err := os.WriteFile(path, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, assembly, bound, fixedLocalRequireContext) +} + +func TestGoAMD64FixedLocalReadonlyDataRejectsCoalescedDefinitions(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + // Go's bare and qualified static names are distinct definitions. The + // ordinary LLVM resolver cannot fold them into one global in a binding. + data := fixedLocalLiteralData + "DATA ·literal<>+0(SB)/8,$99\nGLOBL ·literal<>(SB),8,$32\n" + pkg, request, file, options, assembly := fixedLocalDataFixture(t, + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", data) + x86LEAActualTargetGoObject(t, "amd64", options.TargetTriple, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("distinct original globals acquired one LLVM identity", err) + } +} diff --git a/amd64_fixed_local_data_matrix_go127_test.go b/amd64_fixed_local_data_matrix_go127_test.go new file mode 100644 index 000000000..2ef82de94 --- /dev/null +++ b/amd64_fixed_local_data_matrix_go127_test.go @@ -0,0 +1,149 @@ +//go:build go1.27 + +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalDataOperations() []string { + var operations []string + for _, name := range []string{"MOVB", "MOVW", "MOVL", "MOVQ"} { + operations = append(operations, name+" literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0") + } + for _, name := range []string{"MOVO", "MOVOA", "MOVOU", "MOVAPS", "MOVAPD", "MOVUPS", "MOVUPD"} { + operations = append(operations, name+" literal<>(SB),X0") + } + for _, row := range fixedLocalPackedMoveOracle { + for _, bank := range []string{"X", "Y", "Z"} { + aligned := row.name == "VMOVDQA" || row.name == "VMOVAPD" || row.name == "VMOVAPS" || + row.name == "VMOVDQA32" || row.name == "VMOVDQA64" + // Original Go's default assembly DATA alignment is at most 32. + // A 64-byte aligned memory instruction needs a separate witness. + if bank == "Z" && (!row.evex || aligned) { + continue + } + operations = append(operations, row.name+" literal<>(SB),"+bank+"0") + if row.evex { + operations = append(operations, + "KMOVQ AX,K1\n"+row.name+" literal<>(SB),K1,"+bank+"0", + "KMOVQ AX,K1\n"+row.name+".Z literal<>(SB),K1,"+bank+"0") + } + } + } + for _, name := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + operations = append(operations, name+" literal<>+1(SB),K1\nKMOVQ K1,DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0") + } + for _, operation := range []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + } { + operations = append(operations, strings.Replace(operation, "16(SP)", "literal<>(SB)", 1)) + } + for _, name := range []string{"AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC"} { + operations = append(operations, name+" literal<>(SB),X0") + } + operations = append(operations, "AESKEYGENASSIST $255,literal<>(SB),X0") + for _, name := range []string{"VAESENC", "VAESENCLAST", "VAESDEC", "VAESDECLAST"} { + operations = append(operations, name+" literal<>(SB),X0,X0") + } + operations = append(operations, "VAESIMC literal<>(SB),X0", "VAESKEYGENASSIST $255,literal<>(SB),X0") + for _, name := range []string{"CMOVWEQ", "CMOVLEQ", "CMOVQEQ"} { + operations = append(operations, "CMPQ AX,AX\n"+name+" literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0") + } + return operations +} + +func TestGoAMD64FixedLocalReadonlyDataAllRegisteredConsumers(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + operations := fixedLocalDataOperations() + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + for start := 0; start < len(operations); start += 32 { + end := start + 32 + if end > len(operations) { + end = len(operations) + } + t.Run(fmt.Sprintf("%s/batch%d", target.name, start/32), func(t *testing.T) { + var assembly, declarations strings.Builder + assembly.WriteString(strings.Replace(fixedLocalLiteralData, "$32", "$64", 1)) + for index := start; index < end; index++ { + name := fmt.Sprintf("Value%d", index) + body, _, _ := x86PrivateAlignedFrameAuditFixture(t, operations[index]+"\nPADDQ 16(SP),X0") + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", name) + } + pkg, request, _, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), target.name, target.triple) + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, []byte(assembly.String())) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalDataAPIs(t, pkg, file, expanded, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "literal-consumers.ll", "literal-consumers.o", ir) + compileLLVMToObject(t, llc, target.triple, "literal-consumers-opt.ll", "literal-consumers-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } + } + t.Logf("independent literal consumer operations=%d; three actual Go targets, three APIs, O0/O2; at most 32 functions/module", len(operations)) +} + +func TestGoAMD64FixedLocalReadonlyDataRejectsUnownedAccesses(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + for _, row := range []struct{ name, operation, data string }{ + {"past end", "MOVOU literal<>+24(SB),X0", fixedLocalLiteralData}, + {"negative", "MOVQ literal<>-1(SB),DX", fixedLocalLiteralData}, + {"unaligned", "MOVO literal<>+1(SB),X0", fixedLocalLiteralData}, + {"wide alignment", "VMOVDQA64 literal<>(SB),Z0", strings.Replace(fixedLocalLiteralData, "$32", "$64", 1)}, + {"zero mask cannot hide extent", "MOVQ $0,AX\nKMOVQ AX,K1\nVMOVDQU64.Z literal<>(SB),K1,Z0", fixedLocalLiteralData}, + {"write", "MOVOU X0,literal<>(SB)", fixedLocalLiteralData}, + {"address", "LEAQ literal<>(SB),DX", fixedLocalLiteralData}, + {"mutable", "MOVOU literal<>(SB),X0", strings.Replace(fixedLocalLiteralData, ",8,$32", ",16,$32", 1)}, + {"nonprivate", "MOVOU literal(SB),X0", strings.ReplaceAll(fixedLocalLiteralData, "literal<>", "literal")}, + {"address initializer", "MOVOU literal<>(SB),X0", fixedLocalLiteralData + "DATA pointer<>(SB)/8,$literal<>(SB)\nGLOBL pointer<>(SB),8,$8\n"}, + {"shortened extent", "MOVQ literal<>(SB),DX", strings.Replace(fixedLocalLiteralData, "$32", "$8", 1)}, + } { + t.Run(row.name, func(t *testing.T) { + pkg, request, file, options, assembly := fixedLocalDataFixture(t, row.operation+"\nPADDQ 16(SP),X0", row.data) + x86LEAActualTargetGoObject(t, "amd64", options.TargetTriple, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + file, err = Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unowned literal operation obtained fixed-local permission", err) + } + }) + } +} diff --git a/amd64_fixed_local_data_runtime_go127_test.go b/amd64_fixed_local_data_runtime_go127_test.go new file mode 100644 index 000000000..970560c39 --- /dev/null +++ b/amd64_fixed_local_data_runtime_go127_test.go @@ -0,0 +1,150 @@ +//go:build unix && go1.27 + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalReadonlyData(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + operations := []string{ + "MOVOU literal<>(SB),X0\nPADDQ 16(SP),X0", + "PADDQ literal<>(SB),X0", + "MOVQ literal<>+8(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVOU literal<>+16(SB),X0\nPADDQ 16(SP),X0", + "MOVQ literal<>+3(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVB literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVW literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + "MOVL literal<>+1(SB),DX\nMOVQ DX,48(SP)\nMOVOU 48(SP),X0", + } + var assembly, declarations, calls, cDeclarations, cCalls strings.Builder + assembly.WriteString(fixedLocalLiteralData) + prefix := "" + if runtime.GOOS == "darwin" { + prefix = "_" + } + for index, operation := range operations { + name := fmt.Sprintf("Value%d", index) + body, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + assembly.WriteString(strings.Replace(body, "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a,b,c,d uint64) uint64\n", name) + fmt.Fprintf(&calls, ` + if got := %s(a, b, c, d); got != expected(%d, a, c) { + t.Fatalf("case %d seed=%%d got=%%x", seed, got) + } +`, name, index, index) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t,uint64_t,uint64_t,uint64_t) __asm__(\"%sexample.invalid/framefixture.%s\");\n", index, prefix, name) + fmt.Fprintf(&cCalls, ` + if (value%d(a, b, c, d) != expected(%d, a, c)) { + fprintf(stderr, "case %d seed=%%u\n", seed); + return 1; + } +`, index, index, index) + } + goOracle := `package framefixture +import "testing" +func expected(op int, a, c uint64) uint64 { + const lo = uint64(0x0102030405060708) + const hi = uint64(0x1112131415161718) + switch op { + case 0: + return lo+a + case 1: + return lo+c + case 2: + return hi + case 3: + return a + case 4: + return lo>>24 | (hi&0xffffff)<<40 + case 5: + return c&^255 | 7 + case 6: + return c&^65535 | 0x0607 + case 7: + return 0x04050607 + } + panic("unknown case") +} +func TestOriginal(t *testing.T) { + for seed := uint64(0); seed < 256; seed++ { + a := seed*0x102030405060708 + b, c, d := ^a, a^0xfedcba9876543210, a+0x1234 +` + calls.String() + ` + } +} +` + cOracle := `#include +#include +` + cDeclarations.String() + ` +static uint64_t expected(unsigned op, uint64_t a, uint64_t c) { + const uint64_t lo = UINT64_C(0x0102030405060708); + const uint64_t hi = UINT64_C(0x1112131415161718); + switch (op) { + case 0: + return lo+a; + case 1: + return lo+c; + case 2: + return hi; + case 3: + return a; + case 4: + return lo>>24 | (hi&0xffffff)<<40; + case 5: + return (c&~UINT64_C(255)) | 7; + case 6: + return (c&~UINT64_C(65535)) | 0x0607; + case 7: + return 0x04050607; + } + return 0; +} +int main(void) { + for (unsigned seed=0; seed<256; seed++) { + uint64_t a = seed*UINT64_C(0x102030405060708); + uint64_t b = ~a, c = a^UINT64_C(0xfedcba9876543210), d = a+0x1234; +` + cCalls.String() + ` + } + return 0; +} +` + pkg, request, _, options := x86CMOVFixedLocalFixture(t, assembly.String(), declarations.String(), runtime.GOOS+"/amd64", execution.triple) + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, []byte(assembly.String())) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalPackedRunOriginal(t, execution, assembly.String(), declarations.String(), goOracle, true) + fixedLocalDataAPIs(t, pkg, file, expanded, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + input := ir + if level == "-O2" { + input = x86MulDivRoundOptimize(t, ir) + } + execution.compileAndRunLLVM(t, "readonly-frame", input, cOracle, level) + } + }) + t.Log("2048 independent numeric outcomes: original Go and three LLVM APIs at O0/O2; includes sparse zero tails and unaligned/partial scalar reads") + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("Rosetta legacy SSE diagnostic, not native Linux or AVX/AVX512 execution") + } +} diff --git a/amd64_fixed_local_frame_test.go b/amd64_fixed_local_frame_test.go new file mode 100644 index 000000000..281b47bf9 --- /dev/null +++ b/amd64_fixed_local_frame_test.go @@ -0,0 +1,714 @@ +package plan9asm + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" +) + +func fixedLocalFrameFixture(t *testing.T, operation string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + source, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + declaration := []byte("package framefixture\nfunc Value(a, b, c, d uint64) uint64\n") + for name, data := range map[string][]byte{ + "value.go": declaration, "value_amd64.s": []byte(source), + } { + if err := os.WriteFile(filepath.Join(request.Directory, name), data, 0600); err != nil { + t.Fatal(err) + } + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", declaration, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check("example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + sig, err := DeriveGoFuncABI0Signature(typed.Path()+".Value", typed.Scope().Lookup("Value").(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", typed.Path()+".", 1) } + pkg := GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}} + return pkg, request, file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, + Sigs: map[string]FuncSig{sig.Name: sig}, + } +} + +func TestGoAMD64FixedLocalFrameSevenOriginalNativeOperations(t *testing.T) { + for _, operation := range []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + } { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, operation) + source := requireGoAMD64FrameCapture(t, pkg, request) + observed := source.functions[pkg.Path+".Value"] + if observed.locals != 168 || observed.args != 40 || len(observed.code) == 0 { + t.Fatalf("missing actual original Go frame/object: %+v", observed) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + options, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, options, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatalf("original numeric frame needs sealed native backing: %v", err) + } + for _, required := range []string{"fixed_local_native", "align 8", "asm sideeffect"} { + if !strings.Contains(ir, required) { + t.Fatalf("missing actual native storage/opcode %q", required) + } + } + }) + }) + } +} + +func TestGoAMD64FixedLocalFrameActualTargetObjectsPreserveProducerBoundary(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64", "linux/386", "windows/386"} { + t.Run(target, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + request.Target = target + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if strings.HasSuffix(target, "/386") || target == "windows/amd64" && err != nil { + if !errors.Is(err, ErrProbeNeedsContext) || source != nil || + target == "windows/amd64" && !strings.Contains(err.Error(), "SSEHUNWINDINFO") { + t.Fatalf("unclosed physical target unexpectedly obtained authority: %v", err) + } + t.Logf("existing actual observer boundary remains Context, not a positive fixed-local object: %v", err) + return + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + if target == "darwin/amd64" { + options.TargetTriple = "x86_64-apple-darwin" + } + if target == "windows/amd64" { + options.TargetTriple = "x86_64-pc-windows-msvc" + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, options.TargetTriple, "fixed-local.ll", "fixed-local.o", ir) + compileLLVMToObject(t, llc, options.TargetTriple, "fixed-local-opt.ll", "fixed-local-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64FixedLocalFrameScalarMemoryWidthsHaveExactCertificates(t *testing.T) { + for _, form := range []struct { + opcode string + width int64 + }{{"MOVB", 1}, {"MOVW", 2}, {"MOVL", 4}, {"MOVQ", 8}} { + t.Run(form.opcode, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + var source, stores strings.Builder + for _, ins := range file.Funcs[0].Instrs { + source.WriteString(ins.Raw + "\n") + } + for offset := int64(16); offset < 32; offset += form.width { + fmt.Fprintf(&stores, "%s $1,%d(SP)\n", form.opcode, offset) + } + fmt.Fprintf(&stores, "%s 16(SP),DX\n", form.opcode) + original := strings.Replace(source.String(), "MOVQ AX,16(SP)\nMOVQ CX,24(SP)\n", stores.String(), 1) + requireX86GoAssemblerResult(t, "amd64", original, true) + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(original), 0600); err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, original) + if err != nil { + t.Fatal(err) + } + observed := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := observed.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", bound) + if err != nil { + t.Fatal(err) + } + read, write := false, false + for _, ins := range normalized.Funcs[0].Instrs { + if string(ins.Op) != form.opcode { + continue + } + for _, access := range ins.x86FixedLocalMemory { + if access.width != form.width { + t.Fatalf("scalar byte envelope changed: %+v", access) + } + read = read || !access.write + write = write || access.write + } + } + if !read || !write { + t.Fatal("missing independently proved scalar read/write") + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || ir == "" { + t.Fatalf("actual scalar memory width failed consumer: %v", err) + } + }) + }) + } +} + +func fixedLocalAPIs(t *testing.T, pkg GoPackage, file *File, options Options, check func(*testing.T, string, error)) { + t.Helper() + t.Run("Translate", func(t *testing.T) { + ir, err := Translate(file, options) + check(t, ir, err) + }) + t.Run("Module", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + ir := "" + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + check(t, ir, err) + }) + t.Run("GoModule", func(t *testing.T) { + var source strings.Builder + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + source.WriteString(ins.Raw + "\n") + } + } + goos := "linux" + if strings.Contains(options.TargetTriple, "apple") { + goos = "darwin" + } + if strings.Contains(options.TargetTriple, "windows") { + goos = "windows" + } + translation, err := TranslateGoModule(pkg, []byte(source.String()), GoModuleOptions{ + GOOS: goos, GOARCH: options.Goarch, + TargetTriple: options.TargetTriple, Sizes: types.SizesFor("gc", options.Goarch), WASMABI: options.WASMABI, + ResolveSym: options.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { sig, found := options.Sigs[name]; return sig, found }, + }) + ir := "" + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + ir = translation.Module.String() + } + check(t, ir, err) + }) +} + +func fixedLocalRequireContext(t *testing.T, ir string, err error) { + t.Helper() + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("unbound frame/effect produced IR: %v %s", err, ir) + } +} + +func TestGoAMD64FixedLocalFrameThreeAPIsCannotForgeOrMutateBinding(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + fixedLocalAPIs(t, pkg, file, options, fixedLocalRequireContext) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + for _, mutation := range []string{"JSON", "signature", "slot", "attrs", "target", "arch", "resolver", "source", "missing function", "ABI"} { + t.Run(mutation, func(t *testing.T) { + changed := bound + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + for name, sig := range changed.Sigs { + sig.x86FixedLocal = bound.Sigs[name].x86FixedLocal + changed.Sigs[name] = sig + } + changedFile := *file + changedFile.Funcs = append([]Func(nil), file.Funcs...) + sig := changed.Sigs[pkg.Path+".Value"] + switch mutation { + case "JSON": + changed.Sigs, err = cloneFixedLocalSigs(bound.Sigs) + if err != nil { + t.Fatal(err) + } + case "signature": + sig.Args[0] = I32 + case "slot": + sig.Frame.Params[0].Offset++ + case "attrs": + sig.Attrs = "nounwind" + case "target": + changed.TargetTriple = "x86_64-apple-darwin" + case "arch": + changed.Goarch = "386" + case "resolver": + changed.ResolveSym = func(string) string { return "foreign.Value" } + case "ABI": + changed.WASMABI = WASMABIGo + case "source": + changedFile.Funcs[0].FrameSize += 8 + changedFile.Funcs[0].Instrs = append([]Instr(nil), file.Funcs[0].Instrs...) + changedFile.Funcs[0].Instrs[0].Raw = strings.Replace(changedFile.Funcs[0].Instrs[0].Raw, "$160-40", "$168-40", 1) + case "missing function": + changedFile.Funcs = nil + } + if mutation != "JSON" { + changed.Sigs[pkg.Path+".Value"] = sig + } + check := fixedLocalRequireContext + if mutation == "missing function" { + check = func(t *testing.T, ir string, err error) { + if err == nil || ir != "" { + t.Fatalf("missing entire source produced a module: %v", err) + } + } + } + fixedLocalAPIs(t, pkg, &changedFile, changed, check) + }) + } + // Mutating the original input does not mutate the returned signature nor + // the separately cloned sealed expected declaration. + input := options.Sigs[pkg.Path+".Value"] + input.Args[0] = I32 + options.Sigs[pkg.Path+".Value"] = input + if _, err := Translate(file, bound); err != nil { + t.Fatalf("caller-owned Options changed the immutable binding: %v", err) + } + // Tail groups exist only in Options, not in GoModuleOptions. Exercise the + // actual two consumers that can receive this request, not a fake transport. + changed := bound + changed.X86TailGroups = []X86TailGroup{{Root: "·Value", Helpers: []string{"·Value"}}} + ir, err := Translate(file, changed) + fixedLocalRequireContext(t, ir, err) + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + module, err := TranslateModuleInContext(llvmContext, file, changed) + if !module.IsNil() { + defer module.Dispose() + t.Fatal("unbound continuation request produced a module") + } + fixedLocalRequireContext(t, "", err) +} + +func TestGoAMD64FixedLocalFrameBindResolverMayRevokeWithoutLockingDeadlock(t *testing.T) { + if os.Getenv("PLAN9ASM_FIXED_LOCAL_REENTRANCY_CHILD") != "1" { + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestGoAMD64FixedLocalFrameBindResolverMayRevokeWithoutLockingDeadlock$", "-test.timeout=1m") + command.Env = append(os.Environ(), "PLAN9ASM_FIXED_LOCAL_REENTRANCY_CHILD=1") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("bounded actual bind resolver/Close child failed or deadlocked: %v\n%s", err, output) + } + return + } + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + resolve := options.ResolveSym + options.ResolveSym = func(symbol string) string { + if err := source.Close(); err != nil { + t.Fatal(err) + } + return resolve(symbol) + } + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("callback-revoked observation obtained a binding: %v", err) + } +} + +func TestGoAMD64FixedLocalFrameTruncatedConstantCannotAuthorizePointerArithmetic(t *testing.T) { + for _, move := range []string{"MOVL", "MOVW", "MOVB"} { + t.Run(move, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + var original strings.Builder + for _, ins := range file.Funcs[0].Instrs { + original.WriteString(ins.Raw + "\n") + } + changed := strings.Replace(original.String(), "MOVQ $31,R9\nNOTQ R9\nANDQ R9,SP", "MOVQ $-16,AX\n"+move+" AX,BX\nANDQ BX,SP", 1) + requireX86GoAssemblerResult(t, "amd64", changed, true) + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(changed), 0600); err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual Go accepted %s narrow constant acquired full-width pointer authority: %v", move, err) + } + }) + } +} + +func TestGoAMD64FixedLocalFrameClosedExpiredAndChangedSourceRevokes(t *testing.T) { + for _, mutation := range []string{"close", "cancel", "file", "object", "listing", "export", "post-consumer"} { + t.Run(mutation, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + stage, finish := runtimeOracleStageContext(t) + defer finish() + ctx, cancel := context.WithCancel(stage) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + var path string + switch mutation { + case "close": + if err := source.Close(); err != nil { + t.Fatal(err) + } + case "cancel": + cancel() + case "file": + path = filepath.Join(request.Directory, "value.go") + case "object": + path = filepath.Join(source.owned, request.AssemblyFile+".o") + case "listing": + path = filepath.Join(source.owned, request.AssemblyFile+".o.listing") + case "export": + path = source.selection.Export + case "post-consumer": + calls := 0 + bound.ResolveSym = func(symbol string) string { + calls++ + if calls == 1 { + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package changed\n"), 0600); err != nil { + t.Fatal(err) + } + } + return options.ResolveSym(symbol) + } + } + if path != "" { + if err := os.WriteFile(path, []byte("mutation"), 0600); err != nil { + t.Fatal(err) + } + } + fixedLocalAPIs(t, pkg, file, bound, fixedLocalRequireContext) + if mutation != "post-consumer" && source.ready { + t.Fatal("failed observer verification did not permanently revoke the binding") + } + }) + } +} + +func TestGoAMD64FixedLocalFrameResolverCannotChangeParsedSourceAfterValidation(t *testing.T) { + for _, api := range []string{"Translate", "Module", "GoModule"} { + t.Run(api, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + resolve := bound.ResolveSym + bound.ResolveSym = func(symbol string) string { + for index := range file.Funcs[0].Instrs { + ins := &file.Funcs[0].Instrs[index] + if ins.Op == "PADDQ" { + ins.Op = "PADDB" + ins.Raw = strings.Replace(ins.Raw, "PADDQ", "PADDB", 1) + } + } + return resolve(symbol) + } + if api == "Translate" { + ir, err := Translate(file, bound) + fixedLocalRequireContext(t, ir, err) + return + } + if api == "Module" { + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + module, err := TranslateModuleInContext(llvmContext, file, bound) + if !module.IsNil() { + defer module.Dispose() + t.Fatal("resolver-mutated original source produced a module") + } + fixedLocalRequireContext(t, "", err) + return + } + // GoModule reparses the original bytes into its own private file; + // this public-file mutation cannot change that source. Its resolver + // is frozen before consumer normalization, so the original opcode + // must remain independently bound rather than silently substituted. + original := strings.Builder{} + for _, ins := range file.Funcs[0].Instrs { + original.WriteString(ins.Raw + "\n") + } + translation, err := TranslateGoModule(pkg, []byte(original.String()), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: bound.TargetTriple, + Sizes: types.SizesFor("gc", "amd64"), ResolveSym: bound.ResolveSym, + ManualSig: func(name string) (FuncSig, bool) { sig, found := bound.Sigs[name]; return sig, found }, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + if err != nil || translation == nil || !strings.Contains(translation.Module.String(), "paddq") { + t.Fatalf("GoModule did not preserve its independent original source: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalFrameProofFullByteWidthAddressAndControlNegatives(t *testing.T) { + base, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "PADDQ 16(SP),X0") + mutations := map[string]string{ + "alignment": strings.Replace(base, "ADDQ $32,SP", "ADDQ $33,SP", 1), + "read extent": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 152(SP)", 1), + "write extent": strings.Replace(base, "MOVQ AX,16(SP)", "MOVQ AX,160(SP)", 1), + "unknown pointer": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 16(SI)", 1), + "indexed pointer": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 16(SP)(AX*1)", 1), + "segment": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 16(FS)", 1), + "pointer result": strings.Replace(base, "MOVQ AX,ret+32(FP)", "MOVQ R11,ret+32(FP)", 1), + "pointer flags": strings.Replace(base, "MOVQ AX,ret+32(FP)", "CMPQ R11,AX\nMOVQ AX,ret+32(FP)", 1), + "pointer truncated": strings.Replace(base, "MOVQ SP,R11", "MOVL SP,R11", 1), + "missing restore": strings.Replace(base, "MOVQ R11,SP", "MOVQ $0,R10", 1), + "pointer cell clobber": strings.Replace(base, "MOVQ 96(SP),R11", "MOVB $1,96(SP)\nMOVQ 96(SP),R11", 1), + "pointer bytes as numeric": strings.Replace(base, "PADDQ 16(SP)", "PADDQ 96(SP)", 1), + "unknown call": strings.Replace(base, "PADDQ", "CALL ·Other(SB)\nPADDQ", 1), + "unknown dead effect": base + "SYSCALL\n", + "result width": strings.Replace(base, "MOVQ AX,ret+32(FP)", "MOVL AX,ret+32(FP)", 1), + "local spill escape": strings.Replace(base, "MOVQ AX,ret+32(FP)", "MOVQ R11,·global(SB)\nMOVQ AX,ret+32(FP)", 1), + "unknown flags branch": strings.Replace(base, "MOVQ AX,16(SP)", "JEQ done\nMOVQ AX,16(SP)", 1) + "done:\nRET\n", + "FP after SP change": strings.Replace(base, "MOVQ AX,16(SP)", "MOVQ a+0(FP),AX\nMOVQ AX,16(SP)", 1), + "high legacy vector bank": strings.ReplaceAll(base, "X0", "X16"), + } + for byteIndex := 0; byteIndex < 16; byteIndex++ { + var stores strings.Builder + for index := 0; index < 16; index++ { + if index != byteIndex { + fmt.Fprintf(&stores, "MOVB $1,%d(SP)\n", 16+index) + } + } + mutated := strings.Replace(base, "MOVQ AX,16(SP)\nMOVQ CX,24(SP)\n", stores.String(), 1) + mutations[fmt.Sprintf("missing byte %02d", byteIndex)] = mutated + } + for name, source := range mutations { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source admitted: %v", err) + } + }) + } +} + +func TestGoAMD64FixedLocalFrameAllPathInitializationAndResourceBounds(t *testing.T) { + base, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "PADDQ 16(SP),X0") + branch := strings.Replace(base, "MOVQ CX,24(SP)", "MOVQ CX,24(SP)\nCMPQ AX,CX\nJEQ joined\nMOVQ CX,24(SP)\njoined:", 1) + requireX86GoAssemblerResult(t, "amd64", branch, true) + file, err := Parse(ArchAMD64, branch) + if err != nil { + t.Fatal(err) + } + proved, err := proveX86FixedLocal(file.Funcs[0], sig) + if err != nil { + t.Fatal(err) + } + for _, ins := range proved.Instrs { + if ins.Op != "PADDQ" { + continue + } + access := ins.x86FixedLocalMemory[0] + if access.offset != [8]int64{48, 40, 32, 24, 48, 40, 32, 24} { + t.Fatalf("missing all original SP residue accesses: %v", access.offset) + } + } + for name, source := range map[string]string{ + "one predecessor uninitialized": strings.Replace(branch, "MOVQ CX,24(SP)\nCMPQ", "CMPQ", 1), + "stale pointer flags": strings.Replace(branch, "CMPQ AX,CX", "ADDQ $0,SP", 1), + "loop": strings.Replace(branch, "JEQ joined", "before:\nJEQ before", 1), + "different address join": strings.Replace(branch, "JEQ joined\nMOVQ CX,24(SP)", "JEQ joined\nADDQ $8,SP\nMOVQ CX,24(SP)", 1), + "fallthrough": strings.TrimSuffix(base, "RET\n"), + } { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved CFG gained backing: %v", err) + } + }) + } + tooLarge := file.Funcs[0] + tooLarge.FrameSize = 1 << 20 + if _, err := proveX86FixedLocal(tooLarge, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("oversized physical frame was admitted") + } + tooLarge.FrameSize = 64 << 10 + tooLarge.Instrs = append([]Instr(nil), tooLarge.Instrs...) + for len(tooLarge.Instrs) <= 32 { + tooLarge.Instrs = append(tooLarge.Instrs, Instr{Op: "MOVQ", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + } + if _, err := proveX86FixedLocal(tooLarge, sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("byte-state product budget was ignored") + } +} + +func TestGoAMD64FixedLocalFrameResidueDependentAliasesCannotRelocate(t *testing.T) { + base, sig, _ := x86PrivateAlignedFrameAuditFixture(t, "PADDQ 16(SP),X0") + // Both address origins belong to the same local frame, and every byte is + // initialized for every residue. Nevertheless this write overlaps the + // source's low word, high word or neither depending on original SP mod 32. + // A new physical alloca cannot silently choose one of these observations. + source := strings.Replace(base, "MOVQ BX,56(SP)", "MOVQ BX,56(SP)\nMOVQ $1,48(R11)", 1) + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("residue-dependent initialized aliases were relocated: %v", err) + } + // Independent little-endian byte-memory mathematics, not the production + // address/state/certificate implementation, exposes the actual data change. + want := [4][2]uint64{{6, 10}, {7, 8}, {7, 10}, {7, 10}} + for residue := 0; residue < 4; residue++ { + var memory [256]byte + sp := 32 + residue*8 + aligned := sp&^31 + 32 + for index, value := range []uint64{2, 3} { + binary.LittleEndian.PutUint64(memory[aligned+16+index*8:], value) + } + for index, value := range []uint64{5, 7} { + binary.LittleEndian.PutUint64(memory[aligned+48+index*8:], value) + } + binary.LittleEndian.PutUint64(memory[sp+48:], 1) + got := [2]uint64{} + for index := range got { + got[index] = binary.LittleEndian.Uint64(memory[aligned+16+index*8:]) + + binary.LittleEndian.Uint64(memory[aligned+48+index*8:]) + } + if got != want[residue] { + t.Fatalf("independent original residue %d data: got %v want %v", residue, got, want[residue]) + } + } +} + +func TestGoAMD64FixedLocalFrameCertificatesArePrivateExactAndRevocable(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", bound) + if err != nil { + t.Fatal(err) + } + fn := normalized.Funcs[0] + lower := &amd64Ctx{fixedLocal: fn.x86FixedLocal} + for _, ins := range fn.Instrs { + for index, access := range ins.x86FixedLocalMemory { + if !lower.fixedLocalMemoryCertified(ins, index) { + t.Fatal("missing independently replayed original access") + } + for residue, offset := range access.offset { + if offset < 0 || offset+access.width > 160 { + t.Fatalf("invalid residue %d certified extent", residue) + } + } + for _, mutation := range []string{"op", "args", "width", "owner", "JSON", "address size"} { + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86FixedLocalMemory = make(map[int]x86FixedLocalAccess) + for key, value := range ins.x86FixedLocalMemory { + changed.x86FixedLocalMemory[key] = value + } + switch mutation { + case "op": + changed.Op = "UNDEF" + case "args": + changed.Args[index].Mem.Off++ + case "width": + value := changed.x86FixedLocalMemory[index] + value.width++ + changed.x86FixedLocalMemory[index] = value + case "owner": + value := changed.x86FixedLocalMemory[index] + value.owner = &x86FixedLocalProof{} + changed.x86FixedLocalMemory[index] = value + case "JSON": + data, _ := json.Marshal(ins) + changed = Instr{} + if err := json.Unmarshal(data, &changed); err != nil { + t.Fatal(err) + } + case "address size": + changed.x86AddressBits = 32 + } + if lower.fixedLocalMemoryCertified(changed, index) { + t.Fatalf("%s preserved a stale native access certificate", mutation) + } + } + } + } +} diff --git a/amd64_fixed_local_numerical.go b/amd64_fixed_local_numerical.go new file mode 100644 index 000000000..4c8999215 --- /dev/null +++ b/amd64_fixed_local_numerical.go @@ -0,0 +1,39 @@ +package plan9asm + +// Only an exact private read certificate supplies the SB bytes. The local +// proof learns numerical definedness, never a constant, pointer or address. +func fixedLocalNumericalReadStep(ins Instr, state *x86FixedLocalState) (bool, error) { + if ins.x86NumericalRead == nil { + return false, nil + } + effect, valid := x86NumericalReadEffect(ins, "amd64") + if !valid { + return true, goAMD64FrameContext("fixed local numerical read certificate changed") + } + for _, operand := range effect.operands { + if operand.read && !fixedLocalNumericRead(operand.operand, operand.width, state) { + return true, goAMD64FrameContext("fixed local numerical compare needs initialized numeric GP bytes") + } + if !operand.write { + continue + } + root, valid := x86CallerGP(operand.operand.Reg) + mask := x86CallerByteMask(operand.operand.Reg, operand.width) + _, pointer := state.pointers[root] + if !valid || root == SP || root == BP || x86SourceGRegister(root) || mask == 0 || pointer && operand.width != 8 { + return true, goAMD64FrameContext("fixed local numerical read cannot replace a private address or stack role") + } + if operand.width == 4 { + mask = 255 // Architectural dword writes zero the upper GP bytes. + } + delete(state.pointers, root) + delete(state.opaque, root) + delete(state.constants, root) + fixedLocalWriteBooleans(state, operand.operand.Reg, operand.width, 0) + state.known[root] |= mask + } + if effect.flagsDefined != 0 { + state.flags = true + } + return true, nil +} diff --git a/amd64_fixed_local_original_test.go b/amd64_fixed_local_original_test.go new file mode 100644 index 000000000..1e29f1006 --- /dev/null +++ b/amd64_fixed_local_original_test.go @@ -0,0 +1,43 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +// Shared by portable object tests and the Unix native runtime oracles. +var fixedLocalNativeOperations = []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", +} + +func fixedLocalPackedRunOriginal(t *testing.T, execution x86FloatCompareExecution, assembly, declarations, oracle string, execute bool) { + t.Helper() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module example.invalid/framefixture\ngo 1.20\n", + "value.go": "package framefixture\n" + declarations, + "value_amd64.s": assembly, "value_test.go": oracle, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + binary := filepath.Join(dir, gotoolchain.HostExecutable("original.test")) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-c", "-p=2", "-vet=off", "-o", binary, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original Go packed-frame oracle: %v\n%s", err, output) + } + if execute { + t.Logf("original Go: %s", execution.run(t, binary, "-test.run=^TestOriginal$")) + } +} diff --git a/amd64_fixed_local_packed.go b/amd64_fixed_local_packed.go new file mode 100644 index 000000000..57c82bbbc --- /dev/null +++ b/amd64_fixed_local_packed.go @@ -0,0 +1,127 @@ +package plan9asm + +// Reuse the complete source grammars and their byte effects. This adapter adds +// independently replayed private-frame memory and separately bound literal +// reads; it never supplies a caller, FP, incoming-pointer or raw capability. +func fixedLocalPackedMoveStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + form, known, err := amd64PackedVectorMoveProperties(*ins, "amd64") + legacy := false + alignment := int64(1) + if !known { + original, recognized, originalErr := x86LegacyPackedMoveProperties(ins.Op, *ins, "amd64") + if !recognized { + return false, nil + } + known, err, legacy = true, originalErr, true + form = amd64PackedVectorMoveForm{ + byteWidth: 16, destination: 1, mask: -1, + memory: original.memory, memoryWrite: original.memoryWrite, + } + form.spec.aligned = original.aligned + } + if err != nil || form.undefinedOpcode() { + return known, goAMD64FrameContext("fixed local packed move has no numerical continuation") + } + if form.spec.aligned { + alignment = int64(form.byteWidth) + } + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return true, goAMD64FrameContext("fixed local packed move has an undefined mask") + } + width := int64(form.byteWidth) + widthMask := x86CallerVectorByteMask(width) + numerical := widthMask + if form.memory == 0 { + // The private proof does not model runtime mask values: require every + // possible read byte. Lowering still executes the original masked + // opcode, including its alignment and masked fault behavior. + if _, _, err := fixedLocalReadMemory(ins, 0, state, proof, residue, width, alignment); err != nil { + return true, err + } + } else { + numerical = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] & widthMask + } + if form.memoryWrite { + if numerical != widthMask { + return true, goAMD64FrameContext("fixed local packed store has undefined source bytes") + } + offset, err := fixedLocalMemory(ins, form.memory, state, proof, residue, width, alignment, true, false) + if err != nil { + return true, err + } + // A runtime mask is not an all-one mask. A partial store may invalidate + // a private pointer spill, but cannot initialize its retained bytes. + fixedLocalNumericalStore(state, offset, width, form.mask < 0) + return true, nil + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + previous := state.vectors[destination] + numerical = form.definedActiveBytes(numerical, previous) + if legacy { + state.vectors[destination] = previous&^widthMask | numerical + } else { + state.vectors[destination] = ^widthMask | numerical + } + return true, nil +} + +func fixedLocalNumericalStore(state *x86FixedLocalState, offset, width int64, complete bool) { + fixedLocalRevokeSpillRoles(state, offset, width) + if complete { + for index := offset; index < offset+width; index++ { + state.bytes[index] = true + } + } +} + +// KMOV supplies numerical predicates through its existing four-width grammar. +// Full-word private addresses never become mask values. Local reads/writes +// retain the same byte envelope and owner as every other fixed-frame access. +func fixedLocalMaskMoveStep(ins *Instr, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int) (bool, error) { + bits, known := amd64MaskMoveSpecs[ins.Op] + if !known { + return false, nil + } + fail := func() (bool, error) { + return true, goAMD64FrameContext("fixed local mask move has unproved shape or numerical bytes") + } + if !amd64MaskMoveOperandsValid(*ins, "amd64") { + return fail() + } + width := int64(bits / 8) + source, destination := ins.Args[0], ins.Args[1] + if source.Kind == OpMem || source.Kind == OpSym { + if _, _, err := fixedLocalReadMemory(ins, 0, state, proof, residue, width, 1); err != nil { + return true, err + } + } else if amd64IsKOperand(source) { + if !state.masks[source.Reg] { + return fail() + } + } else if !fixedLocalNumericRead(source, width, state) { + return fail() + } + if destination.Kind == OpMem { + offset, err := fixedLocalMemory(ins, 1, state, proof, residue, width, 1, true, false) + if err != nil { + return true, err + } + fixedLocalNumericalStore(state, offset, width, true) + return true, nil + } + if amd64IsKOperand(destination) { + state.masks[destination.Reg] = true + return true, nil + } + root, valid := x86CallerGP(destination.Reg) + if !valid || root == SP || root == BP || x86SourceGRegister(root) { + return fail() + } + // Every KMOV GP destination zero-extends to the full architectural word. + delete(state.pointers, root) + delete(state.opaque, root) + delete(state.constants, root) + fixedLocalWriteBooleans(state, destination.Reg, 8, 0) + state.known[root] = 255 + return true, nil +} diff --git a/amd64_fixed_local_packed_effects_test.go b/amd64_fixed_local_packed_effects_test.go new file mode 100644 index 000000000..39dc264d9 --- /dev/null +++ b/amd64_fixed_local_packed_effects_test.go @@ -0,0 +1,169 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestGoAMD64FixedLocalPackedMasksAndUpperAliases(t *testing.T) { + for _, row := range []struct { + name, body string + valid bool + }{ + {"legacy preserves unknown upper", "MOVOU 64(BP),X0\nVMOVDQU Y0,128(BP)", false}, + {"VEX zeroes upper", "VMOVDQU 64(BP),X0\nVMOVDQU64 Z0,128(BP)", true}, + {"EVEX zeroes upper", "VMOVDQU64 64(BP),Y31\nVMOVDQU64 Z31,128(BP)", true}, + {"legacy retains known upper", "VMOVDQU64 64(BP),Z0\nMOVOU 64(BP),X0\nVMOVDQU64 Z0,128(BP)", true}, + {"merge needs old bytes", "KMOVQ AX,K1\nVMOVDQA64 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", false}, + {"zero needs no old bytes", "KMOVQ AX,K1\nVMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", true}, + {"zero cannot bless unknown source", "KMOVQ AX,K1\nVMOVDQA64.Z Z1,K1,Z0\nVMOVDQA64 Z0,128(BP)", false}, + {"mask is undefined", "VMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", false}, + {"unknown source store", "VMOVDQA64 Z0,128(BP)", false}, + {"Go zero store traps", "VMOVDQA64 64(BP),Z0\nKMOVQ AX,K1\nVMOVDQA64.Z Z0,K1,128(BP)", false}, + } { + t.Run(row.name, func(t *testing.T) { + pkg, _, file, options := fixedLocalPackedFixture(t, row.body) + requireX86GoAssemblerResult(t, "amd64", fixedLocalPackedAssembly(row.body), true) + _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]) + if row.valid && err != nil || !row.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("private byte transfer valid=%v: %v", row.valid, err) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedMaskedStoreCannotInitializeUnknownBytes(t *testing.T) { + const body = "VMOVDQA64 64(BP),Z0\nKMOVQ AX,K1\nVMOVDQA64 Z0,K1,128(BP)" + pkg, _, _, options := fixedLocalPackedFixture(t, body) + sig := options.Sigs[pkg.Path+".Value"] + for missing := 0; missing < 64; missing++ { + t.Run(fmt.Sprint(missing), func(t *testing.T) { + assembly := fixedLocalPackedAssembly(body) + var stores strings.Builder + for index := 0; index < 8; index++ { + if index != missing%8 { + fmt.Fprintf(&stores, "MOVB $1,%d(BP)\n", 128+missing/8*8+index) + } + } + assembly = strings.Replace(assembly, fmt.Sprintf("MOVQ AX,%d(BP)\n", 128+missing/8*8), stores.String(), 1) + assembly = strings.Replace(assembly, "MOVQ 128(BP),AX", fmt.Sprintf("MOVQ %d(BP),AX", 128+missing/8*8), 1) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("dynamic mask initialized retained byte %d: %v", missing, err) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedMaskMovesCompleteLocalRoles(t *testing.T) { + for _, opcode := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + t.Run(opcode, func(t *testing.T) { + body := fmt.Sprintf("%s AX,K0\n%s K0,K7\n%s K7,128(BP)\n%s 64(BP),K1\n%s K1,CX\nVMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", opcode, opcode, opcode, opcode, opcode) + pkg, _, file, options := fixedLocalPackedFixture(t, body) + requireX86GoAssemblerResult(t, "amd64", fixedLocalPackedAssembly(body), true) + sig := options.Sigs[pkg.Path+".Value"] + if _, err := proveX86FixedLocal(file.Funcs[0], sig); err != nil { + t.Fatal("complete local mask transfer", err) + } + for name, changed := range map[string]string{ + "private pointer": strings.Replace(body, opcode+" AX,K0", opcode+" BP,K0", 1), + "unknown GP": strings.Replace(body, opcode+" AX,K0", opcode+" DX,K0", 1), + "unknown K": strings.Replace(body, opcode+" AX,K0", opcode+" K2,K0", 1), + "BP output": strings.Replace(body, opcode+" K1,CX", opcode+" K1,BP", 1), + "SP output": strings.Replace(body, opcode+" K1,CX", opcode+" K1,SP", 1), + "outside frame": strings.Replace(body, "128(BP)", "256(BP)", 1), + "global output": strings.Replace(body, "128(BP)", "external(SB)", 1), + } { + t.Run(name, func(t *testing.T) { + file, err := Parse(ArchAMD64, fixedLocalPackedAssembly(changed)) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("mask move acquired another native role", err) + } + }) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedCFGMergeMasksAndBytes(t *testing.T) { + for name, body := range map[string]string{ + "mask omitted on one path": "CMPQ AX,AX\nJEQ joined\nKMOVQ AX,K1\njoined:\nVMOVDQA64.Z 64(BP),K1,Z0\nVMOVDQA64 Z0,128(BP)", + "vector omitted on one path": "CMPQ AX,AX\nJEQ joined\nVMOVDQA64 64(BP),Z0\njoined:\nVMOVDQA64 Z0,128(BP)", + "BP differs on one path": "CMPQ AX,AX\nJEQ joined\nADDQ $64,BP\njoined:\nVMOVDQA64 64(BP),Z0\nVMOVDQA64 Z0,128(BP)", + } { + t.Run(name, func(t *testing.T) { + pkg, _, file, options := fixedLocalPackedFixture(t, body) + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("predecessor lost its must-defined/source role", err) + } + }) + } +} + +func TestGoAMD64FixedLocalPackedMaskedStoreRevokesPrivatePointerSpills(t *testing.T) { + for _, mask := range []string{"AX", "$0", "$1", "$-1"} { + for _, use := range []string{"MOVQ 32(BP),R11\nMOVQ 64(R11),AX", "MOVQ 32(BP),AX"} { + t.Run(mask+"/"+use, func(t *testing.T) { + body := "VMOVDQU64 64(BP),Z0\nMOVQ BP,32(BP)\nMOVQ " + mask + ",DX\nKMOVQ DX,K1\nVMOVDQU64 Z0,K1,32(BP)\n" + use + pkg, _, file, options := fixedLocalPackedFixture(t, body) + requireX86GoAssemblerResult(t, "amd64", fixedLocalPackedAssembly(body), true) + if _, err := proveX86FixedLocal(file.Funcs[0], options.Sigs[pkg.Path+".Value"]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("partial store preserved pointer authority or invented numerical bytes", err) + } + }) + } + } + // This domain proves mask definedness, not constant mask values. Even a + // zero mask must conservatively revoke the overlapping pointer certificate. + state := x86FixedLocalState{bytes: make([]bool, 96), booleanBytes: make([]bool, 96), addresses: map[int64]int64{32: 8, 80: 16}} + fixedLocalNumericalStore(&state, 36, 16, false) + if _, found := state.addresses[32]; found || state.addresses[80] != 16 { + t.Fatal("partial overlap did not revoke exactly the overlapping pointer spill") + } + for index, known := range state.bytes { + if known { + t.Fatalf("masked store invented initialized byte %d", index) + } + } +} + +func TestGoAMD64FixedLocalPackedMergeRequiresEveryDestinationByte(t *testing.T) { + for _, bank := range []struct { + name string + width int64 + }{{"X", 16}, {"Y", 32}, {"Z", 64}} { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$256-0\nVMOVDQU8 "+bank.name+"1,K1,"+bank.name+"0\nVMOVDQU8 "+bank.name+"0,0(BP)\nRET\n") + if err != nil { + t.Fatal(err) + } + for missing := int64(0); missing < bank.width; missing++ { + t.Run(fmt.Sprintf("%s/%d", bank.name, missing), func(t *testing.T) { + state := x86FixedLocalState{ + pointers: map[Reg]int64{BP: 0}, masks: map[Reg]bool{"K1": true}, + vectors: map[Reg]uint64{"Z0": ^(uint64(1) << uint(missing)), "Z1": ^uint64(0)}, + bytes: make([]bool, 256), addresses: make(map[int64]int64), + } + proof := &x86FixedLocalProof{frame: 256, accesses: make(map[int]map[int]x86FixedLocalAccess)} + copy := file.Funcs[0].Instrs[1] + if known, err := fixedLocalPackedMoveStep(©, &state, proof, 0); !known || err != nil { + t.Fatalf("register merge: known=%v error=%v", known, err) + } + if got, want := state.vectors["Z0"], ^(uint64(1) << uint(missing)); got != want { + t.Fatalf("merge/upper-zero bytes=%016x want=%016x", got, want) + } + store := file.Funcs[0].Instrs[2] + if _, err := fixedLocalPackedMoveStep(&store, &state, proof, 0); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("merge result exposed an undefined destination byte", err) + } + }) + } + } +} diff --git a/amd64_fixed_local_packed_matrix_test.go b/amd64_fixed_local_packed_matrix_test.go new file mode 100644 index 000000000..0342ad9b9 --- /dev/null +++ b/amd64_fixed_local_packed_matrix_test.go @@ -0,0 +1,189 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func fixedLocalPackedCompleteProgram() (string, string, int) { + var assembly, declarations strings.Builder + count := 0 + for _, row := range fixedLocalPackedMoveOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if bank == "Z" && !row.evex { + continue + } + for _, high := range []bool{false, true} { + first, last := bank+"0", bank+"1" + if high { + first, last = bank+"15", bank+"14" + if row.evex { + first, last = bank+"31", bank+"30" + } + } + for _, predicate := range []string{"none", "merge", "zero"} { + if predicate != "none" && !row.evex { + continue + } + operation := fmt.Sprintf("%s 64(BP),%s\n%s 128(BP),%s\n", row.name, first, row.name, last) + mask, opcode := "", row.name + if predicate != "none" { + operation += "KMOVQ AX,K7\n" + mask = "K7," + } + if predicate == "zero" { + opcode += ".Z" + } + operation += fmt.Sprintf("%s 64(BP),%s%s\n%s %s,%s%s\n%s %s,%s128(BP)", + opcode, mask, first, opcode, first, mask, last, row.name, last, mask) + name := fmt.Sprintf("Value%d", count) + assembly.WriteString(strings.Replace(fixedLocalPackedAssembly(operation), "·Value", "·"+name, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", name) + count++ + } + } + } + } + // Legacy aliases retain upper bytes and share the exact same local envelope. + for _, name := range []string{"MOVO", "MOVOA", "MOVOU", "MOVAPS", "MOVAPD", "MOVUPS", "MOVUPD"} { + operation := fmt.Sprintf("%s 64(BP),X15\n%s X15,X0\n%s X0,128(BP)", name, name, name) + function := fmt.Sprintf("Value%d", count) + assembly.WriteString(strings.Replace(fixedLocalPackedAssembly(operation), "·Value", "·"+function, 1)) + fmt.Fprintf(&declarations, "func %s(a uint64) uint64\n", function) + count++ + } + return assembly.String(), declarations.String(), count +} + +func TestGoAMD64FixedLocalPackedCompleteFormsActualObjects(t *testing.T) { + assembly, declarations, count := fixedLocalPackedCompleteProgram() + if count != 195 { + t.Fatalf("independent width/register/mask/legacy matrix changed: %d", count) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + pkg, request, file, options := x86CMOVFixedLocalFixture(t, assembly, declarations, target.name, target.triple) + source := requireGoAMD64FrameCapture(t, pkg, request) + if len(source.functions) != count { + t.Fatal("incomplete actual original Go objects") + } + expanded, err := source.preprocess(request.AssemblyFile, []byte(assembly)) + if err != nil { + t.Fatal(err) + } + file, err = Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + // Consume this owner's actual complete CPP result. Its token + // formatting inserts a space before .Z; unexpanded Raw strings + // cannot impersonate that immutable source fingerprint. + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil || strings.Count(ir, "%fixed_local_native = alloca") != count { + t.Fatalf("complete original frame inventory lost: %v", err) + } + compileLLVMToObject(t, llc, target.triple, "complete-packed.ll", "complete-packed.o", ir) + compileLLVMToObject(t, llc, target.triple, "complete-packed-opt.ll", "complete-packed-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + }) + } +} + +func TestGoAMD64FixedLocalPackedEveryByteAndUnknownRole(t *testing.T) { + for _, bank := range []struct { + name string + width int + }{{"X", 16}, {"Y", 32}, {"Z", 64}} { + operation := fmt.Sprintf("VMOVDQA64 64(BP),%s0\nVMOVDQA64 %s0,128(BP)", bank.name, bank.name) + pkg, _, _, options := fixedLocalPackedFixture(t, operation) + sig := options.Sigs[pkg.Path+".Value"] + base := fixedLocalPackedAssembly(operation) + for missing := 0; missing < bank.width; missing++ { + t.Run(fmt.Sprintf("%s/byte%d", bank.name, missing), func(t *testing.T) { + var stores strings.Builder + for index := 0; index < 8; index++ { + if index != missing%8 { + fmt.Fprintf(&stores, "MOVB $1,%d(BP)\n", 64+missing/8*8+index) + } + } + assembly := strings.Replace(base, fmt.Sprintf("MOVQ AX,%d(BP)\n", 64+missing/8*8), stores.String(), 1) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing private byte obtained native read authority", err) + } + }) + } + for name, changed := range map[string]string{ + "past frame": strings.Replace(base, operation, strings.Replace(operation, "64(BP)", "256(BP)", 1), 1), + "misaligned": strings.Replace(base, operation, strings.Replace(operation, "64(BP)", "65(BP)", 1), 1), + "unknown base": strings.Replace(base, operation, strings.Replace(operation, "64(BP)", "64(DX)", 1), 1), + "external escape": strings.Replace(base, operation, "MOVQ BP,(AX)\n"+operation, 1), + "private mask": strings.Replace(base, operation, "KMOVQ BP,K1\n"+operation, 1), + "unaligned residue": strings.Replace(base, "ANDQ $-64,BP", "ANDQ $-8,BP", 1), + "unknown address size": strings.Replace(base, "ADDQ $64,BP", "ADDQ DX,BP", 1), + } { + t.Run(bank.name+"/"+name, func(t *testing.T) { + file, err := Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native frame boundary was weakened", err) + } + }) + } + } +} + +func TestGoAMD64FixedLocalPackedMod64IndependentAliasReplay(t *testing.T) { + operation := "VMOVDQA64 64(BP),Z0\nVMOVDQA64 Z0,128(BP)" + pkg, _, file, options := fixedLocalPackedFixture(t, operation) + sig := options.Sigs[pkg.Path+".Value"] + proved, err := proveX86FixedLocal(file.Funcs[0], sig) + if err != nil { + t.Fatal(err) + } + for _, ins := range proved.Instrs { + if ins.Op != "VMOVDQA64" { + continue + } + for operand, access := range ins.x86FixedLocalMemory { + for residue, offset := range access.offset { + base := int64(((residue*8 + 64) & -64) - residue*8) + if offset != base+ins.Args[operand].Mem.Off || (offset+int64(residue*8))%64 != 0 { + t.Fatalf("mod64 residue %d: %+v", residue, access) + } + } + } + } + // Only residues six and seven overlap this scalar store with source bytes. + // Checking the old four residues would miss this real alias difference. + changed := strings.Replace(fixedLocalPackedAssembly(operation), operation, "MOVQ $0,80(SP)\n"+operation, 1) + requireX86GoAssemblerResult(t, "amd64", changed, true) + file, err = Parse(ArchAMD64, changed) + if err != nil { + t.Fatal(err) + } + if _, err := proveX86FixedLocal(file.Funcs[0], sig); !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "byte identity") { + t.Fatal("residue-dependent byte aliases were relocated", err) + } +} diff --git a/amd64_fixed_local_packed_native_linux_amd64_test.go b/amd64_fixed_local_packed_native_linux_amd64_test.go new file mode 100644 index 000000000..f11df9772 --- /dev/null +++ b/amd64_fixed_local_packed_native_linux_amd64_test.go @@ -0,0 +1,8 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixGoAMD64FixedLocalPackedNativeWide(t *testing.T) { + execution := x86PackedDoubleDwordNativeTools(t) + fixedLocalPackedRuntime(t, execution, true, true) +} diff --git a/amd64_fixed_local_packed_runtime_test.go b/amd64_fixed_local_packed_runtime_test.go new file mode 100644 index 000000000..b7b8f4297 --- /dev/null +++ b/amd64_fixed_local_packed_runtime_test.go @@ -0,0 +1,260 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +type fixedLocalPackedRuntimeCase struct { + width, bits, word, mode int +} + +func fixedLocalPackedRuntimeProgram(all bool) (string, string, []fixedLocalPackedRuntimeCase) { + var assembly, declarations strings.Builder + var cases []fixedLocalPackedRuntimeCase + add := func(opcode, bank string, width, bits, mode int) { + for word := -1; word <= width/8; word++ { + index := len(cases) + cases = append(cases, fixedLocalPackedRuntimeCase{width, bits, word, mode}) + fmt.Fprintf(&declarations, "func Value%d(a, mask uint64) uint64\n", index) + fmt.Fprintf(&assembly, "TEXT ·Value%d(SB),4,$512-24\nMOVQ a+0(FP),AX\nMOVQ mask+8(FP),CX\n", index) + assembly.WriteString("MOVQ SP,BP\nADDQ $64,BP\nANDQ $-64,BP\n") + assembly.WriteString("MOVQ $0x5a5a5a5a5a5a5a5a,DX\nMOVQ DX,248(BP)\n") + fmt.Fprintf(&assembly, "MOVQ DX,%d(BP)\n", 256+width) + for offset := 0; offset < width; offset += 8 { + fmt.Fprintf(&assembly, "MOVQ DX,%d(BP)\n", 256+offset) + } + for offset := 0; offset < width; offset += 8 { + fmt.Fprintf(&assembly, "MOVQ $%d,DX\nMOVQ DX,%d(BP)\n", int64(0x102030405060708)+int64(offset), 128+offset) + } + if word >= 0 && word < width/8 { + fmt.Fprintf(&assembly, "MOVQ AX,%d(BP)\n", 128+word*8) + } + first, last := bank+"0", bank+"1" + fmt.Fprintf(&assembly, "%s 256(BP),%s\n%s 256(BP),%s\n", opcode, first, opcode, last) + mask, effective := "", opcode + if mode != 0 { + assembly.WriteString("KMOVQ CX,K7\n") + mask = "K7," + } + if mode == 2 { + effective += ".Z" + } + fmt.Fprintf(&assembly, "%s 128(BP),%s%s\n%s %s,%s%s\n", effective, mask, first, effective, first, mask, last) + if mode == 2 { + mask = "" // Observe inactive zeroed bytes, not retained memory. + } + fmt.Fprintf(&assembly, "%s %s,%s256(BP)\nMOVQ %d(BP),AX\nMOVQ AX,ret+16(FP)\nRET\n", opcode, last, mask, 256+word*8) + } + } + for _, opcode := range []string{"MOVO", "MOVOA", "MOVOU", "MOVAPS", "MOVAPD", "MOVUPS", "MOVUPD"} { + add(opcode, "X", 16, 8, 0) + } + if all { + for _, row := range fixedLocalPackedMoveOracle { + width, bank, bits := 32, "Y", 8 + if row.evex { + width, bank = 64, "Z" + } + switch row.name { + case "VMOVAPD", "VMOVUPD", "VMOVDQA64", "VMOVDQU64": + bits = 64 + case "VMOVAPS", "VMOVUPS", "VMOVDQA32", "VMOVDQU32": + bits = 32 + case "VMOVDQU16": + bits = 16 + } + add(row.name, bank, width, bits, 0) + if row.evex { + add(row.name, bank, width, bits, 1) + add(row.name, bank, width, bits, 2) + } + } + } + return assembly.String(), declarations.String(), cases +} + +func fixedLocalPackedRuntimeOracles(cases []fixedLocalPackedRuntimeCase, triple string) (string, string) { + var goRows, cRows, cDeclarations strings.Builder + prefix := "" + if strings.Contains(triple, "apple") { + prefix = "_" + } + for index, row := range cases { + fmt.Fprintf(&goRows, "{Value%d,%d,%d,%d,%d},\n", index, row.width, row.bits, row.word, row.mode) + fmt.Fprintf(&cRows, "{value%d,%d,%d,%d,%d},\n", index, row.width, row.bits, row.word, row.mode) + fmt.Fprintf(&cDeclarations, "extern uint64_t value%d(uint64_t,uint64_t) __asm__(\"%sexample.invalid/framefixture.Value%d\");\n", index, prefix, index) + } + goOracle := `package framefixture +import "testing" +func TestOriginal(t *testing.T) { + cases := []struct { + call func(uint64,uint64) uint64 + width, bits, word, mode int + }{ +` + goRows.String() + ` + } + values := []uint64{0,1,^uint64(0),0x8000000000000000,0x123456789abcdef0,0x7ff81234567890ab} + masks := []uint64{0,1,^uint64(0),0x5555555555555555,0xaaaaaaaaaaaaaaaa,0x8000000000000000} + for index, row := range cases { + for _, value := range values { + for _, mask := range masks { + want := uint64(0x5a5a5a5a5a5a5a5a) + if row.word >= 0 && row.word < row.width/8 { + want = 0 + for bit := 0; bit < 64; bit++ { + input := uint64(0x5a5a5a5a5a5a5a5a) + if row.mode == 0 || mask >> uint((row.word*64+bit)/row.bits)&1 != 0 { + input = value + } else if row.mode == 2 { + input = 0 + } + want |= input & (uint64(1) << uint(bit)) + } + } + if got := row.call(value,mask); got != want { + t.Fatalf("case=%d input=%x mask=%x got=%x want=%x",index,value,mask,got,want) + } + } + } + } +} +` + cOracle := `#include +#include +` + cDeclarations.String() + ` +struct entry { + uint64_t (*call)(uint64_t,uint64_t); + int width,bits,word,mode; +}; +static const struct entry cases[]={ +` + cRows.String() + ` +}; +int main(void) { + const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000),UINT64_C(0x123456789abcdef0),UINT64_C(0x7ff81234567890ab)}; + const uint64_t masks[]={0,1,UINT64_MAX,UINT64_C(0x5555555555555555),UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_C(0x8000000000000000)}; + for(unsigned n=0;n=0 && row.word>lane)&1) source=values[v]; + else if(row.mode==2) source=0; + want|=source&(UINT64_C(1)< 0 && len(sig.Frame.Results) <= 4096 + if fn.FrameSize <= 0 || fn.FrameSize > 64<<10 || fn.FrameSize%8 != 0 || + len(fn.Instrs) == 0 || len(fn.Instrs) > 8192 || fn.FrameSize > (1<<20)/int64(len(fn.Instrs)) || + len(fn.X86RawText) != 0 || len(sig.ArgRegs) != 0 || sig.ARMEntry != nil || + sig.ARM64GoRegisterABI != nil || sig.x86CallerRegions != nil || !voidReturn && !valueReturn { + return fn, goAMD64FrameContext("fixed local source has an unbound frame or entry contract") + } + copyFn := fn + copyFn.Instrs = append([]Instr(nil), fn.Instrs...) + proof := &x86FixedLocalProof{binding: sig.x86FixedLocal, frame: fn.FrameSize, accesses: make(map[int]map[int]x86FixedLocalAccess)} + for index := range copyFn.Instrs { + copyFn.Instrs[index].Args = append([]Operand(nil), fn.Instrs[index].Args...) + copyFn.Instrs[index].x86FixedLocalMemory = nil + ins := copyFn.Instrs[index] + if ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86AddressBits != 0 || ins.x86SegmentPrefix != 0 || x86SourceGOperand(ins) { + return fn, goAMD64FrameContext("fixed local source contains raw/native/g effects") + } + if !fixedLocalInstructionShape(ins) { + return fn, goAMD64FrameContext("fixed local complete source has an unregistered instruction shape: %s", ins.Op) + } + } + for residue := 0; residue < 8; residue++ { + if err := replayX86FixedLocal(©Fn, sig, proof, residue); err != nil { + return fn, err + } + } + // Independent final replay must recreate exactly the same access set and + // offset in every possible word-aligned original Go SP congruence class. + final := copyFn + final.Instrs = append([]Instr(nil), copyFn.Instrs...) + for index := range final.Instrs { + final.Instrs[index].x86FixedLocalMemory = nil + } + for residue := 0; residue < 8; residue++ { + if err := replayX86FixedLocal(&final, sig, proof, residue); err != nil { + return fn, err + } + } + for index := range final.Instrs { + if !reflect.DeepEqual(final.Instrs[index].x86FixedLocalMemory, copyFn.Instrs[index].x86FixedLocalMemory) { + return fn, goAMD64FrameContext("fixed local final access replay changed") + } + if len(final.Instrs[index].x86FixedLocalMemory) != 0 { + proof.accesses[index] = make(map[int]x86FixedLocalAccess) + for operand, access := range final.Instrs[index].x86FixedLocalMemory { + proof.accesses[index][operand] = access + } + } + } + if err := fixedLocalRelocationAliases(proof.accesses); err != nil { + return fn, err + } + var err error + proof.function, err = x86CallerSourceFingerprint(final) + if err != nil { + return fn, goAMD64FrameContext("fixed local normalized function cannot be bound: %v", err) + } + final.x86FixedLocal = proof + return final, nil +} + +// Every reachable access, including initialization stores and private address +// spills, contributes all its bytes. Each original residue must induce a +// bijection on those bytes: equal bytes remain equal and distinct bytes stay +// distinct. Different distances between permanently disjoint accesses are +// unobservable because the source cannot expose numeric addresses or flags. +func fixedLocalRelocationAliases(accesses map[int]map[int]x86FixedLocalAccess) error { + var forward, reverse [8]map[int64]int64 + for residue := range forward { + forward[residue] = make(map[int64]int64) + reverse[residue] = make(map[int64]int64) + } + var total int64 + for _, operands := range accesses { + for _, access := range operands { + if access.width <= 0 || access.width > (1<<20)-total { + return goAMD64FrameContext("fixed local complete access-byte budget exceeded") + } + total += access.width + for byteIndex := int64(0); byteIndex < access.width; byteIndex++ { + original := access.offset[0] + byteIndex + for residue := range forward { + current := access.offset[residue] + byteIndex + if mapped, found := forward[residue][original]; found && mapped != current { + return goAMD64FrameContext("fixed local overlapping-byte identity depends on original SP residue") + } + if mapped, found := reverse[residue][current]; found && mapped != original { + return goAMD64FrameContext("fixed local distinct-byte identity depends on original SP residue") + } + forward[residue][original] = current + reverse[residue][current] = original + } + } + } + } + return nil +} + +func fixedLocalInstructionShape(ins Instr) bool { + if form, known, err := fixedLocalConversionProperties(ins); known { + return err == nil && !form.mmx + } + if _, known, err := x86ConditionalSetProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if ins.x86NumericalRead != nil { + _, valid := x86NumericalReadEffect(ins, "amd64") + return valid + } + if _, known, err := amd64PackedVectorMoveProperties(ins, "amd64"); known { + return err == nil + } + if _, known, err := x86LegacyPackedMoveProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known := amd64MaskMoveSpecs[ins.Op]; known { + return amd64MaskMoveOperandsValid(ins, "amd64") + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && amd64VectorByteWidth(operand.Reg) != 0 { + index, valid := amd64VectorRegisterIndex(operand.Reg, 16) + if !valid || index > 15 { + return false + } + } + } + if ins.Op == OpTEXT || ins.Op == OpLABEL { + return true // Parser/complete CFG separately binds TEXT and label operands. + } + if ins.Op == OpRET { + return len(ins.Args) == 0 + } + if isAMD64ConditionalBranch(ins.Op) || ins.Op == OpJMP { + _, named := fixedLocalBranchLabel(ins) + return named + } + if x86RawReturnScalarMoveBytes(ins.Op) != 0 { + return len(ins.Args) == 2 + } + if ins.Op == "NOTQ" { + return len(ins.Args) == 1 && ins.Args[0].Kind == OpReg + } + if ins.Op == "ANDQ" || ins.Op == "ADDQ" || ins.Op == "CMPQ" { + return len(ins.Args) == 2 + } + if _, known, err := x86TypedConditionalMoveByteEffect(ins.Op, ins, "amd64"); known { + return err == nil + } + if form, known, err := x86AESProperties(ins.Op, ins, "amd64"); known { + return err == nil && form.byteWidth == 16 + } + if form, known, err := x86PackedWrappingAddProperties(ins, "amd64"); known { + return err == nil && form.legacy && !form.mmx && form.byteWidth == 16 + } + if form, known, err := x86WordMultiplyAddProperties(ins, "amd64"); known { + return err == nil && form.legacy && !form.mmx && form.byteWidth == 16 + } + if form, known, err := x86FloatCompareProperties(ins, "amd64"); known { + return err == nil && form.legacy && !form.scalar && form.byteWidth == 16 + } + return false +} + +func fixedLocalBranchLabel(ins Instr) (string, bool) { + if len(ins.Args) != 1 { + return "", false + } + operand := ins.Args[0] + if operand.Kind == OpLabel && operand.Sym != "" { + return operand.Sym, true + } + if operand.Kind == OpIdent && operand.Ident != "" { + return operand.Ident, true + } + return "", false +} + +func fixedLocalEdges(fn Func) (map[int][]int, []int, error) { + labels := make(map[string]int) + var instructions []int + for index, ins := range fn.Instrs { + if ins.Op == OpLABEL { + if len(ins.Args) != 1 || ins.Args[0].Kind != OpLabel { + return nil, nil, goAMD64FrameContext("malformed local label") + } + name := ins.Args[0].Sym + if _, exists := labels[name]; exists { + return nil, nil, goAMD64FrameContext("duplicate local label") + } + labels[name] = len(instructions) + } else { + instructions = append(instructions, index) + } + } + if !x86CallerReachableAcyclic(fn, instructions, labels) { + return nil, nil, goAMD64FrameContext("fixed local CFG is not a closed reachable DAG") + } + edges := make(map[int][]int) + for pc, index := range instructions { + ins := fn.Instrs[index] + if ins.Op == OpRET { + continue + } + next := []int{pc + 1} + if ins.Op == OpJMP || isAMD64ConditionalBranch(ins.Op) { + label, named := fixedLocalBranchLabel(ins) + if !named { + return nil, nil, goAMD64FrameContext("fixed local branch needs a complete named target") + } + target, found := labels[label] + if !found { + return nil, nil, goAMD64FrameContext("fixed local branch target is missing") + } + if ins.Op == OpJMP { + next = []int{target} + } else { + next = append(next, target) + } + } + edges[pc] = next + } + return edges, instructions, nil +} + +func replayX86FixedLocal(fn *Func, sig FuncSig, proof *x86FixedLocalProof, residue int) error { + edges, instructions, err := fixedLocalEdges(*fn) + if err != nil { + return err + } + state := x86FixedLocalState{ + pointers: map[Reg]int64{SP: 0}, constants: make(map[Reg]int64), known: make(map[Reg]uint8), + opaque: make(map[Reg]int64), opaqueCells: make(map[int64]int64), booleans: make(map[Reg]uint8), + booleanBytes: make([]bool, proof.frame), + vectors: make(map[Reg]uint64), masks: make(map[Reg]bool), + bytes: make([]bool, proof.frame), addresses: make(map[int64]int64), results: make(map[int]bool), + } + states := map[int]x86FixedLocalState{0: state} + // Wait for every reachable predecessor before transferring a block. This + // topological DAG replay creates no provisional initialization certificate, + // visits each instruction once, and keeps total byte-state storage bounded. + reachable := map[int]bool{} + find := []int{0} + for len(find) != 0 { + pc := find[0] + find = find[1:] + if reachable[pc] { + continue + } + reachable[pc] = true + find = append(find, edges[pc]...) + } + pending := make(map[int]int) + for pc := range reachable { + for _, target := range edges[pc] { + pending[target]++ + } + } + work := []int{0} + returned := false + for visits := 0; len(work) != 0; visits++ { + if visits >= len(instructions) { + return goAMD64FrameContext("fixed local CFG work bound exceeded") + } + pc := work[0] + work = work[1:] + if pc < 0 || pc >= len(instructions) { + return goAMD64FrameContext("fixed local fallthrough lacks a source return") + } + current := cloneFixedLocalState(states[pc]) + ins := &fn.Instrs[instructions[pc]] + proof.ordinal = instructions[pc] + if err := fixedLocalStep(ins, ¤t, sig, proof, residue); err != nil { + return fmt.Errorf("%s: %q: %w", fn.Sym, ins.Raw, err) + } + if ins.Op == OpRET { + returned = true + continue + } + for _, target := range edges[pc] { + if previous, found := states[target]; found { + // Acyclic replay permits only all-path initialization to decrease. + // Address identity and saved private pointer cells remain exact. + if !reflect.DeepEqual(previous.pointers, current.pointers) || !reflect.DeepEqual(previous.addresses, current.addresses) || + !reflect.DeepEqual(previous.opaque, current.opaque) || !reflect.DeepEqual(previous.opaqueCells, current.opaqueCells) || + !reflect.DeepEqual(previous.constants, current.constants) || previous.flags != current.flags { + return goAMD64FrameContext("fixed local CFG address/constant/flags identity disagrees") + } + merged := cloneFixedLocalState(previous) + for reg, bits := range merged.known { + merged.known[reg] = bits & current.known[reg] + } + for reg, bits := range merged.booleans { + merged.booleans[reg] = bits & current.booleans[reg] + } + for reg, bits := range merged.vectors { + merged.vectors[reg] = bits & current.vectors[reg] + } + for reg, known := range merged.masks { + merged.masks[reg] = known && current.masks[reg] + } + for index := range merged.bytes { + merged.bytes[index] = merged.bytes[index] && current.bytes[index] + merged.booleanBytes[index] = merged.booleanBytes[index] && current.booleanBytes[index] + } + for index, written := range merged.results { + merged.results[index] = written && current.results[index] + } + states[target] = merged + } else { + states[target] = cloneFixedLocalState(current) + } + pending[target]-- + if pending[target] == 0 { + work = append(work, target) + } + } + } + if !returned { + return goAMD64FrameContext("fixed local source has no reachable return") + } + return nil +} + +func fixedLocalMemory(ins *Instr, index int, state *x86FixedLocalState, proof *x86FixedLocalProof, residue int, width, alignment int64, write, pointer bool) (int64, error) { + if index < 0 || index >= len(ins.Args) || ins.Args[index].Kind != OpMem || width <= 0 { + return 0, goAMD64FrameContext("fixed local access lacks native memory") + } + memory, err := fixedLocalMemoryAddress(*ins, index, proof.frame) + if err != nil { + return 0, err + } + base, found := state.pointers[memory.Base] + if !found || memory.Sym != "" || memory.Segment != "" || memory.Index != "" || + memory.Off > 0 && base > math.MaxInt64-memory.Off || memory.Off < 0 && base < math.MinInt64-memory.Off { + return 0, goAMD64FrameContext("fixed local access has unknown/escaping address provenance") + } + offset := base + memory.Off + if offset < 0 || offset > proof.frame || width > proof.frame-offset || alignment <= 0 || + (int64(residue*8)+offset)%alignment != 0 { + return 0, goAMD64FrameContext("fixed local access crosses storage or has residue-dependent native alignment") + } + if !write && !pointer { + for byteIndex := offset; byteIndex < offset+width; byteIndex++ { + if !state.bytes[byteIndex] { + return 0, goAMD64FrameContext("fixed local native read contains uninitialized or address bytes") + } + } + } + if ins.x86FixedLocalMemory == nil { + ins.x86FixedLocalMemory = make(map[int]x86FixedLocalAccess) + } + shape, err := fixedLocalInstructionDigest(*ins) + if err != nil { + return 0, err + } + access, found := ins.x86FixedLocalMemory[index] + if !found { + access = x86FixedLocalAccess{owner: proof, shape: shape, width: width, write: write, ordinal: proof.ordinal} + } + if access.owner != proof || access.shape != shape || access.width != width || access.write != write { + return 0, goAMD64FrameContext("fixed local access shape changed") + } + access.offset[residue] = offset + ins.x86FixedLocalMemory[index] = access + return offset, nil +} + +func fixedLocalStep(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int) error { + fail := func() error { + return goAMD64FrameContext("unproved fixed local width/initialization/address/control effect: %s", ins.Op) + } + if ins.Op == OpTEXT { + return nil + } + if ins.Op == OpRET { + sp, found := state.pointers[SP] + if len(ins.Args) != 0 || !found || sp != 0 { + return fail() + } + for index := range sig.Frame.Results { + if !state.results[index] { + return fail() + } + } + return nil + } + if isAMD64ConditionalBranch(ins.Op) { + if !state.flags || isAMD64CounterZeroBranch(ins.Op) { + return fail() + } + return nil + } + if ins.Op == OpJMP { + return nil + } + if known, err := fixedLocalNumericalReadStep(*ins, state); known { + return err + } + if width := x86RawReturnScalarMoveBytes(ins.Op); width != 0 { + return fixedLocalMove(ins, state, sig, proof, residue, width) + } + if known, err := fixedLocalConditionalSetStep(ins, state, sig, proof, residue); known { + return err + } + if ins.Op == "NOTQ" && len(ins.Args) == 1 && ins.Args[0].Kind == OpReg { + reg := ins.Args[0].Reg + if _, pointer := state.pointers[reg]; pointer { + return fail() + } + value, known := state.constants[reg] + if !known { + return fail() + } + state.constants[reg] = ^value + fixedLocalWriteBooleans(state, reg, 8, fixedLocalConstantBooleans(^value, 8)) + return nil + } + if (ins.Op == "ADDQ" || ins.Op == "ANDQ") && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg { + reg := ins.Args[1].Reg + value := ins.Args[0].Imm + known := ins.Args[0].Kind == OpImm && ins.Args[0].ImmRaw == "" && !ins.Args[0].ImmIsFloat + if ins.Args[0].Kind == OpReg { + value, known = state.constants[ins.Args[0].Reg] + } + if !known { + return fail() + } + if offset, pointer := state.pointers[reg]; pointer { + if ins.Op == "ANDQ" { + // Only an exact power-of-two residue projection preserves the + // same private address origin. No pointer bit becomes numeric. + if value != -8 && value != -16 && value != -32 && value != -64 { + return fail() + } + offset = (offset+int64(residue*8))&value - int64(residue*8) + } else { + if value > 0 && offset > math.MaxInt64-value || value < 0 && offset < math.MinInt64-value { + return fail() + } + offset += value + } + if offset < -64 || offset > proof.frame { + return fail() + } + state.pointers[reg] = offset + state.flags = false // Relocated pointer arithmetic flags are opaque. + return nil + } + return fail() // Ordinary numeric arithmetic needs its complete family. + } + if ins.Op == "CMPQ" && len(ins.Args) == 2 { + for _, operand := range ins.Args { + if !fixedLocalNumericRead(operand, 8, state) { + return fail() + } + } + state.flags = true + return nil + } + if known, err := fixedLocalConditionalMoveStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalAESByteStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalPackedMoveStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalMaskMoveStep(ins, state, proof, residue); known { + return err + } + if known, err := fixedLocalConversionStep(ins, state, proof, residue); known { + return err + } + return fixedLocalVectorStep(ins, state, proof, residue) +} + +func fixedLocalNumericRead(operand Operand, width int64, state *x86FixedLocalState) bool { + if operand.Kind == OpImm { + return operand.ImmRaw == "" && !operand.ImmIsFloat + } + if operand.Kind != OpReg { + return false + } + root, valid := x86CallerGP(operand.Reg) + _, pointer := state.pointers[root] + _, opaque := state.opaque[root] + mask := x86CallerByteMask(operand.Reg, width) + return valid && root != SP && root != BP && !pointer && !opaque && mask != 0 && state.known[root]&mask == mask +} + +func fixedLocalMove(ins *Instr, state *x86FixedLocalState, sig FuncSig, proof *x86FixedLocalProof, residue int, width int64) error { + fail := func() error { return goAMD64FrameContext("unproved fixed local scalar move") } + if len(ins.Args) != 2 { + return fail() + } + source, destination := ins.Args[0], ins.Args[1] + var address int64 + var pointer bool + var opaque bool + var identity int64 + var booleans uint8 + var constant int64 + var constantKnown bool + switch source.Kind { + case OpReg: + address, pointer = state.pointers[source.Reg] + identity, opaque = state.opaque[source.Reg] + if pointer || opaque { + if width != 8 { + return fail() + } + } else if amd64VectorByteWidth(source.Reg) == 16 { + if width != 8 || state.vectors[x86CallerVectorRoot(source.Reg)]&255 != 255 { + return fail() + } + } else { + if !fixedLocalNumericRead(source, width, state) { + return fail() + } + constant, constantKnown = state.constants[source.Reg] + // Narrow moves truncate or preserve a destination's old upper + // bytes. They cannot transport a complete 64-bit address mask. + constantKnown = constantKnown && width == 8 + booleans = fixedLocalBooleanRead(source, width, state) + } + case OpImm: + if !fixedLocalNumericRead(source, width, state) { + return fail() + } + constant, constantKnown = source.Imm, width == 8 + booleans = fixedLocalBooleanRead(source, width, state) + case OpFP: + sp, original := state.pointers[SP] + slot, _, found := fixedLocalTransportSlot(source.FPOffset, width, sig.Frame.Params) + if !original || sp != 0 || !found { + return fail() + } + if slot.Type == Ptr { + opaque, identity = true, slot.Offset + } else if slot.Type == I1 { + booleans = 1 + } + case OpSym: + if _, _, err := fixedLocalReadMemory(ins, 0, state, proof, residue, width, 1); err != nil { + return err + } + case OpMem: + memory, err := fixedLocalMemoryAddress(*ins, 0, proof.frame) + if err != nil { + return err + } + base, found := state.pointers[memory.Base] + if found && memory.Off >= -proof.frame && memory.Off <= proof.frame { + address, pointer = state.addresses[base+memory.Off] + identity, opaque = state.opaqueCells[base+memory.Off] + } + if (pointer || opaque) && width != 8 { + return fail() + } + offset, err := fixedLocalMemory(ins, 0, state, proof, residue, width, 1, false, pointer || opaque) + if err != nil { + return err + } + if !pointer && !opaque { + booleans = fixedLocalMemoryBooleans(state, offset, width) + } + default: + return fail() + } + switch destination.Kind { + case OpReg: + root, valid := x86CallerGP(destination.Reg) + if !valid || root == BP && !pointer || x86SourceGRegister(root) || (pointer || opaque) && (width != 8 || destination.Reg != root) { + return fail() + } + if root == SP && !pointer { + return fail() + } + if _, oldPointer := state.pointers[root]; oldPointer && width != 8 { + return fail() + } + delete(state.pointers, root) + delete(state.opaque, root) + delete(state.constants, root) + fixedLocalWriteBooleans(state, destination.Reg, width, booleans) + if pointer { + state.pointers[root] = address + state.known[root] = 0 + } else if opaque { + state.opaque[root] = identity + state.known[root] = 0 + } else { + mask := x86CallerByteMask(destination.Reg, width) + if mask == 0 { + return fail() + } + if width == 4 { + mask = 255 + } + state.known[root] |= mask + if constantKnown { + state.constants[root] = constant + } + } + case OpMem: + if source.Kind == OpMem { + return fail() + } + offset, err := fixedLocalMemory(ins, 1, state, proof, residue, width, 1, true, false) + if err != nil { + return err + } + fixedLocalRevokeSpillRoles(state, offset, width) + for index := offset; index < offset+width; index++ { + state.bytes[index] = !pointer && !opaque + state.booleanBytes[index] = booleans&(1<= 0 { + if _, _, err := fixedLocalReadMemory(ins, memory, state, proof, residue, 16, 16); err != nil { + return err + } + } else if state.vectors[x86CallerVectorRoot(ins.Args[first].Reg)]&width != width { + return goAMD64FrameContext("fixed local vector operation reads undefined source bytes") + } + state.vectors[x86CallerVectorRoot(ins.Args[destination].Reg)] |= width + return nil +} + +func (c *amd64Ctx) fixedLocalMemoryCertified(ins Instr, index int) bool { + access, found := ins.x86FixedLocalMemory[index] + shape, err := fixedLocalInstructionDigest(ins) + if !found || c.fixedLocal == nil || access.owner != c.fixedLocal || ins.x86Encoded || + ins.x86SourceRaw != nil || ins.x86AddressBits != 0 || ins.x86SegmentPrefix != 0 || + err != nil || access.shape != shape || access.width <= 0 { + return false + } + expected, bound := c.fixedLocal.accesses[access.ordinal][index] + return bound && reflect.DeepEqual(access, expected) +} diff --git a/amd64_fixed_local_runtime_test.go b/amd64_fixed_local_runtime_test.go new file mode 100644 index 000000000..f02cf83d7 --- /dev/null +++ b/amd64_fixed_local_runtime_test.go @@ -0,0 +1,267 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func fixedLocalCompleteFixture(t *testing.T, target string) (GoPackage, GoAMD64FrameCaptureRequest, *File, Options, string) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + request.Target = target + var declarations, assembly strings.Builder + declarations.WriteString("package framefixture\n") + for index, operation := range fixedLocalNativeOperations { + fmt.Fprintf(&declarations, "func Value%d(a, b, c, d uint64) uint64\n", index) + source, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + assembly.WriteString(strings.Replace(source, "·Value", fmt.Sprintf("·Value%d", index), 1)) + } + for name, content := range map[string]string{"value.go": declarations.String(), "value_amd64.s": assembly.String()} { + if err := os.WriteFile(filepath.Join(request.Directory, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", declarations.String(), 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check("example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, name := range typed.Scope().Names() { + sig, err := DeriveGoFuncABI0Signature(typed.Path()+"."+name, typed.Scope().Lookup(name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + sigs[sig.Name] = sig + } + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", typed.Path()+".", 1) } + triple := "x86_64-unknown-linux-gnu" + if strings.HasPrefix(target, "darwin/") { + triple = "x86_64-apple-darwin" + } + if strings.HasPrefix(target, "windows/") { + triple = "x86_64-pc-windows-msvc" + } + return GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}}, request, file, + Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs}, declarations.String() +} + +// Integer-only independent expected values; the comparison inputs remain raw +// IEEE bits, including signed zero, infinities, NaNs and subnormals. +const fixedLocalGoMath = ` +func equalBits(a,b uint64,bits uint, daz bool) bool { + exp,frac,sign:=uint64(0x7ff0000000000000),uint64(0xfffffffffffff),uint64(1)<<63 + if bits==32 { exp,frac,sign=0x7f800000,0x7fffff,1<<31; a=uint64(uint32(a)); b=uint64(uint32(b)) } + if a&exp==exp && a&frac!=0 || b&exp==exp && b&frac!=0 { return false } + if daz && a&exp==0 { a &= sign }; if daz && b&exp==0 { b &= sign } + return a==b || a&^sign==0 && b&^sign==0 +} +func expected(op int,a,c uint64) uint64 { + if op<4 { + width:=uint(8)<>shift)+(c>>shift))&mask<>shift)),int64(int16(a>>(shift+16))) + u,v:=int64(int16(c>>shift)),int64(int16(c>>(shift+16))) + result|=uint64(uint32(x*u+y*v))<>shift,c>>shift,32,false) { result|=uint64(0xffffffff)< +#include +#include +#include +` + declarations + ` +static __attribute__((noinline)) uint64_t readword(const volatile uint64_t *p) { return *p; } +static __attribute__((noinline)) uint64_t reference(int op,const uint64_t *data) { + _Alignas(16) uint64_t input[2]={data[0],data[1]}; + __m128i value=_mm_set_epi64x((long long)data[3],(long long)data[2]); + switch(op) { +` + opcodes.String() + ` + } + return (uint64_t)_mm_cvtsi128_si64(value); +} +int main(void) { + const uint64_t values[]={0,1,UINT64_MAX,UINT64_C(0x8000800080008000),UINT64_C(0x7fff7fff7fff7fff),UINT64_C(0x3f8000003f800000),UINT64_C(0x7fc012347f801234),UINT64_C(0x3ff0000000000000),UINT64_C(0x8000000000000000),UINT64_C(0x123456789abcdef0)}; + unsigned saved=_mm_getcsr(); + /* Controlled first-party storage realizes all four word-aligned source + residues. This is an independent exact-SSE witness of the refused alias + fragment, not a claim that these are the actual Go runtime entry SPs. */ + const uint64_t aliasWant[4][2]={{6,10},{7,8},{7,10},{7,10}}; + for(int residue=0;residue<4;residue++) { + _Alignas(32) unsigned char memory[256]={0}; + unsigned char *sp=memory+32+residue*8; + unsigned char *aligned=(unsigned char*)(((uintptr_t)sp&~(uintptr_t)31)+32); + uint64_t input[2]={2,3}, other[2]={5,7}, one=1, observed[2]; + memcpy(aligned+16,input,16); memcpy(aligned+48,other,16); + memcpy(sp+48,&one,8); + __m128i vector; memcpy(&vector,aligned+48,16); + __asm__ volatile("paddq %1,%0" : "+x"(vector) : "m"(*(const unsigned char (*)[16])(aligned+16)) : "memory"); + memcpy(observed,&vector,16); + if(observed[0]!=aliasWant[residue][0] || observed[1]!=aliasWant[residue][1]) return 3; + } + for(int seed=0;seed<128;seed++) { + volatile uint64_t guarded[12]={11,17,29,53,values[seed%10],values[(seed+1)%10],values[(seed*3)%10],values[(seed*7+1)%10],0xaabbccdd,0x12345678,0x87654321,UINT64_MAX}; + uint64_t data[4],before[12]; + for(int i=0;i<12;i++) before[i]=readword(&guarded[i]); + for(int i=0;i<4;i++) data[i]=readword(&guarded[i+4]); + for(int op=0;op<7;op++) for(int rc=0;rc<4;rc++) for(int ctl=0;ctl<4;ctl++) for(int sticky=0;sticky<3;sticky++) { + unsigned start=0x1f80u|(unsigned)rc<<13|(ctl&1?0x40u:0)|(ctl&2?0x8000u:0)|(sticky==1?1u:sticky==2?0x3fu:0); + _mm_setcsr(start); uint64_t want=reference(op,data); unsigned wantCSR=_mm_getcsr(); + _mm_setcsr(start); uint64_t got=0; + switch(op) { +` + cases.String() + ` + } + unsigned gotCSR=_mm_getcsr(); + if(got!=want || gotCSR!=wantCSR) { fprintf(stderr,"op=%d seed=%d RC=%d ctl=%d sticky=%d got=%llx/%x want=%llx/%x\n",op,seed,rc,ctl,sticky,(unsigned long long)got,gotCSR,(unsigned long long)want,wantCSR); return 1; } + for(int i=0;i<12;i++) if(readword(&guarded[i])!=before[i]) return 2; + } + } + _mm_setcsr(saved); return 0; +} +` +} diff --git a/amd64_fixed_local_target_test.go b/amd64_fixed_local_target_test.go new file mode 100644 index 000000000..afb2a4046 --- /dev/null +++ b/amd64_fixed_local_target_test.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "errors" + "testing" +) + +func TestGoAMD64FixedLocalDarwinTripleClosure(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + request.Target = "darwin/amd64" + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-apple-macosx"} { + t.Run(triple, func(t *testing.T) { + options.TargetTriple = triple + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatalf("actual Darwin frame rejected its exact LLVM target spelling: %v", err) + } + if bound.TargetTriple != triple { + t.Fatal("binding rewrote the consumer's actual triple") + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "triple.ll", "triple.o", ir) + compileLLVMToObject(t, llc, triple, "triple-opt.ll", "triple-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + // Equivalent initial target spellings cannot mutate an already + // bound consumer. Every certificate still pins its exact options. + changed := bound + changed.TargetTriple = "x86_64-apple-macosx" + if changed.TargetTriple == triple { + changed.TargetTriple = "x86_64-apple-darwin" + } + if _, err := Translate(file, changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("retargeted fixed-local certificate survived: %v", err) + } + }) + } + for _, triple := range []string{ + "", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + "arm64-apple-macosx", "i386-apple-macosx", "aarch64-apple-darwin", + "x86_64-unknown-macosx", "x86_64-apple-ios", "x86_64-apple-macosx-junk", + "x86_64-apple-darwin-junk", "x86_64-apple-macosx11.0.0", "x86_64-apple-darwin20.0.0", + "prefix-x86_64-apple-macosx", "x86_64-apple-macosx ", + } { + t.Run("reject/"+triple, func(t *testing.T) { + options.TargetTriple = triple + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("foreign or unregistered triple acquired a Darwin native frame: %q: %v", triple, err) + } + }) + } +} diff --git a/amd64_fixed_local_transport.go b/amd64_fixed_local_transport.go new file mode 100644 index 000000000..8e5befc6c --- /dev/null +++ b/amd64_fixed_local_transport.go @@ -0,0 +1,133 @@ +package plan9asm + +// Opaque inputs identify ABI0 pointer components, not pointee allocations. +// They may be copied through full-width GP/local cells and pointer results. +// They never enter the private-SP address map or numerical initialized bytes. +// Boolean masks independently record bytes whose values are exactly zero/one. +// This is needed because an ABI0 bool occupies one byte, but LLVM uses i1. +func fixedLocalTransportRole(role goAMD64FrameTransport) bool { + return role == goAMD64FrameNumericTransport || role == goAMD64FrameCompositeTransport +} + +func fixedLocalTransportSlot(offset, width int64, slots []FrameSlot) (FrameSlot, int, bool) { + for index, slot := range slots { + switch slot.Type { + case I1, I8, I16, I32, I64, Ptr: + default: + continue + } + if slot.Offset == offset && frameTypeSize(slot.Type, 8) == width { + return slot, index, true + } + } + return FrameSlot{}, 0, false +} + +func fixedLocalBooleanShift(reg Reg) uint { + if _, bits, alias := amd64ByteAlias(reg); alias { + return bits / 8 + } + return 0 +} + +func fixedLocalConstantBooleans(value int64, width int64) uint8 { + var mask uint8 + for index := int64(0); index < width; index++ { + if byte(uint64(value)>>uint(index*8)) <= 1 { + mask |= 1 << uint(index) + } + } + return mask +} + +func fixedLocalBooleanRead(operand Operand, width int64, state *x86FixedLocalState) uint8 { + if operand.Kind == OpImm { + return fixedLocalConstantBooleans(operand.Imm, width) + } + if operand.Kind == OpReg { + root, _ := x86CallerGP(operand.Reg) + return state.booleans[root] >> fixedLocalBooleanShift(operand.Reg) & uint8((1<0 && &s[0]!=&a[0] { t.Fatal("slice identity") } + if a==nil && s!=nil { t.Fatal("nil identity") } + if buffer!=[4]byte{0x11,0x22,0x33,0x44}||word!=0x123456789abcdef0 { t.Fatal("pointee mutation") } + } + } + } + } +} +` + goConditions + fixedLocalPackedRunOriginal(t, execution, assembly+originalObservers, declarations+observerDeclarations, goOracle, true) + // Call the translated aggregate ABI from an exact LLVM wrapper, not a + // guessed C struct ABI. C only crosses a scalar wrapper signature. + const wrapper = ` +define i32 @transport_probe(ptr %data, i64 %len, i64 %cap, ptr %p, i1 %b, i64 %n) { +entry: + %a0 = insertvalue {ptr,i64,i64} zeroinitializer, ptr %data, 0 + %a1 = insertvalue {ptr,i64,i64} %a0, i64 %len, 1 + %a2 = insertvalue {ptr,i64,i64} %a1, i64 %cap, 2 + %b2 = call i1 @"example.invalid/framefixture.Bool"({ptr,i64,i64} %a2, ptr %p, i1 %b, i64 %n) + %p2 = call ptr @"example.invalid/framefixture.Pointer"({ptr,i64,i64} %a2, ptr %p, i1 %b, i64 %n) + %s2 = call {ptr,i64,i64} @"example.invalid/framefixture.Slice"({ptr,i64,i64} %a2, ptr %p, i1 %b, i64 %n) + %d2 = extractvalue {ptr,i64,i64} %s2, 0 + %l2 = extractvalue {ptr,i64,i64} %s2, 1 + %c2 = extractvalue {ptr,i64,i64} %s2, 2 + %badb = icmp ne i1 %b2, %b + %badp = icmp ne ptr %p2, %p + %badd = icmp ne ptr %d2, %data + %badl = icmp ne i64 %l2, %len + %badc = icmp ne i64 %c2, %cap + %bad0 = or i1 %badb, %badp + %bad1 = or i1 %bad0, %badd + %bad2 = or i1 %bad1, %badl + %bad3 = or i1 %bad2, %badc + %result = zext i1 %bad3 to i32 + ret i32 %result +} +` + oracle := `#include +#include +#include +` + cConditions + ` +extern int transport_probe(void *,uint64_t,uint64_t,void *,bool,uint64_t); +int main(void) { + if(check_conditions()) return 3; + volatile uint8_t buffer[4]={0x11,0x22,0x33,0x44}; + volatile uint64_t word=UINT64_C(0x123456789abcdef0); + uint64_t inputs[]={0,1,UINT64_MAX,UINT64_C(0x8000000000000000)}; + for(unsigned a=0;a<3;a++) for(unsigned p=0;p<2;p++) + for(unsigned b=0;b<2;b++) for(unsigned n=0;n<4;n++) { + void *data=a?(void *)buffer:0; + uint64_t len=a==1?2:a==2?4:0,cap=a?4:0; + if(transport_probe(data,len,cap,p?(void *)&word:0,b,inputs[n])) return 1; + if(buffer[0]!=0x11||buffer[1]!=0x22||buffer[2]!=0x33||buffer[3]!=0x44||word!=UINT64_C(0x123456789abcdef0)) return 2; + } + return 0; +} +` + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + input, level := ir+wrapper+llvmConditions, "-O0" + if optimized { + input, level = x86MulDivRoundOptimize(t, input), "-O2" + } + execution.compileAndRunLLVM(t, fmt.Sprintf("transport-%v", optimized), input, oracle, level) + } + }) + t.Log("48 inputs x bool/pointer/slice plus all16 SETcc true/false and original physical bool-byte canaries; LLVM three APIs O0/O2; pointees unchanged") + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + t.Log("Rosetta SSE diagnostic, not native AVX/AVX512 execution") + } +} + +func fixedLocalTransportConditionOracles() (string, string, string, string, string, string, string) { + var assembly, declarations, observers, observerDeclarations strings.Builder + var goRows, llvmRows, cDeclarations, cRows strings.Builder + names := strings.Fields("SETCC SETCS SETEQ SETGE SETGT SETHI SETLE SETLS SETLT SETMI SETNE SETOC SETOS SETPC SETPL SETPS") + var bodies []string + for _, name := range names { + bodies = append(bodies, "MOVQ a_len+8(FP),DX\nCMPQ AX,DX\n"+name+" BX\nMOVB BX,ret+48(FP)") + } + bodies = append(bodies, fixedLocalCanonicalBoolBodies...) + bodies = append(bodies, "MOVQ $-1,BX\nMOVB $1,BX\nMOVB BX,ret+48(FP)") + for index, body := range bodies { + function := fmt.Sprintf("Condition%d", index) + kernel := fixedLocalTransportAssembly(function, 49, body) + kernel = strings.ReplaceAll(strings.ReplaceAll(kernel, "VMOVDQA", "MOVOA"), "Y14", "X14") + assembly.WriteString(kernel) + fmt.Fprintf(&declarations, "func %s(%s) bool\n", function, fixedLocalTransportParams) + fmt.Fprintf(&observerDeclarations, "func Observe%d(%s) (raw, adjacent, input uint64)\n", index, fixedLocalTransportParams) + // Original-only ABI0 adapter. The callee and this NOSPLIT observer + // contain no Go calls, allocation or pointee access. It snapshots the + // actual output word and neighbors, not a Go-normalized bool value. + fmt.Fprintf(&observers, `TEXT ·Observe%d(SB),4,$64-72 +MOVQ a_base+0(FP),AX +MOVQ AX,0(SP) +MOVQ a_len+8(FP),AX +MOVQ AX,8(SP) +MOVQ a_cap+16(FP),AX +MOVQ AX,16(SP) +MOVQ p+24(FP),AX +MOVQ AX,24(SP) +MOVB b+32(FP),AX +MOVB AX,32(SP) +MOVQ n+40(FP),AX +MOVQ AX,40(SP) +MOVQ $0x5a5a5a5a5a5a5a5a,AX +MOVQ AX,48(SP) +MOVQ $0x6b6b6b6b6b6b6b6b,AX +MOVQ AX,56(SP) +CALL ·%s(SB) +MOVQ 48(SP),AX +MOVQ AX,raw+48(FP) +MOVQ 56(SP),AX +MOVQ AX,adjacent+56(FP) +MOVQ 40(SP),AX +MOVQ AX,input+64(FP) +RET +`, index, function) + fmt.Fprintf(&goRows, "{Condition%d,Observe%d},\n", index, index) + fmt.Fprintf(&llvmRows, ` +define i32 @condition_probe%d(i64 %%n, i64 %%len) { + %%a0 = insertvalue {ptr,i64,i64} zeroinitializer, i64 %%len, 1 + %%a1 = insertvalue {ptr,i64,i64} %%a0, i64 %%len, 2 + %%flag = trunc i64 %%len to i1 + %%b = call i1 @"example.invalid/framefixture.%s"({ptr,i64,i64} %%a1, ptr null, i1 %%flag, i64 %%n) + %%result = zext i1 %%b to i32 + ret i32 %%result +} +`, index, function) + fmt.Fprintf(&cDeclarations, "extern int condition_probe%d(uint64_t,uint64_t);\n", index) + fmt.Fprintf(&cRows, "condition_probe%d,", index) + } + goOracle := ` +func checkConditions(t *testing.T) { + rows:=[]struct{ + call func([]byte,*uint64,bool,uint64) bool + observe func([]byte,*uint64,bool,uint64)(uint64,uint64,uint64) + }{ +` + goRows.String() + ` + } + inputs:=[]uint64{0,1,2,3,1<<63,(1<<63)-1,^uint64(0)} + buffer:=[4]byte{1,2,3,4} + for i,row:=range rows { + seen:=0 + for _,a:=range inputs { for b:=uint64(0);b<4;b++ { + r:=a-b + cf,zf,sf:=a>63!=0 + of:=(a^b)&(a^r)>>63!=0 + parity:=byte(r); parity^=parity>>4; parity^=parity>>2; parity^=parity>>1 + pf:=parity&1==0 + want:=[]bool{!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf,false,true,true,b&1!=0,a==0,a==0,a==0,true}[i] + if row.call(buffer[:b],nil,b&1!=0,a)!=want { t.Fatalf("condition %%d",i) } + value:=uint64(0); if want { value=1 } + raw,adjacent,input:=row.observe(buffer[:b],nil,b&1!=0,a) + if raw!=0x5a5a5a5a5a5a5a00|value||adjacent!=0x6b6b6b6b6b6b6b6b||input!=a { t.Fatalf("physical byte/canary %%d %%x %%x %%x",i,raw,adjacent,input) } + seen|=1<>63,of=((a^b)&(a^r))>>63; + uint8_t parity=r; parity^=parity>>4; parity^=parity>>2; parity^=parity>>1; + bool pf=(parity&1)==0; + bool want[]={!cf,cf,zf,sf==of,!zf&&sf==of,!cf&&!zf,zf||sf!=of,cf||zf,sf!=of,sf,!zf,!of,of,!pf,!sf,pf,false,true,true,(b&1)!=0,a==0,a==0,a==0,true}; + int actual=condition_probes[i](a,b); + if(actual!=want[i]) { + fprintf(stderr,"condition=%u a=%llx b=%llx actual=%d expected=%d\n",i,(unsigned long long)a,(unsigned long long)b,(int)actual,(int)want[i]); + return 1; + } + seen|=1U< 0 { + flags |= 1 + } + fmt.Fprintf(&tests, "if got,flags:=%s(0x%x,0x%x,0x1f80);got!=0x%x || flags!=0x%x {t.Fatalf(\"%s NaN got=%%x flags=%%x\",got,flags)}\n", name, vector[0], vector[1], vector[2], flags, name) + } + one, half, next := uint64(0x3f800000), uint64(0x33800000), uint64(0x3f800001) + if bits == 64 { + one, half, next = 0x3ff0000000000000, 0x3ca0000000000000, 0x3ff0000000000001 + } + if operation == "SUB" { + half |= uint64(1) << (bits - 1) // 1-(-half ULP) has the same tie + } + for mode := 0; mode < 4; mode++ { + want := one + if mode == 2 { + want = next + } + fmt.Fprintf(&tests, "if got,flags:=%s(0x%x,0x%x,0x%x);got!=0x%x || flags!=0x%x {t.Fatalf(\"%s mode%d got=%%x flags=%%x\",got,flags)}\n", name, one, half, 0x1f80|mode<<13, want, 0x1fa0|mode<<13, name, mode) + } + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go SSE bit/round oracle: %v\n%s", err, output) + } +} diff --git a/amd64_float_add_sub_literal.go b/amd64_float_add_sub_literal.go new file mode 100644 index 000000000..f7b6dfab1 --- /dev/null +++ b/amd64_float_add_sub_literal.go @@ -0,0 +1,41 @@ +package plan9asm + +import ( + "math" + "strings" +) + +func x86FloatAddSubLiteral(ins Instr, spec x86FloatAddSubSpec) bool { + return x86FloatArithmeticLiteral(ins, spec.legacy, spec.scalar) +} + +// obj6.progedit rewrites legacy scalar ADD/SUB/MUL/DIV constants to +// Float32Sym/Float64Sym. A literal is not caller memory or a register input. +// Both reviewed Go encoders use the same arithmetic rewrite, including ±0. +func x86FloatArithmeticLiteral(ins Instr, legacy, scalar bool) bool { + if !legacy || !scalar || len(ins.Args) != 2 { + return false + } + operand := ins.Args[0] + if operand.Kind != OpImm || !operand.ImmIsFloat || operand.ImmRaw != "" { + return false + } + value := math.Float64frombits(uint64(operand.Imm)) + if math.IsInf(value, 0) || math.IsNaN(value) { + return false // these are not stock Go FLOAT tokens + } + if ins.Raw == "" { + return true // explicitly typed finite value, without source syntax + } + _, rest := splitOpcode(ins.Raw) + arguments := splitTopLevelCSV(rest) + if len(arguments) != 2 { + return false + } + text := strings.TrimSpace(arguments[0]) + if !strings.HasPrefix(text, "$") { + return false + } + parsed, ok := parseGoFloatLiteral(strings.TrimSpace(text[1:])) + return ok && math.Float64bits(parsed) == uint64(operand.Imm) +} diff --git a/amd64_float_add_sub_literal_runtime_test.go b/amd64_float_add_sub_literal_runtime_test.go new file mode 100644 index 000000000..0319cb882 --- /dev/null +++ b/amd64_float_add_sub_literal_runtime_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86AddSubLiteralNumericalSource() (declarations, assembly string) { + var goSource, machine strings.Builder + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + for _, value := range x86AddSubLiteralValues { + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&goSource, "func %s(input *[32]byte, output *[80]byte, csr uint32)\n", name) + fmt.Fprintf(&machine, `TEXT ·%s(SB),4,$8-20 +STMXCSR saved-4(SP) +LDMXCSR csr+16(FP) +MOVQ input+0(FP),AX +MOVQ output+8(FP),BX +VMOVUPS (AX),Y0 +%s $%s,X0 +MOVUPS X0,8(BX) +VMOVUPS Y0,24(BX) +STMXCSR 56(BX) +LDMXCSR saved-4(SP) +RET +`, name, opcode, value.text) + } + } + return goSource.String(), machine.String() +} + +func x86AddSubLiteralCOracle() string { + var declarations, checks strings.Builder + index := 0 + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86AddSubLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&declarations, "extern void %s(void*,void*,unsigned);\n", name) + fmt.Fprintf(&checks, " if (check_literal(%s,%q,%d,UINT64_C(0x%x),%d)) return %d;\n", name, name, bits, literal, x86FloatAddSubBoolInt(strings.HasPrefix(opcode, "SUB")), index+1) + index++ + } + } + return x86FloatAddSubSSEOracle + declarations.String() + ` +typedef void (*literal_fn)(void*,void*,unsigned); +static int check_literal(literal_fn fn,const char *name,unsigned bits,uint64_t literal,int subtract) { + const uint64_t edge32[]={0,0x80000000,1,0x80000001,0x3f800000,0xbf800000, + 0x7f7fffff,0x7f800000,0xff800000,0x7fc12345,0x7f812345}; + const uint64_t edge64[]={0,UINT64_C(0x8000000000000000),1,UINT64_C(0x8000000000000001), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x7fefffffffffffff), + UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0x7ff0123456789abc)}; + const unsigned controls[]={0x1f80,0x1f85,0x1fc0,0x9f80,0x9fc0,0x3f80,0x5f80,0x7f80}; + for(unsigned sample=0;sample<11;sample++) { + unsigned char input[32],before[32],output[80],want[80]; + for(unsigned byte=0;byte<32;byte++) input[byte]=(unsigned char)(byte*37+19); + uint64_t left=bits==32?edge32[sample]:edge64[sample]; + memcpy(input,&left,bits/8); + memcpy(before,input,32); + for(unsigned mode=0;mode<8;mode++) { + unsigned ambient=0x3f85,after; + memset(output,0xa7,80); + memset(want,0xa7,80); + uint64_t result=sse_result(bits,left,literal,subtract,controls[mode]); + unsigned expectedCSR=last_sse_csr; + memcpy(want+8,input,16); + memcpy(want+24,input,32); + memcpy(want+8,&result,bits/8); + memcpy(want+24,&result,bits/8); + memcpy(want+56,&expectedCSR,4); + __asm__ volatile("ldmxcsr %0"::"m"(ambient)); + fn(input,output,controls[mode]); + __asm__ volatile("stmxcsr %0":"=m"(after)); + if(memcmp(input,before,32)||memcmp(output,want,80)||after!=ambient) { + fprintf(stderr,"%s bits=%u sample=%u csr=%x after=%x\n",name,bits,sample,controls[mode],after); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + ` + unsigned csr=0x1f80; + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + return 0; +} +` +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubLiteralNumerical(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86AddSubLiteralNumericalSource() + x86ScalarLiteralGoListing(t, "amd64", assembly) + pkg := x86AddSubLiteralPackage(t, "amd64", declarations) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.TrimPrefix(sym, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var translation *GoModuleTranslation + translation, err = translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{GOARCH: "amd64", TargetTriple: triple, ResolveSym: resolve}) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + ir = translation.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "addsub-literal-"+api, triple, ir, x86AddSubLiteralCOracle(), prefix) + }) + } + t.Log("56 original literal forms × 11 independent SSE bit inputs × 8 full MXCSR controls × 3 translation APIs; whole X/Y bytes and canaries") +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubLiteralOriginalGo(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86AddSubLiteralNumericalSource() + var checks strings.Builder + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86AddSubLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + fmt.Fprintf(&checks, " check(t,%s,%d,0x%x,%t)\n", strings.ToLower(opcode)+value.name, bits, literal, strings.HasPrefix(opcode, "SUB")) + } + } + goTest := `package literal +import("encoding/binary";"math";"testing") +type literalFn func(*[32]byte,*[80]byte,uint32) +func check(t *testing.T,fn literalFn,bits int,literal uint64,subtract bool) { + t.Helper() + for _,negative:=range []bool{false,true} { + var input [32]byte + var output [80]byte + for i:=range input {input[i]=byte(i*37+19)} + for i:=range output {output[i]=0xa7} + var result uint64 + if bits==32 { + left:=float32(4) + if negative {left=-left} + binary.LittleEndian.PutUint32(input[:4],math.Float32bits(left)) + right:=math.Float32frombits(uint32(literal)) + if subtract {left-=right}else{left+=right} + result=uint64(math.Float32bits(left)) + }else{ + left:=float64(4) + if negative {left=-left} + binary.LittleEndian.PutUint64(input[:8],math.Float64bits(left)) + right:=math.Float64frombits(literal) + if subtract {left-=right}else{left+=right} + result=math.Float64bits(left) + } + before:=input + fn(&input,&output,0x1f80) + if input!=before {t.Fatal("input changed")} + for i,got:=range output { + if i>=56&&i<60 {continue} // CSR is checked independently by the C SSE oracle. + want:=byte(0xa7) + if i>=8&&i<24 {want=input[i-8]} + if i>=24&&i<56 {want=input[i-24]} + if i>=8&&i<8+bits/8 {want=byte(result>>(8*(i-8)))} + if i>=24&&i<24+bits/8 {want=byte(result>>(8*(i-24)))} + if got!=want {t.Fatalf("bits=%d literal=%x byte=%d got=%x want=%x",bits,literal,i,got,want)} + } + } +} +func TestOriginalLiteralNumerical(t *testing.T) { +` + checks.String() + "}\n" + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module literal\n\ngo 1.20\n", "literal.go": "package literal\n" + declarations, + "literal_amd64.s": assembly, "literal_test.go": goTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go original literal numerical: %v\n%s", err, output) + } + t.Log("actual Go original 56 literal kernels × 2 initialized finite inputs; X/Y aliases and whole output canaries (host execution may be Rosetta)") +} diff --git a/amd64_float_add_sub_literal_test.go b/amd64_float_add_sub_literal_test.go new file mode 100644 index 000000000..9eee3c08e --- /dev/null +++ b/amd64_float_add_sub_literal_test.go @@ -0,0 +1,360 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86AddSubLiteralValues = []x86ScalarLiteralValue{ + {name: "one", text: "1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "plus", text: "+1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "minus", text: "-1.0", bits32: 0xbf800000, bits64: 0xbff0000000000000}, + {name: "zero", text: "0.0"}, + {name: "negativeZero", text: "-0.0", bits32: 0x80000000, bits64: 0x8000000000000000}, + {name: "hex", text: "0x1.4p+0", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "scientific", text: "6.25e-1", bits32: 0x3f200000, bits64: 0x3fe4000000000000}, + {name: "parentheses", text: "(1.25)", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "signedParentheses", text: "(-2.5)", bits32: 0xc0200000, bits64: 0xc004000000000000}, + {name: "roundEven", text: "1.000000059604644775390625", bits32: 0x3f800000, bits64: 0x3ff0000010000000}, + {name: "underflow32", text: "1e-50", bits64: 0x358dee7a4ad4b81f}, + {name: "overflow32", text: "1e40", bits32: 0x7f800000, bits64: 0x483d6329f1c35ca5}, + {name: "underscore", text: "1_000.0", bits32: 0x447a0000, bits64: 0x408f400000000000}, + {name: "nestedSign", text: "+(-(+1.25))", bits32: 0xbfa00000, bits64: 0xbff4000000000000}, +} + +func x86AddSubLiteralPackage(t *testing.T, arch, declarations string) GoPackage { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "literal.go", "package literal\n"+declarations, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("literal", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "literal", Types: pkg, Syntax: []*ast.File{file}} +} + +func TestX86FloatAddSubLiteralActualGoObjects(t *testing.T) { + // This shared oracle authenticates the invoked SDK/assembler and hashes + // obj6/asm6/AVX tables before and after these original-source objects. + x86ScalarLiteralActualGoRewriteSet(t) + for _, name := range []string{"obj6.go", "asm6.go"} { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86", name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if name == "obj6.go" { + if err := x86AddSubLiteralRewriteSource(data); err != nil { + t.Fatal(err) + } + } else { + for _, op := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + if !bytes.Contains(data, []byte("{A"+op+", yxm,")) { + t.Fatalf("actual Go no longer uses yxm for %s", op) + } + } + } + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + for _, op := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + for _, value := range x86AddSubLiteralValues { + for _, register := range []int{0, 7, 8, 15} { + row := fmt.Sprintf("%s $%s,X%d", op, value.text, register) + entry := fmt.Sprintf("TEXT literal%d(SB),4,$0-0\n%s\nRET\n", source.Len(), row) + if arch == "386" && register >= 8 { + x86ScalarLiteralExpectedGoListing(t, arch, entry, false) + continue + } + source.WriteString(entry) + } + } + } + listing, err := x86ScalarLiteralGoListing(t, arch, source.String()) + if err != nil { + t.Fatal(err) + } + for _, value := range x86AddSubLiteralValues { + for _, pool := range []string{fmt.Sprintf("$f32.%08x", value.bits32), fmt.Sprintf("$f64.%016x", value.bits64)} { + if !strings.Contains(listing, pool) { + t.Fatalf("actual Go pool rounding/sign missing %s", pool) + } + } + } + if strings.Contains(listing, "XORPS") { + t.Fatal("arithmetic zero literal incorrectly acquired MOV zero rewrite") + } + } +} + +// Independently bind the actual encoder's complete scalar pool-rewrite body, +// not just the presence of an opcode name somewhere in the source file. +func x86AddSubLiteralRewriteSource(source []byte) error { + file, err := parser.ParseFile(token.NewFileSet(), "obj6.go", source, 0) + if err != nil { + return err + } + seen := make(map[string]int) + var progedit *ast.FuncDecl + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == "progedit" { + if progedit != nil || function.Body == nil { + return fmt.Errorf("missing or duplicate progedit body") + } + progedit = function + } + } + if progedit == nil { + return fmt.Errorf("actual Go progedit missing") + } + allowed := make(map[*ast.CaseClause]bool) + for _, statement := range progedit.Body.List { + switchStatement, ok := statement.(*ast.SwitchStmt) + if !ok || switchStatement.Init != nil { + continue + } + selector, ok := switchStatement.Tag.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "As" { + continue + } + base, ok := selector.X.(*ast.Ident) + if !ok || base.Name != "p" { + continue + } + for _, entry := range switchStatement.Body.List { + if clause, ok := entry.(*ast.CaseClause); ok { + allowed[clause] = true + } + } + } + var failure error + ast.Inspect(progedit.Body, func(node ast.Node) bool { + clause, ok := node.(*ast.CaseClause) + if !ok { + return true + } + for _, expression := range clause.List { + name, ok := expression.(*ast.Ident) + if !ok || name.Name != "AADDSS" && name.Name != "AADDSD" && name.Name != "ASUBSS" && name.Name != "ASUBSD" { + continue + } + seen[name.Name]++ + if !allowed[clause] { + failure = fmt.Errorf("nested or conditional literal rewrite %s", name.Name) + } + bits := 64 + if strings.HasSuffix(name.Name, "SS") { + bits = 32 + } + expected := fmt.Sprintf(`package p +func f() { +if p.From.Type == obj.TYPE_FCONST { + f%d := %s + p.From.Type = obj.TYPE_MEM + p.From.Name = obj.NAME_EXTERN + p.From.Sym = ctxt.Float%dSym(f%d) + p.From.Offset = 0 +} +}`, bits, map[int]string{32: "float32(p.From.Val.(float64))", 64: "p.From.Val.(float64)"}[bits], bits, bits) + fixture, err := parser.ParseFile(token.NewFileSet(), "expected.go", expected, 0) + if err != nil { + failure = err + continue + } + body, err := x86ScalarLiteralBodySyntax(clause.Body) + want, wantErr := x86ScalarLiteralBodySyntax(fixture.Decls[0].(*ast.FuncDecl).Body.List) + if err != nil || wantErr != nil || body != want { + failure = fmt.Errorf("unrecognized actual Go %s literal rewrite body", name.Name) + } + } + return true + }) + for _, name := range []string{"AADDSS", "AADDSD", "ASUBSS", "ASUBSD"} { + if seen[name] != 1 { + return fmt.Errorf("actual Go literal rewrite registration %s count=%d", name, seen[name]) + } + } + return failure +} + +func TestX86FloatAddSubLiteralRewriteSourceRejectsMutation(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/obj6.go") + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := x86AddSubLiteralRewriteSource(source); err != nil { + t.Fatal(err) + } + before := sha256.Sum256(source) + for _, replacement := range [][2]string{ + {"func progedit(", "func unrelated("}, + {"AADDSS,", "AADDPS,"}, {"ASUBSD,", "ASUBPD,"}, + {"ctxt.Float32Sym(f32)", "ctxt.Float64Sym(f32)"}, {"p.From.Offset = 0", "p.From.Offset = 4"}, + } { + mutated := bytes.ReplaceAll(source, []byte(replacement[0]), []byte(replacement[1])) + if bytes.Equal(mutated, source) || x86AddSubLiteralRewriteSource(mutated) == nil { + t.Fatalf("unknown source rewrite accepted: %q", replacement) + } + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(after) != before { + t.Fatalf("source mutation oracle changed the real SDK: %v", err) + } +} + +func TestX86FloatAddSubLiteralOriginalInstructionBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + opcode string + prefix, opcodeByte byte + }{ + {"ADDSS", 0xf3, 0x58}, {"ADDSD", 0xf2, 0x58}, + {"SUBSS", 0xf3, 0x5c}, {"SUBSD", 0xf2, 0x5c}, + } { + for _, literal := range []string{"0.0", "-0.0", "1.0", "0x1.4p0"} { + code := x86PackedFirstGoInstructionBytes(t, arch, row.opcode+" $"+literal+",X0") + if !bytes.Equal(code[:4], []byte{row.prefix, 0x0f, row.opcodeByte, 0x05}) { + t.Fatalf("%s %s literal %s original bytes=%x", arch, row.opcode, literal, code) + } + } + } + } +} + +func TestX86FloatAddSubLiteralCallerKeepsDefinedBytes(t *testing.T) { + for _, opcode := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + file, err := Parse(ArchAMD64, "TEXT literal(SB),4,$0-0\n"+opcode+" $-0.0,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatAddSubProperties(ins, "amd64") + if !recognized || err != nil || !form.literal || form.memory != -1 || form.memoryBytes != 0 { + t.Fatalf("literal confused with caller memory: %+v / %v", form, err) + } + low := x86CallerVectorByteMask(int64(form.bits / 8)) + for _, old := range []uint64{0, low &^ 1, low, low | 0xf0000000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": old}, masks: map[Reg]bool{}} + before, _ := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFloatAddSubStep(&ins, form, &state, nil) + if accepted != (old&low == low) || !accepted && !reflect.DeepEqual(before, state.vectors) || accepted && state.vectors["Z0"] != old { + t.Fatalf("%s invented destination/upper knowledge: old=%x state=%x accepted=%t", opcode, old, state.vectors["Z0"], accepted) + } + } + // Source spelling is not permission to use a different typed value. + ins.Args[0].Imm++ + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + _, _, err = ctx.lowerFloatAddSub(ins.Op, ins) + if err == nil || output.String() != before { + t.Fatal("mismatched literal/source bits emitted IR") + } + } +} + +func TestX86FloatAddSubLiteralThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var declarations, source strings.Builder + for _, op := range []string{"ADDSS", "ADDSD", "SUBSS", "SUBSD"} { + for _, value := range x86AddSubLiteralValues { + name := strings.ToLower(op) + value.name + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\nPXOR X0,X0\n%s $%s,X0\nRET\n", name, op, value.text) + } + } + pkg := x86AddSubLiteralPackage(t, target.arch, declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.Replace(sym, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var result *GoModuleTranslation + result, err = translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + ir = result.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "add-sub-literal.ll", "add-sub-literal.o", ir) + }) + } + }) + } +} + +func TestX86FloatAddSubLiteralRejectsNonGoFormats(t *testing.T) { + for _, row := range []string{ + "ADDSD $1,X0", "ADDSS $0,X0", "SUBSD $0x3ff0000000000000,X0", + "ADDSD $(1.0+2.0),X0", "SUBSS $(1.0/2.0),X0", "ADDSS $1e999,X0", + "ADDPS $1.0,X0", "ADDPD $1.0,X0", "SUBPS $1.0,X0", "SUBPD $1.0,X0", + "VADDSS $1.0,X0,X1", "VADDSD $1.0,X0,X1", "VSUBSS $1.0,X0,X1", "VSUBSD $1.0,X0,X1", + "ADDSD.Z $1.0,X0", "ADDSS $1.0,Y0", "SUBSD $1.0,X16", "ADDSD X0,$1.0", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT literal(SB),4,$0-0\n" + row + "\nRET\n" + x86ScalarLiteralExpectedGoListing(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerFloatAddSub(ins.Op, ins) + if !known || term || err == nil || before != output.String() { + t.Fatalf("non-Go literal emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} diff --git a/amd64_float_add_sub_numerical_test.go b/amd64_float_add_sub_numerical_test.go new file mode 100644 index 000000000..94efa6b03 --- /dev/null +++ b/amd64_float_add_sub_numerical_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// This oracle executes real legacy SSE under an explicitly saved/restored +// MXCSR. It is independent of the translated opcode table and numerical +// accumulator, including NaN payload priority and DAZ/FTZ signed zeros. +const x86FloatAddSubSSEOracle = `#include +#include +#include +static unsigned last_sse_csr; +static uint64_t sse_result(unsigned width,uint64_t left,uint64_t right,int sub,unsigned csr) { + unsigned saved; __asm__ volatile("stmxcsr %0":"=m"(saved)); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + uint64_t bits=0; + if(width==32) { + uint32_t l=(uint32_t)left,r=(uint32_t)right,out; float a,b; + memcpy(&a,&l,4);memcpy(&b,&r,4); + if(sub) __asm__ volatile("subss %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("addss %1,%0":"+x"(a):"x"(b)); + memcpy(&out,&a,4);bits=out; + } else { + double a,b;memcpy(&a,&left,8);memcpy(&b,&right,8); + if(sub) __asm__ volatile("subsd %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("addsd %1,%0":"+x"(a):"x"(b)); + memcpy(&bits,&a,8); + } + __asm__ volatile("stmxcsr %0":"=m"(last_sse_csr)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); return bits; +} +` + +func x86FloatAddSubRuntimeTools(t *testing.T) (llc, clang, triple string, prefix []string) { + t.Helper() + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 numerical counterpart") + } + var found bool + llc, clang, found = findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang missing") + } + triple = testTargetTriple(runtime.GOOS, "amd64") + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + return +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubFixedRoundSSEExtremes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var ir strings.Builder + ir.WriteString("declare i128 @llvm.ctlz.i128(i128, i1)\n") + for _, bits := range []int{32, 64} { + typ := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(&ir, "declare %s @llvm.fma.f%d(%s,%s,%s)\n", typ, bits, typ, typ, typ) + emitAMD64FMA3RoundingHelper(&ir, bits) + emitX86FloatAddSubRoundingHelper(&ir, bits) + fmt.Fprintf(&ir, "define i%d @addsub_round%d(i%d %%a,i%d %%b,i1 %%sub,i32 %%mode,i32 %%csr){\n %%a_float=bitcast i%d %%a to %s\n %%b_float=bitcast i%d %%b to %s\n %%value=call %s @__plan9asm_add_sub_round_f%d(%s %%a_float,%s %%b_float,i1 %%sub,i32 %%mode,i32 %%csr)\n %%bits=bitcast %s %%value to i%d\n ret i%d %%bits\n}\n", bits, bits, bits, bits, bits, typ, bits, typ, typ, bits, typ, typ, typ, bits, bits) + } + main := x86FloatAddSubSSEOracle + ` +extern uint32_t addsub_round32(uint32_t,uint32_t,_Bool,unsigned,unsigned); +extern uint64_t addsub_round64(uint64_t,uint64_t,_Bool,unsigned,unsigned); +int main(void) { + const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x800000, + 0x7f7fffff,0xff7fffff,0x3f800000,0xbf800000,0x3f800001,0x33800000, + 0x7f800000,0xff800000,0x7fc12345,0xffc45678,0x7f812345,0xff845678}; + const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1, + UINT64_C(0x8000000000000001),UINT64_C(0xfffffffffffff),UINT64_C(0x10000000000000), + UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff),UINT64_C(0x3ff0000000000000), + UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000001),UINT64_C(0x3ca0000000000000), + UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000),UINT64_C(0x7ff8123456789abc), + UINT64_C(0xfff8fedcba987654),UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff0fedcba987654)}; + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0}; + uint64_t state=UINT64_C(0x7182329ab3293821); + for(unsigned width=32;width<=64;width+=32) for(unsigned pair=0;pair<1000;pair++) { + const uint64_t *edges=width==32?edges32:edges64; + uint64_t a=edges[pair%18],b=edges[(pair/18)%18]; + if(pair>=324){ state^=state<<13;state^=state>>7;state^=state<<17;a=state; + state^=state<<13;state^=state>>7;state^=state<<17;b=state; } + for(unsigned sub=0;sub<2;sub++) for(unsigned mode=0;mode<4;mode++) + for(unsigned ambient=0;ambient<4;ambient++) for(unsigned control=0;control<6;control++) { + unsigned csr=controls[control],ambientCSR=csr|(ambient<<13),after; + __asm__ volatile("ldmxcsr %0"::"m"(ambientCSR)); + uint64_t got=width==32?addsub_round32(a,b,sub,mode,csr):addsub_round64(a,b,sub,mode,csr); + __asm__ volatile("stmxcsr %0":"=m"(after)); + uint64_t want=sse_result(width,a,b,sub,csr|(mode<<13)); + if(got!=want || after!=ambientCSR) { + fprintf(stderr,"binary width/sub/mode/ambient/csr=%u/%u/%u/%u/%x a=%llx b=%llx got=%llx want=%llx after=%x\n", + width,sub,mode,ambient,csr,(unsigned long long)a,(unsigned long long)b,(unsigned long long)got,(unsigned long long)want,after);return 1; + } + } + } + unsigned csr=0x1f80;__asm__ volatile("ldmxcsr %0"::"m"(csr));return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "binary-fixed-sse", triple, ir.String(), main, prefix) +} + +func TestCrossLinuxRuntimeMatrixX86FloatAddSubCompletePhysicalSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, row := range x86FloatAddSubFormats("amd64") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + form, _, err := x86FloatAddSubProperties(file.Funcs[0].Instrs[1], "amd64") + if err != nil { + t.Fatal(err) + } + fixedMode := -1 + for mode, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + if form.rounding == suffix { + fixedMode = mode + } + } + name := fmt.Sprintf("binary%d", count) + count++ + row = strings.Replace(row, "8(AX)", "(BX)", 1) + row = strings.Replace(row, "(AX)", "(BX)", 1) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", name, row, map[bool]int{true: 1, false: 2}[form.spec.legacy]) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t);\n", name) + arithmeticBytes := form.byteWidth + if form.spec.scalar { + arithmeticBytes = form.spec.bits / 8 + } + opcode := strings.Split(strings.Fields(row)[0], ".")[0] + independentSubtract := strings.Contains(opcode, "SUB") + // Only legacy packed native memory has m128 alignment semantics. + // Keep scalar/VEX/EVEX inputs unaligned; the separate exact-opcode + // fault oracle retains every old mod16 case instead of erasing it. + aligned := form.legacy && !form.scalar && form.memory >= 0 + fmt.Fprintf(&checks, ` + for(int round=0;round<4;round++) for(int m=0;m<4;m++) for(int control=0;control<6;control++) for(int ambient=0;ambient<4;ambient++) { + uint8_t aStorage[96] __attribute__((aligned(16))),bStorage[68],out[67];uint8_t *a=aStorage+(%[13]d?16:1),*b=bStorage+3; + memset(aStorage,0xa7,96);memset(bStorage,0xb9,68);memset(out,0xc3,67); + for(int i=0;i<64/%[2]d;i++) { uint64_t av=%[3]d==32?UINT64_C(0x3f000001)+i*0x12345+round*0x10000:UINT64_C(0x3fe0000000000001)+i*0x123456789; + uint64_t bv=%[3]d==32?UINT64_C(0xbf800001)+i*0x12345:UINT64_C(0xbff0000000000001)+i*0x2345678; + if(round==1){av=%[3]d==32?UINT64_C(0x7fc12345)+i:UINT64_C(0x7ff8123456789abc)+i;bv=%[3]d==32?UINT64_C(0xffc45678)+i:UINT64_C(0xfff8fedcba987654)+i;} + if(round==2){av=%[3]d==32?UINT64_C(0x7f812345)+i:UINT64_C(0x7ff0123456789abc)+i;bv=%[3]d==32?UINT64_C(0xff800000):UINT64_C(0xfff0000000000000);} + if(round==3){av=1+i;bv=(%[3]d==32?UINT64_C(0x80000000):UINT64_C(0x8000000000000000))|i;} + memcpy(a+i*%[2]d,&av,%[2]d);memcpy(b+i*%[2]d,&bv,%[2]d); } + unsigned csr=controls[control]|(ambient<<13),observed,expectedFlags=0; + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + uint64_t mask=masks[m];%[1]s(out+1,a,b,mask); + __asm__ volatile("stmxcsr %%0":"=m"(observed)); + for(int i=0;i<64/%[2]d;i++) { + uint64_t av=0,bv=0,got=0;memcpy(&av,a+(%[4]d?0:i*%[2]d),%[2]d);memcpy(&bv,b+i*%[2]d,%[2]d);memcpy(&got,out+1+i*%[2]d,%[2]d); + int active=%[5]d<0 || ((mask>>i)&1); + unsigned operationCSR=%[12]d<0?csr:((csr&~0x6000u)|((unsigned)%[12]d<<13)); + uint64_t want=i*%[2]d<%[6]d?(active?sse_result(%[3]d,bv,av,%[7]d,operationCSR):(%[8]d?0:bv)):(%[9]d || (%[10]d && i*%[2]d<16)?bv:0); + if(i*%[2]d<%[6]d && active && %[12]d<0)expectedFlags|=last_sse_csr&63; + if(got!=want){fprintf(stderr,"%[1]s lane=%%d got=%%llx want=%%llx\n",i,(unsigned long long)got,(unsigned long long)want);return %[11]d;} + } + if(observed!=(csr|expectedFlags)){fprintf(stderr,"%[1]s sticky observed=%%x expected=%%x\n",observed,csr|expectedFlags);return 3;} + if(out[0]!=0xc3 || out[65]!=1 || out[66]!=0xc3 || a[-1]!=0xa7 || a[64]!=0xa7 || bStorage[2]!=0xb9 || bStorage[67]!=0xb9)return 2; + } +`, name, form.spec.bits/8, form.spec.bits, x86FloatAddSubBoolInt(form.broadcast), form.mask, arithmeticBytes, x86FloatAddSubBoolInt(independentSubtract), x86FloatAddSubBoolInt(form.zeroing), x86FloatAddSubBoolInt(form.spec.legacy), x86FloatAddSubBoolInt(form.spec.scalar), count+10, fixedMode, x86FloatAddSubBoolInt(aligned)) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86FloatAddSubSSEOracle + declarations.String() + "int main(void){const uint64_t masks[]={0,1,0x55aa55aa55aa55aaULL,~0ULL}; const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0};\n" + checks.String() + "unsigned csr=0x1f80;__asm__ volatile(\"ldmxcsr %0\"::\"m\"(csr));return 0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "binary-complete-semantics", triple, ir, main, prefix) + t.Logf("%d independent opcode/width/memory/mask programs x384 IEEE/MXCSR/canary inputs", count) +} + +func x86FloatAddSubBoolInt(value bool) int { + if value { + return 1 + } + return 0 +} diff --git a/amd64_float_add_sub_runtime_test.go b/amd64_float_add_sub_runtime_test.go new file mode 100644 index 000000000..fc24a049c --- /dev/null +++ b/amd64_float_add_sub_runtime_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86FloatAddSubPhysicalAliasesAndInactiveMemory(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang missing") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + for _, fixture := range []struct { + name, body, checks string + }{ + {"packed upper aliases", "VADDPS X1,X0,X2", "for(int i=0;i<16;i++) if(out[i]!=(i<4?5.0f:0.0f)) return 11;"}, + {"scalar second-source passthrough", "VSUBSS X1,X0,X2", "for(int i=0;i<16;i++) if(out[i]!=(i==0?-1.0f:i<4?2.0f:0.0f)) return 12;"}, + {"inactive memory", "MOVQ $0,DX\nKMOVQ DX,K1\nVADDPS.Z (CX),X0,K1,X2", "for(int i=0;i<16;i++) if(out[i]!=0.0f) return 13;"}, + {"inactive scalar memory", "MOVQ $0,DX\nKMOVQ DX,K1\nVSUBSD.Z (CX),X0,K1,X2", "for(int i=0;i<16;i++) if(out[i]!=(i<2?0.0f:i<4?2.0f:0.0f)) return 14;"}, + {"inactive broadcast memory", "MOVQ $0,DX\nKMOVQ DX,K1\nVADDPD.BCST.Z (CX),Z0,K1,Z2", "for(int i=0;i<16;i++) if(out[i]!=0.0f) return 15;"}, + } { + t.Run(fixture.name, func(t *testing.T) { + source := "TEXT addsub(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (BX),Z0\nVMOVUPS (BX),Z2\n" + if !strings.HasPrefix(fixture.name, "inactive") { + source += "VMOVUPS (CX),Z1\n" + } + source += fixture.body + "\nVMOVUPS Z2,(AX)\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{ + "addsub": {Name: "addsub", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}}, + }}) + if err != nil { + t.Fatal(err) + } + argument := "b" + if strings.HasPrefix(fixture.name, "inactive") { + argument = "0" + } + main := fmt.Sprintf("extern void addsub(float*,const float*,const float*);\nint main(void){float a[16],b[16],out[16]; for(int i=0;i<16;i++){a[i]=2;b[i]=3;} addsub(out,a,%s); %s return 0;}\n", argument, fixture.checks) + compileAndRunRuntimeTestForTarget(t, llc, clang, "addsub-alias", triple, ir, strings.TrimSpace(main), prefix) + }) + } +} + +func TestX86FloatAddSubFixedRoundingIgnoresAmbientMode(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang missing") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for mode, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + name := fmt.Sprintf("roundadd%d", mode) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVSS (BX),X0\nMOVSS (CX),X1\nVADDSS.%s X1,X0,X2\nMOVSS X2,(AX)\nRET\n", name, suffix) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint32_t*,const uint32_t*,const uint32_t*);\n", name) + expected := uint32(0x3f800000) + if mode == 2 { + expected++ + } + fmt.Fprintf(&checks, "for(int ambient=0;ambient<4;ambient++){if(fesetround(modes[ambient]))return 2; uint32_t a=0x3f800000,b=0x33800000,out=0; %s(&out,&a,&b); if(out!=0x%x)return %d;}\n", name, expected, mode+10) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n" + declarations.String() + "int main(void){const int modes[]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO};\n" + checks.String() + "return fesetround(FE_TONEAREST)!=0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "addsub-fixed-ambient", triple, ir, main, prefix) +} diff --git a/amd64_float_binary_forms.go b/amd64_float_binary_forms.go new file mode 100644 index 000000000..0b36a9b08 --- /dev/null +++ b/amd64_float_binary_forms.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's yxm, _yvaddpd and _yvaddsd shapes are shared by binary floating +// arithmetic. Operation semantics are deliberately not part of this grammar. +type x86FloatBinaryForm struct { + bits int + legacy, scalar bool + first, second, destination, mask, memory int + byteWidth, memoryBytes int + broadcast, zeroing bool + literal bool // family-authorized Go constant-pool rewrite + rounding string +} + +// Parse only shared operand axes. The registered family supplies precision +// and encoding generation; its lowerer retains operation-specific semantics. +func x86FloatBinaryOperandProperties(ins Instr, goarch string, bits int, legacy, scalar, literal bool, family string) (x86FloatBinaryForm, error) { + _, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + form := x86FloatBinaryForm{ + bits: bits, legacy: legacy, scalar: scalar, literal: literal, + first: 0, second: 1, destination: len(ins.Args) - 1, mask: -1, memory: -1, + } + fail := func() (x86FloatBinaryForm, error) { + return form, fmt.Errorf("%s %s operands/suffix outside Go floating %s grammar: %q", goarch, ins.Op, family, ins.Raw) + } + if goarch != "386" && goarch != "amd64" { + return fail() + } + if legacy { + if dotted || len(ins.Args) != 2 { + return fail() + } + form.byteWidth = 16 + } else { + properties, valid := parseAMD64FMA3Suffix(suffix) + if !valid || dotted && suffix == "" || scalar && properties.broadcast || len(ins.Args) != 3 && len(ins.Args) != 4 { + return fail() + } + form.broadcast, form.zeroing, form.rounding = properties.broadcast, properties.zeroing, properties.rounding + if len(ins.Args) == 4 { + if goarch == "386" && !ins.x86Encoded || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + form.mask = 2 + } else if form.zeroing { + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if scalar && form.byteWidth != 16 || form.rounding != "" && !scalar && form.byteWidth != 64 { + return fail() + } + } + register := func(operand Operand) bool { + if legacy { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + return amd64GoPackedVectorMoveRegister(operand, form.byteWidth, goarch) + } + if !register(ins.Args[form.second]) || !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if literal { + if !legacy || !scalar || first.Kind != OpImm || !first.ImmIsFloat || first.ImmRaw != "" { + return fail() + } + } else if isAMD64MemoryOperand(first) { + if form.rounding != "" { + return fail() + } + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory, form.memoryBytes = 0, form.byteWidth + if scalar || form.broadcast { + form.memoryBytes = bits / 8 + } + } else if form.broadcast || !register(first) { + return fail() + } + return form, nil +} diff --git a/amd64_float_binary_fp_load.go b/amd64_float_binary_fp_load.go new file mode 100644 index 000000000..e409b9c08 --- /dev/null +++ b/amd64_float_binary_fp_load.go @@ -0,0 +1,89 @@ +package plan9asm + +import "fmt" + +// FP operands describe checked virtual ABI0 fields. Loading one field using +// the whole instruction width loses later fields (or reads past an alloca). +func (c *amd64Ctx) loadFloatBinaryFPBits(source Operand, bits int) (string, error) { + bytes, err := c.loadFPVectorBytes(source.FPOffset, bits/8) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, bits/8, bytes, bits) + return "%" + value, nil +} + +func (c *amd64Ctx) loadFloatBinaryScalar(source Operand, bits int, mask string) (string, error) { + if source.Kind != OpFP { + return c.loadFMA3Scalar(source, bits, mask) + } + if err := c.validateFPVectorSpan(source.FPOffset, bits/8); err != nil { + return "", err + } + load := func() (string, error) { return c.loadFloatBinaryFPBits(source, bits) } + var value string + var err error + if mask == "" { + value, err = load() + } else { + value, err = c.loadX86ScalarIf(bits, amd64MaskBitI1(c, mask, 0), load) + } + if err != nil { + return "", err + } + floating := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to %s\n", floating, bits, value, amd64FMA3LLVMType(1, bits)) + return "%" + floating, nil +} + +func (c *amd64Ctx) loadFloatBinaryPacked(source Operand, width, bits int, broadcast bool, mask string) (string, error) { + if source.Kind != OpFP { + return c.loadMaskedPackedCompareLanes(source, width, bits, broadcast, mask) + } + memoryBytes := width + if broadcast { + memoryBytes = bits / 8 + } + if err := c.validateFPVectorSpan(source.FPOffset, memoryBytes); err != nil { + return "", err + } + lanes := width * 8 / bits + if broadcast { + load := func() (string, error) { return c.loadFloatBinaryFPBits(source, bits) } + var value string + var err error + if mask == "" { + value, err = load() + } else { + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s, i%d %s, i32 %d\n", inserted, lanes, bits, result, bits, value, lane) + result = "%" + inserted + } + return result, nil +} diff --git a/amd64_float_binary_fp_span_test.go b/amd64_float_binary_fp_span_test.go new file mode 100644 index 000000000..46777003e --- /dev/null +++ b/amd64_float_binary_fp_span_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +type x86FloatBinaryFPFixture struct { + name, opcode, operation, declaration, assembly string + bits, bytes, resultOffset int + sig FuncSig +} + +// Derive these layouts from actual Go declarations. Aggregate byte fields have +// separate virtual slots, not a pointer to one contiguous native FP word. +func x86FloatBinaryFPFixtures(t *testing.T) []x86FloatBinaryFPFixture { + t.Helper() + var fixtures []x86FloatBinaryFPFixture + var declarations strings.Builder + declarations.WriteString("package oracle\n") + for _, operation := range []string{"ADD", "SUB", "MUL", "DIV"} { + for _, bits := range []int{32, 64} { + for _, spelling := range []string{"legacy", "vex", "broadcast"} { + fixture := x86FloatBinaryFPFixture{operation: operation, bits: bits, bytes: bits / 8} + fixture.name = fmt.Sprintf("%s%d%s", strings.ToLower(operation), bits, spelling) + fixture.declaration = fmt.Sprintf("func %s(source [%d]byte,left uint%d) uint%d\n", fixture.name, fixture.bytes, bits, bits) + declarations.WriteString(fixture.declaration) + fixtures = append(fixtures, fixture) + } + } + } + set := token.NewFileSet() + file, err := parser.ParseFile(set, "value.go", declarations.String(), 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + pkg, err := (&types.Config{Sizes: sizes}).Check("oracle", set, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + for i := range fixtures { + fixture := &fixtures[i] + fixture.sig, err = DeriveGoFuncABI0Signature(fixture.name, pkg.Scope().Lookup(fixture.name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + fixture.resultOffset = int(fixture.sig.Frame.Results[0].Offset) + if fixture.resultOffset != 2*fixture.bytes || len(fixture.sig.Frame.Params) != fixture.bytes+1 { + t.Fatalf("independent Go ABI0 frame changed: %+v", fixture.sig.Frame) + } + move, suffix := "MOVSS", "SS" + if fixture.bits == 64 { + move, suffix = "MOVSD", "SD" + } + fixture.opcode = fixture.operation + suffix + row, destination := fixture.opcode+" source+0(FP),X0", "X0" + if strings.HasSuffix(fixture.name, "vex") { + fixture.opcode = "V" + fixture.opcode + row, destination = fixture.opcode+" source+0(FP),X0,X2", "X2" + } else if strings.HasSuffix(fixture.name, "broadcast") { + packedSuffix := "PS" + if fixture.bits == 64 { + packedSuffix = "PD" + } + fixture.opcode = "V" + fixture.operation + packedSuffix + ".BCST" + row, destination = fixture.opcode+" source+0(FP),X0,X2", "X2" + } + fixture.assembly = fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s left+%d(FP),X0\n%s\n%s %s,ret+%d(FP)\nRET\n", + fixture.name, fixture.resultOffset+fixture.bytes, move, fixture.bytes, row, move, destination, fixture.resultOffset) + } + return fixtures +} + +func TestCrossLinuxRuntimeMatrixX86FloatBinaryOriginalGoFPByteFields(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + checks.WriteString("package oracle\nimport \"testing\"\nfunc TestOriginalByteFields(t *testing.T){\n") + for _, fixture := range x86FloatBinaryFPFixtures(t) { + declarations.WriteString(fixture.declaration) + // asmdecl refuses scalar arithmetic on array types even when its byte + // size matches. A vetted LEAQ exposes the same original Go FP bytes; + // the floating opcode, operand order and independently known bits stay + // unchanged. The translator test below keeps the direct FP spelling. + memoryRow := fixture.opcode + " source+0(FP)" + addressedRow := "LEAQ source+0(FP),CX\n" + fixture.opcode + " (CX)" + assembly.WriteString(strings.Replace(fixture.assembly, memoryRow, addressedRow, 1)) + // The original executable uses baseline SSE only. Go still assembles + // every VEX/EVEX form; LLVM software lanes are checked separately. + if !strings.HasSuffix(fixture.name, "legacy") { + continue + } + left, source := uint64(0x3f800000), uint64(0x40200000) + want := map[string]uint64{"ADD": 0x40600000, "SUB": 0xbfc00000, "MUL": 0x40200000, "DIV": 0x3ecccccd}[fixture.operation] + if fixture.bits == 64 { + left, source = 0x3ff0000000000000, 0x4004000000000000 + want = map[string]uint64{"ADD": 0x400c000000000000, "SUB": 0xbff8000000000000, "MUL": 0x4004000000000000, "DIV": 0x3fd999999999999a}[fixture.operation] + } + var args []string + for i := 0; i < fixture.bytes; i++ { + args = append(args, fmt.Sprint(byte(source>>uint(i*8)))) + } + arguments := fmt.Sprintf("[%d]byte{%s},uint%d(0x%x)", fixture.bytes, strings.Join(args, ","), fixture.bits, left) + fmt.Fprintf(&checks, "if got:=%s(%s);got!=0x%x{t.Fatalf(%q,got)}\n", fixture.name, arguments, want, fixture.name+" got=%x") + } + checks.WriteString("}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": checks.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=asmdecl", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go cross-field floating memory oracle: %v\n%s", err, output) + } +} + +func TestCrossLinuxRuntimeMatrixX86FloatBinaryFPByteFields(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, wrappers, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for _, fixture := range x86FloatBinaryFPFixtures(t) { + assembly.WriteString(fixture.assembly) + sigs[fixture.name] = fixture.sig + // A bit-carrier wrapper constructs the LLVM array argument explicitly; + // C aggregate calling conventions are not the translated Go signature. + fmt.Fprintf(&wrappers, "define i%d @check_%s(i%d %%right,i%d %%left){\n", fixture.bits, fixture.name, fixture.bits, fixture.bits) + packed := "poison" + for i := 0; i < fixture.bytes; i++ { + fmt.Fprintf(&wrappers, " %%shift%d=lshr i%d %%right,%d\n %%byte%d=trunc i%d %%shift%d to i8\n %%pack%d=insertvalue [%d x i8] %s,i8 %%byte%d,%d\n", + i, fixture.bits, i*8, i, fixture.bits, i, i, fixture.bytes, packed, i, i) + packed = fmt.Sprintf("%%pack%d", i) + } + fmt.Fprintf(&wrappers, " %%result=call i%d @%s([%d x i8] %s,i%d %%left)\n ret i%d %%result\n}\n", fixture.bits, fixture.name, fixture.bytes, packed, fixture.bits, fixture.bits) + fmt.Fprintf(&declarations, "extern uint%d_t check_%s(uint%d_t,uint%d_t);\n", fixture.bits, fixture.name, fixture.bits, fixture.bits) + oracle, flags, secondOperation := "sse_result", "last_sse_csr", fixture.operation == "SUB" + if fixture.operation == "MUL" || fixture.operation == "DIV" { + oracle, flags, secondOperation = "sse_muldiv_result", "last_sse_muldiv_csr", fixture.operation == "DIV" + } + fmt.Fprintf(&checks, ` + for(unsigned rc=0;rc<4;rc++) for(unsigned index=0;index<64;index++) { + uint64_t left=%[1]d==32?UINT64_C(0x3f800001)+index*3571:UINT64_C(0x3ff0000000000001)+index*7912347; + uint64_t right=%[1]d==32?UINT64_C(0x40200001)+index*761:UINT64_C(0x4004000000000001)+index*1234567; + unsigned csr=0x1f80|(rc<<13),after; + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + uint64_t got=check_%[2]s(right,left); + __asm__ volatile("stmxcsr %%0":"=m"(after)); + uint64_t want=%[3]s(%[1]d,left,right,%[4]d,csr); + if(got!=want || after!=%[5]s){fprintf(stderr,"%[2]s rc=%%u got=%%llx want=%%llx after=%%x expectedCSR=%%x\n",rc,(unsigned long long)got,(unsigned long long)want,after,%[5]s);return 1;} + } +`, fixture.bits, fixture.name, oracle, x86FloatAddSubBoolInt(secondOperation), flags) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(name string) string { return strings.TrimPrefix(name, "·") }, + }) + if err != nil { + t.Fatal(err) + } + mulDivOracle := strings.NewReplacer("subss", "divss", "subsd", "divsd", "addss", "mulss", "addsd", "mulsd", + "sse_result", "sse_muldiv_result", "last_sse_csr", "last_sse_muldiv_csr").Replace(x86FloatAddSubSSEOracle) + main := x86FloatAddSubSSEOracle + mulDivOracle + declarations.String() + "int main(void){\n" + checks.String() + + "unsigned csr=0x1f80;__asm__ volatile(\"ldmxcsr %0\"::\"m\"(csr));return 0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "float-binary-fp-bytes", triple, ir+wrappers.String(), main, prefix) +} diff --git a/amd64_float_compare_forms.go b/amd64_float_compare_forms.go new file mode 100644 index 000000000..7c2f9ac30 --- /dev/null +++ b/amd64_float_compare_forms.go @@ -0,0 +1,133 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86FloatCompareSpec struct { + bits int + legacy, scalar bool +} + +var x86FloatCompareSpecs = map[Op]x86FloatCompareSpec{ + "CMPPS": {32, true, false}, "CMPPD": {64, true, false}, + "CMPSS": {32, true, true}, "CMPSD": {64, true, true}, + "VCMPPS": {32, false, false}, "VCMPPD": {64, false, false}, + "VCMPSS": {32, false, true}, "VCMPSD": {64, false, true}, +} + +// Compare has an immediate position and a K-result generation distinct from +// binary arithmetic. The shared source-width axes do not imply merge/zero +// destination semantics or shared floating exception policy. +type x86FloatCompareForm struct { + spec x86FloatCompareSpec + x86FloatBinaryForm + immediate uint8 + maskResult, sae bool +} + +func x86FloatCompareProperties(ins Instr, goarch string) (x86FloatCompareForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86FloatCompareSpecs[Op(base)] + if !known { + return x86FloatCompareForm{}, false, nil + } + form := x86FloatCompareForm{spec: spec, x86FloatBinaryForm: x86FloatBinaryForm{ + bits: spec.bits, legacy: spec.legacy, scalar: spec.scalar, + mask: -1, memory: -1, + }} + fail := func() (x86FloatCompareForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go floating compare grammar: %q", goarch, ins.Op, ins.Raw) + } + if goarch != "386" && goarch != "amd64" { + return fail() + } + if spec.legacy { + if dotted || len(ins.Args) != 3 { + return fail() + } + if ins.Args[2].Kind == OpImm && (ins.Args[2].ImmIsFloat || ins.Args[2].ImmRaw != "") { + return fail() + } + immediate, valid := amd64LegacyFloatingCompareImmediate(ins.Args[2]) + if !valid || immediate < -128 || immediate > 127 { + return fail() + } + form.immediate = uint8(immediate) + form.first, form.second, form.destination, form.byteWidth = 0, 1, 1, 16 + } else { + if goarch == "386" && !ins.x86Encoded || len(ins.Args) != 4 && len(ins.Args) != 5 || + dotted && suffix != "BCST" && suffix != "SAE" || ins.Args[0].Kind != OpImm || ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return fail() + } + form.immediate = uint8(ins.Args[0].Imm) + form.first, form.second, form.destination = 1, 2, len(ins.Args)-1 + form.broadcast, form.sae = suffix == "BCST", suffix == "SAE" + form.maskResult = amd64IsKOperand(ins.Args[form.destination]) + if form.maskResult { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.second].Reg) + if len(ins.Args) == 5 { + if !amd64NonzeroKOperand(ins.Args[3]) { + return fail() + } + form.mask = 3 + } + } else { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if dotted || len(ins.Args) != 4 || form.byteWidth != 16 && form.byteWidth != 32 { + return fail() + } + } + if spec.scalar && form.byteWidth != 16 || form.byteWidth != 16 && form.byteWidth != 32 && form.byteWidth != 64 { + return fail() + } + } + register := func(operand Operand) bool { + if operand.Kind != OpReg || amd64VectorByteWidth(operand.Reg) != form.byteWidth { + return false + } + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + limit := 16 + if goarch == "386" { + limit = 8 + } else if form.maskResult { + limit = 32 + } + return valid && index < limit + } + if !register(ins.Args[form.second]) || !form.maskResult && !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if form.sae { + return fail() + } + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory, form.memoryBytes = form.first, form.byteWidth + if spec.scalar || form.broadcast { + form.memoryBytes = spec.bits / 8 + } + } else if !register(first) || form.broadcast { + return fail() + } + if form.broadcast && (spec.scalar || !form.maskResult) || form.sae && (!form.maskResult || !spec.scalar && form.byteWidth != 64) { + return fail() + } + return form, true, nil +} diff --git a/amd64_float_compare_forms_test.go b/amd64_float_compare_forms_test.go new file mode 100644 index 000000000..939cc4c01 --- /dev/null +++ b/amd64_float_compare_forms_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86FloatCompareOpcodes() []string { + return []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} +} + +// Enumerate independently from the production grammar: yxcmpi has one +// signed-imm8 row, _yvcmppd has eight rows, and _yvcmpsd has three rows. +func x86FloatCompareGoForms(arch string) []string { + var rows []string + for _, opcode := range x86FloatCompareOpcodes() { + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + if !strings.HasPrefix(opcode, "V") { + for _, source := range []string{"X0", "8(AX)", "(AX)(CX*4)"} { + for _, immediate := range []int{-128, -1, 0, 7, 127} { + rows = append(rows, fmt.Sprintf("%s %s,X1,$%d", opcode, source, immediate)) + } + } + continue + } + if arch == "386" { + continue // Go rejects named VCMP; physical raw forms are separate. + } + widths := []string{"X", "Y"} + if scalar { + widths = widths[:1] + } + for _, width := range widths { + for _, source := range []string{width + "0", "8(AX)", "(AX)(CX*4)"} { + for _, immediate := range []int{0, 31, 255} { + rows = append(rows, fmt.Sprintf("%s $%d,%s,%s1,%s2", opcode, immediate, source, width, width)) + } + } + } + widths = []string{"X", "Y", "Z"} + if scalar { + widths = widths[:1] + } + for _, width := range widths { + for _, mask := range []string{"", "K1,"} { + for _, destination := range []string{"K0", "K7"} { + for _, source := range []string{width + "20", "8(AX)", "(AX)(CX*4)"} { + rows = append(rows, fmt.Sprintf("%s $255,%s,%s21,%s%s", opcode, source, width, mask, destination)) + } + if !scalar { + rows = append(rows, fmt.Sprintf("%s.BCST $31,8(AX),%s21,%s%s", opcode, width, mask, destination)) + } + if scalar || width == "Z" { + rows = append(rows, fmt.Sprintf("%s.SAE $31,%s20,%s21,%s%s", opcode, width, width, mask, destination)) + } + } + } + } + } + return rows +} + +func TestX86FloatingCompareCompleteGoGrammarFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + sigs := map[string]FuncSig{} + for index, row := range x86FloatCompareGoForms(arch) { + name := fmt.Sprintf("compare_form%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-macosx", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "compare-formats.ll", "compare-formats.o", ir) + } + t.Logf("%s: %d independent Go compare forms", arch, len(sigs)) + } +} + +func TestX86FloatingCompareMalformedAndDispatchIdentityBeforeEmission(t *testing.T) { + for _, row := range []string{ + "CMPPS X0,X1,$128", "CMPSD X0,Y1,$0", "CMPSS $0,X1,$0", "CMPPD.SAE X0,X1,$0", + "VCMPPS $256,X0,X1,X2", "VCMPPD $0,X0,X1,Z2", "VCMPSS.Z $0,X0,X1,K2", + "VCMPPD $0,X0,X1,K0,K2", "VCMPPS.BCST $0,X0,X1,K2", "VCMPSS.BCST $0,(AX),X1,K2", + "VCMPPS.SAE $0,Y0,Y1,K2", "VCMPSS.SAE $0,(AX),X1,K2", "VCMPPS $0,X0,X1,X17", + "CMPPS (AX)(SP*2),X1,$0", "VCMPPS $0,(AX)(SP*2),X1,X2", "VCMPPS $0,$data(SB),X1,X2", + } { + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + return + } + context, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + context.goarch = "amd64" + instruction := file.Funcs[0].Instrs[1] + before := output.Len() + known, term, err := context.lowerFloatingCompare(instruction.Op, instruction) + if !known || term || err == nil || output.Len() != before { + t.Fatalf("malformed compare emitted IR: %t/%t/%v", known, term, err) + } + }) + } + for _, opcode := range x86FloatCompareOpcodes() { + row := opcode + " X0,X1,$0" + if strings.HasPrefix(opcode, "V") { + row = opcode + " $0,X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + context, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + context.goarch = "amd64" + instruction := file.Funcs[0].Instrs[1] + instruction.Op = "MULSS" + before := output.Len() + if known, term, err := context.lowerFloatingCompare(Op(opcode), instruction); !known || term || err == nil || output.Len() != before { + t.Fatalf("conflicting compare identity emitted IR: %s/%t/%t/%v", opcode, known, term, err) + } + } +} diff --git a/amd64_float_compare_fp_runtime_test.go b/amd64_float_compare_fp_runtime_test.go new file mode 100644 index 000000000..a81a5d3bd --- /dev/null +++ b/amd64_float_compare_fp_runtime_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" +) + +// Only fixture layout uses the ordinary Go ABI derivation. Neither numerical +// truth nor MXCSR expectations use the production compare implementation. +type x86FloatCompareFPOracle struct { + name, declaration, row string + bits, predicate int + mask bool + axis x86FloatCompareRuntimeISA + sig FuncSig +} + +func x86FloatCompareFPOracles(t *testing.T) []x86FloatCompareFPOracle { + t.Helper() + var fixtures []x86FloatCompareFPOracle + var declarations strings.Builder + declarations.WriteString("package oracle\n") + for _, bits := range []int{32, 64} { + for _, spelling := range []string{"legacy", "vex", "evex"} { + for predicate := 0; predicate < 8; predicate++ { + fixture := x86FloatCompareFPOracle{ + name: fmt.Sprintf("fp%d%s%d", bits, spelling, predicate), + bits: bits, predicate: predicate, mask: spelling == "evex", + axis: map[string]x86FloatCompareRuntimeISA{"legacy": x86FloatCompareSSE, "vex": x86FloatCompareVEX, "evex": x86FloatCompareEVEX}[spelling], + } + fixture.declaration = fmt.Sprintf("func %s(out *[20]byte,source [%d]byte,left uint%d)\n", fixture.name, bits/8, bits) + declarations.WriteString(fixture.declaration) + opcode := "CMPSS" + if bits == 64 { + opcode = "CMPSD" + } + fixture.row = fmt.Sprintf("%s source+8(FP),X0,$%d", opcode, predicate) + if spelling == "vex" { + fixture.row = fmt.Sprintf("V%s $%d,source+8(FP),X0,X2", opcode, predicate) + } else if spelling == "evex" { + fixture.row = fmt.Sprintf("V%s $%d,source+8(FP),X0,K1,K2", opcode, predicate) + } + fixtures = append(fixtures, fixture) + } + } + } + set := token.NewFileSet() + file, err := parser.ParseFile(set, "oracle.go", declarations.String(), 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + pkg, err := (&types.Config{Sizes: sizes}).Check("oracle", set, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + for index := range fixtures { + fixture := &fixtures[index] + fixture.sig, err = DeriveGoFuncABI0Signature(fixture.name, pkg.Scope().Lookup(fixture.name).(*types.Func), "amd64", sizes) + if err != nil { + t.Fatal(err) + } + if len(fixture.sig.Frame.Params) != fixture.bits/8+2 { + t.Fatalf("Go array did not derive independent byte fields: %+v", fixture.sig.Frame) + } + } + return fixtures +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareFPScalarByteSpans(t *testing.T) { + x86FloatCompareFPScalarByteSpansRuntime(t, x86FloatCompareSSE) +} + +func x86FloatCompareFPAssembly(fixtures []x86FloatCompareFPOracle) (string, map[string]FuncSig) { + var assembly strings.Builder + sigs := map[string]FuncSig{} + for _, fixture := range fixtures { + move, destination := "MOVSS", "X0" + if fixture.bits == 64 { + move = "MOVSD" + } + if fixture.axis == x86FloatCompareVEX { + destination = "X2" + } + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-%d\nMOVQ out+0(FP),AX\n%s left+%d(FP),X0\nMOVQ $1,DX\nKMOVQ DX,K1\n%s\n", + fixture.name, 8+2*(fixture.bits/8), move, 8+fixture.bits/8, fixture.row) + if fixture.mask { + assembly.WriteString("KMOVQ K2,(AX)\n") + } else { + fmt.Fprintf(&assembly, "MOVOU %s,(AX)\n", destination) + } + assembly.WriteString("STMXCSR 16(AX)\nRET\n") + sigs[fixture.name] = fixture.sig + } + return assembly.String(), sigs +} + +func x86FloatCompareFPScalarByteSpansRuntime(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + var fixtures []x86FloatCompareFPOracle + for _, fixture := range x86FloatCompareFPOracles(t) { + if fixture.axis == axis { + fixtures = append(fixtures, fixture) + } + } + if len(fixtures) != 16 { + t.Fatal("incomplete scalar FP-byte-field execution axis", axis, len(fixtures)) + } + assembly, sigs := x86FloatCompareFPAssembly(fixtures) + var wrappers, declarations, checks strings.Builder + for index, fixture := range fixtures { + + // A C bit carrier is converted into the precise LLVM array value. C's + // aggregate calling convention is not assumed to match Go's signature. + fmt.Fprintf(&wrappers, "define void @check_%s(ptr %%out,i64 %%right,i64 %%left){\n", fixture.name) + packed := "poison" + for field := 0; field < fixture.bits/8; field++ { + fmt.Fprintf(&wrappers, " %%shift%d=lshr i64 %%right,%d\n %%byte%d=trunc i64 %%shift%d to i8\n %%pack%d=insertvalue [%d x i8] %s,i8 %%byte%d,%d\n", + field, field*8, field, field, field, fixture.bits/8, packed, field, field) + packed = fmt.Sprintf("%%pack%d", field) + } + left := "%left" + if fixture.bits == 32 { + wrappers.WriteString(" %left32=trunc i64 %left to i32\n") + left = "%left32" + } + fmt.Fprintf(&wrappers, " call void @%s(ptr %%out,[%d x i8] %s,i%d %s)\n ret void\n}\n", fixture.name, fixture.bits/8, packed, fixture.bits, left) + fmt.Fprintf(&declarations, "extern void check_%s(void*,uint64_t,uint64_t);\n", fixture.name) + fmt.Fprintf(&checks, "if(check(check_%s,%d,%d,%d))return %d;\n", fixture.name, fixture.bits, fixture.predicate, + x86FloatAddSubBoolInt(fixture.mask), index+1) + } + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + const check = ` +typedef void(*compare_fp)(void*,uint64_t,uint64_t); +static int check(compare_fp call,unsigned bits,unsigned predicate,unsigned mask_result){ + const uint64_t *edges=bits==32?edges32:edges64; + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)); + for(unsigned i=0;i<18;i++)for(unsigned j=0;j<18;j++)for(unsigned c=0;c<6;c++){ + uint8_t output[22],expected[20];memset(output,0xc3,sizeof(output));memset(expected,0,20); + unsigned bytes=bits/8,csr=controls[c]; + int truth=independent_compare(bits,edges[i],edges[j],predicate,csr); + if(mask_result){memset(expected+8,0xc3,8);expected[0]=(uint8_t)truth;} + else memset(expected,truth?255:0,bytes); + unsigned effect=original_effect(bits,edges[i],edges[j],predicate,csr); + memcpy(expected+16,&effect,4); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + call(output+1,edges[j],edges[i]); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + if(output[0]!=0xc3||output[21]!=0xc3||memcmp(output+1,expected,20)){ + fprintf(stderr,"FP compare bits=%u pred=%u i=%u j=%u csr=%x mask=%u observed=%llx wanted=%llx\n", + bits,predicate,i,j,csr,mask_result,(unsigned long long)read_bits(output+1,8),(unsigned long long)read_bits(expected,8)); + return 1; + } + } + return 0; +} +` + main := x86FloatCompareIndependentC + declarations.String() + check + "int main(void){\n" + checks.String() + "return 0;}\n" + execution.compileAndRunLLVM(t, "compare-fp-scalar-bytes", ir+wrappers.String(), main, "-O2") + t.Logf("%d scalar compare programs ISA-axis=%d, independently allocated 4/8 FP byte fields ×1944 inputs; actual transport=%v", len(fixtures), axis, execution.prefix) +} diff --git a/amd64_float_compare_immediate_class_test.go b/amd64_float_compare_immediate_class_test.go new file mode 100644 index 000000000..fe6a5a9dc --- /dev/null +++ b/amd64_float_compare_immediate_class_test.go @@ -0,0 +1,35 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86FloatingCompareIntegerImmediateClassBeforeEmission(t *testing.T) { + for _, opcode := range x86FloatCompareOpcodes() { + row, index := opcode+" X0,X1,$0.0", 2 + if strings.HasPrefix(opcode, "V") { + row, index = opcode+" $0.0,X0,X1,X2", 0 + } + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, unresolved := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/unresolved=%t", opcode, unresolved), func(t *testing.T) { + ins := file.Funcs[0].Instrs[1] + ins.Args = append([]Operand(nil), ins.Args...) + if unresolved { + ins.Args[index] = Operand{Kind: OpImm, ImmRaw: "$UNRESOLVED_PREDICATE"} + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + if known, term, err := ctx.lowerFloatingCompare(ins.Op, ins); !known || term || err == nil || output.Len() != before { + t.Fatalf("floating/symbolic predicate acquired integer comparison or emitted IR: %t/%t/%v", known, term, err) + } + }) + } + } +} diff --git a/amd64_float_compare_isa_audit_test.go b/amd64_float_compare_isa_audit_test.go new file mode 100644 index 000000000..fd67bad84 --- /dev/null +++ b/amd64_float_compare_isa_audit_test.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +// Observation only: no ISA permission or substitute numerical verdict. +func TestX86FloatingCompareActualExecutionISAAudit(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + directory := t.TempDir() + source := filepath.Join(directory, "isa.c") + executable := filepath.Join(directory, "isa") + const program = `#include +#include +#include +int main(void) { + unsigned a, b, c, d; + uint64_t xcr0 = 0; + unsigned osxsave = 0, raw_avx = 0, raw_f = 0, raw_vl = 0; + if (__get_cpuid(1, &a, &b, &c, &d)) { + osxsave = (c >> 27) & 1; + raw_avx = (c >> 28) & 1; + if (osxsave) { + unsigned low, high; + __asm__ volatile("xgetbv" : "=a"(low), "=d"(high) : "c"(0)); + xcr0 = ((uint64_t)high << 32) | low; + } + } + if (__get_cpuid_count(7, 0, &a, &b, &c, &d)) { + raw_f = (b >> 16) & 1; + raw_vl = (b >> 31) & 1; + } + __builtin_cpu_init(); + printf("SSE2=%d AVX=%d AVX2=%d AVX512F=%d AVX512VL=%d " + "rawAVX=%u rawAVX512F=%u rawAVX512VL=%u OSXSAVE=%u XCR0=%llx\n", + !!__builtin_cpu_supports("sse2"), !!__builtin_cpu_supports("avx"), + !!__builtin_cpu_supports("avx2"), !!__builtin_cpu_supports("avx512f"), + !!__builtin_cpu_supports("avx512vl"), raw_avx, raw_f, raw_vl, + osxsave, (unsigned long long)xcr0); + return 0; +} +` + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + output := execution.run(t, executable) + t.Logf("actual execution host=%s/%s triple=%s prefix=%q: %s", + runtime.GOOS, runtime.GOARCH, execution.triple, execution.prefix, output) +} diff --git a/amd64_float_compare_native.go b/amd64_float_compare_native.go new file mode 100644 index 000000000..76bd83474 --- /dev/null +++ b/amd64_float_compare_native.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// A relation-only fcmp (including constrained fcmp/fcmps) is not the source +// CMP immediate's MXCSR contract. Keep a single sideeffecting native instruction: +// its complete vector, immediate, active mask and SAE decide IA/DE and traps. +// In particular, a per-lane loop changes packed exception priority. +func (c *amd64Ctx) emitNativeFloatingCompare(ins Instr, f x86FloatCompareForm) error { + first := ins.Args[f.first] + if first.Kind == OpFP && !f.scalar { + return fmt.Errorf("%w: %s packed FP comparison needs original contiguous native storage", ErrProbeNeedsContext, ins.Op) + } + if !f.scalar && first.Kind == OpMem && !x86OrdinaryNativeMemoryOperand(first, c.goarch) && !c.fixedLocalMemoryCertified(ins, f.first) { + // A word-aligned virtual source stack is neither the original native + // address nor an m128 alignment witness. Do not strengthen its alloca + // alignment or synthesize a native shadow to suppress source faults. + return fmt.Errorf("%w: %s packed source stack lacks original native backing", ErrProbeNeedsContext, ins.Op) + } + if ins.x86AddressBits != 0 { + return fmt.Errorf("%w: %s native compare needs its original address-size binding", ErrProbeNeedsContext, ins.Op) + } + if first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, f.bits/8); err != nil { + return err + } + } + + mask := "" + if f.mask >= 0 { + value, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return err + } + mask = value + } + typ := amd64FloatingVectorType(f.byteWidth*8/f.bits, f.bits) + second, err := c.loadFloatingVectorOperand(ins.Args[f.second], f.byteWidth, f.byteWidth*8/f.bits, f.bits) + if err != nil { + return err + } + firstValue, firstType, firstConstraint, firstTemplate, err := c.nativeFloatingCompareSource(first, f, typ, mask) + if err != nil { + return err + } + if !f.maskResult && firstConstraint == "v" { + // A mixed function can also contain EVEX and therefore enable AVX512. + // Legacy/VEX still cannot encode registers 16-31 in this operand. + firstConstraint = "x" + } + mnemonic := strings.ToLower(strings.Split(string(ins.Op), ".")[0]) + immediate := f.immediate + result := c.newTmp() + clobbers := "~{memory},~{fpsr}" + + if f.maskResult { + // The K result is private physical scratch. Inputs/outputs remain + // virtual; real k1/k2 do not become public caller transports. + assembly := "" + constraints := "=r," + firstConstraint + ",v" + args := fmt.Sprintf("%s %s, %s %s", firstType, firstValue, typ, second) + if mask != "" { + assembly = "kmovw ${3:k}, %k2; " + constraints += ",r,~{k2}" + args += ", i32 " + c.approximateMathMask32(mask) + } + assembly += fmt.Sprintf("%s $$%d, ", mnemonic, immediate) + if f.sae { + assembly += "{sae}, " + } + if f.broadcast { + firstTemplate += fmt.Sprintf(" {1to%d}", f.byteWidth*8/f.bits) + } + assembly += firstTemplate + ", $2, %k1" + if mask != "" { + assembly += " {%k2}" + } + assembly += "; kmovw %k1, ${0:k}" + fmt.Fprintf(c.b, " %%%s = call i32 asm sideeffect %q, %q(%s)\n", result, assembly, constraints+",~{k1},"+clobbers, args) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, result) + return c.storeK(ins.Args[f.destination].Reg, "%"+wide) + } + + assembly := fmt.Sprintf("%s $$%d, ", mnemonic, immediate) + constraints := "=x," + args := "" + if f.legacy { + // Tie the full old X value: scalar compares preserve upper X bytes. + assembly += strings.Replace(firstTemplate, "$1", "$2", 1) + ", $0" + constraints += "0," + firstConstraint + args = fmt.Sprintf("%s %s, %s %s", typ, second, firstType, firstValue) + } else { + assembly += firstTemplate + ", $2, $0" + constraints += firstConstraint + ",x" + args = fmt.Sprintf("%s %s, %s %s", firstType, firstValue, typ, second) + } + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", result, typ, assembly, constraints+","+clobbers, args) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, typ, result, f.byteWidth) + if f.legacy { + return c.storeLegacyXViews(ins.Args[f.destination].Reg, "%"+bytes) + } + return c.storePackedMoveOperand(ins.Args[f.destination], f.byteWidth, "%"+bytes) +} + +func (c *amd64Ctx) nativeFloatingCompareSource(source Operand, f x86FloatCompareForm, typ, mask string) (value, sourceType, constraint, template string, err error) { + if source.Kind == OpFP { + // Scalar compare observes only the low field. Assemble its checked + // bytes as integer data, not a fabricated contiguous native FP pointer. + bits, loadErr := c.loadFloatBinaryScalar(source, f.bits, mask) + if loadErr != nil { + return "", "", "", "", loadErr + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s zeroinitializer, %s %s, i32 0\n", inserted, typ, amd64FloatingScalarType(f.bits), bits) + return "%" + inserted, typ, "v", "$1", nil + } + if source.Kind == OpReg { + value, err = c.loadFloatingVectorOperand(source, f.byteWidth, f.byteWidth*8/f.bits, f.bits) + return value, typ, "v", "$1", err + } + if source.Kind != OpMem && source.Kind != OpSym { + return "", "", "", "", fmt.Errorf("%w: native compare source lacks original memory binding", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", "", "", "", err + } + segment := "" + if source.Kind == OpMem && source.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(source.Mem.Segment)) + ":" + } + return pointer, fmt.Sprintf("%s elementtype([%d x i8])", pointerType, f.memoryBytes), "*m", segment + "$1", nil +} diff --git a/amd64_float_compare_native_test.go b/amd64_float_compare_native_test.go new file mode 100644 index 000000000..cbe54908f --- /dev/null +++ b/amd64_float_compare_native_test.go @@ -0,0 +1,329 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +func TestX86FloatCompareNativeOriginalInstruction(t *testing.T) { + for _, row := range []string{ + "CMPPS X0,X1,$3", "CMPSD 3(AX),X1,$1", + "VCMPPS $11,Y0,Y1,Y2", "VCMPPD $15,Z0,Z1,K3,K2", + "VCMPPS.BCST $31,3(AX),Z1,K3,K2", "VCMPSS.SAE $31,X0,X1,K3,K2", + } { + t.Run(row, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": {Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + opcode := strings.ToLower(strings.Split(row, " ")[0]) + opcode = strings.Split(opcode, ".")[0] + if !strings.Contains(ir, "asm sideeffect") || !strings.Contains(ir, opcode+" $$") || strings.Contains(ir, " = fcmp ") { + t.Fatalf("original whole instruction/CSR effect missing:\n%s", ir) + } + }) + } +} + +func TestX86FloatCompareNativeAllPredicatesObjectsAndDeadO2(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + var source strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, opcode := range x86FloatCompareOpcodes() { + for _, row := range x86FloatCompareOracleCases(opcode) { + name := fmt.Sprintf("native_compare%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row.row) + sigs[name] = FuncSig{Name: name, Ret: Void} + count++ + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + for _, triple := range []string{"x86_64-apple-macosx", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if strings.Count(ir, "asm sideeffect") != count { + t.Fatalf("whole-instruction sideeffects=%d want=%d", strings.Count(ir, "asm sideeffect"), count) + } + compileLLVMToObject(t, llc, triple, "native-compare.ll", "native-compare.o", ir) + path := filepath.Join(t.TempDir(), "dead.ll") + if err := os.WriteFile(path, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=asm", path, "-o", "-") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("dead native effect object: %v\n%s", err, output) + } + // LLVM's assembly printer canonicalizes CMP-immediate mnemonics to + // predicate aliases (cmpeqps, vcmptrue_uspd, ...). Count those actual + // instructions, not the source spelling, after optimizing dead results. + compares := regexp.MustCompile(`(?m)^\s+v?cmp[a-z0-9_]*(ps|pd|ss|sd)\s+`).FindAll(output, -1) + if len(compares) != count { + t.Fatalf("dead O2 retained %d native comparisons, want %d", len(compares), count) + } + } + t.Logf("%d independent 8/32-predicate forms: 3 objects plus dead O2 native emission", count) +} + +func TestX86FloatCompareNativePhysicalMemoryAndPrefixContract(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + sigs := map[string]FuncSig{} + for index, row := range []string{ + "CMPPS 16(AX)(CX*4),X1,$127", "CMPSS 3(FS),X1,$-128", + "CMPPD 16(GS),X1,$-1", "CMPSD 256,X1,$7", + } { + name := fmt.Sprintf("memory%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + sigs[name] = FuncSig{Ret: Void} + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + triples := []string{"x86_64-apple-macosx", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"%fs:$2", "%gs:$2", "elementtype([16 x i8])", "elementtype([4 x i8])", "elementtype([8 x i8])", "$$127", "$$128", "$$255"} { + if !strings.Contains(ir, want) { + t.Fatalf("%s original memory/predicate contract lost %s:\n%s", arch, want, ir) + } + } + compileLLVMToObject(t, llc, triple, "native-memory.ll", "native-memory.o", ir) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nCMPPS (AX),X1,$0\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + ins := file.Funcs[0].Instrs[1] + ins.x86AddressBits = 32 + before := output.Len() + if _, _, err := ctx.lowerFloatingCompare(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("unbound native address prefix emitted compare IR: %v", err) + } +} + +func TestX86FloatCompareNativeRawMode32HighBanksRemainContext(t *testing.T) { + for _, code := range [][]byte{ + encodeX86VEXVectorFloatCompare(0, false, 8, 1, 0, 0), + encodeX86VEXVectorFloatCompare(1, true, 2, 8, 0, 0), + encodeX86VEXVectorFloatCompare(2, false, 2, 1, 8, 0), + encodeX86EVEXVectorFloatCompare(0, 2, false, 3, 2, 8, 0, 0), + encodeX86EVEXVectorFloatCompare(1, 0, false, 3, 2, 1, 8, 0), + encodeX86EVEXVectorFloatCompare(2, 0, false, 3, 2, 1, 16, 0), + } { + if _, _, known, err := decodedX86VectorFloatCompareInstruction(code, 32); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mode32 high-bank %x gained a native binding: %v", code, err) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nVCMPPS $0,X0,X1,X2\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", Sigs: map[string]FuncSig{"f": {Ret: Void}}}); err == nil { + t.Fatal("named mode32 source impersonated private raw decoding") + } +} + +func TestX86FloatCompareNativeRawMode32PhysicalForms(t *testing.T) { + mc, llc := findLLVM22Tool("llvm-mc"), findLLVM22Tool("llc") + if mc == "" || llc == "" { + t.Fatal("LLVM22 mc/llc required") + } + var source strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for pp := byte(0); pp < 4; pp++ { + widths := []byte{0, 1, 2} + if pp >= 2 { + widths = widths[:1] + } + for _, width := range widths { + for predicate := byte(0); predicate < 32; predicate++ { + var codes [][]byte + if width != 2 { + codes = append(codes, encodeX86VEXVectorFloatCompare(pp, width == 1, 2, 1, 0, predicate)) + } + codes = append(codes, encodeX86EVEXVectorFloatCompare(pp, width, false, 3, 2, 1, 0, predicate)) + if pp >= 2 || width == 2 { + codes = append(codes, encodeX86EVEXVectorFloatCompare(pp, width, true, 3, 2, 1, 0, predicate)) + } + for _, code := range codes { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=i386-unknown-linux-gnu", "--mattr=+avx,+avx512f,+avx512vl") + var hex strings.Builder + for _, b := range code { + fmt.Fprintf(&hex, "0x%02x ", b) + } + command.Stdin = strings.NewReader(hex.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || strings.Contains(string(output), "warning:") || !strings.Contains(string(output), "vcmp") { + t.Fatalf("physical i386 compare %x: %v\n%s", code, err, output) + } + name := fmt.Sprintf("raw_native%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", name) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%d\n", b) + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Ret: Void} + count++ + } + } + } + } + goAssemblerOracleOutput(t, "386", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-native-compare.ll", "raw-native-compare.o", ir) + } + t.Logf("%d physical low-bank i386 forms independently decoded by LLVM22; 2 objects", count) +} + +func TestX86FloatCompareNativePackedVirtualFPNeedsContext(t *testing.T) { + for _, row := range []string{ + "CMPPS a+0(FP),X1,$0", "VCMPPD $3,a+0(FP),Y1,Y2", "VCMPPS.BCST $3,a+0(FP),Z1,K2", + "CMPPS 16(SP),X1,$0", "CMPPD 16(BP),X1,$0", "VCMPPS $0,16(SP),Z1,K2", "VCMPPD $0,(AX)(BP*4),Y1,Y2", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": {Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s fabricated native backing: %v", row, err) + } + } +} + +func TestX86FloatCompareNativeScalarFPIndependentByteFields(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, opcode := range []string{"CMPSS", "CMPSD", "VCMPSS", "VCMPSD"} { + width := 4 + if strings.HasSuffix(opcode, "SD") { + width = 8 + } + row := opcode + " a+0(FP),X1,$3" + if strings.HasPrefix(opcode, "V") { + row = opcode + " $3,a+0(FP),X1,K3,K2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-8\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + sig := FuncSig{Ret: Void} + for index := 0; index < width; index++ { + sig.Args = append(sig.Args, I8) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(index), Type: I8, Index: index, Field: -1}) + } + ir, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": sig}}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "elementtype(") || !strings.Contains(ir, "asm sideeffect") || strings.Count(ir, "load i8, ptr %fp_") != width { + t.Fatalf("%s invented contiguous native FP backing or omitted field bytes:\n%s", row, ir) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "scalar-fp.ll", "scalar-fp.o", ir) + // Even an inactive EVEX mask cannot bless an invalid source span. + sig.Frame.Params = sig.Frame.Params[:width-1] + if _, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": sig}}); err == nil { + t.Fatalf("%s missing final FP byte accepted", row) + } + } +} + +func TestX86FloatCompareNativeMixedISAUsesLowVEXConstraints(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nCMPSS X0,X1,$3\nVCMPPS $3,Y0,Y1,Y2\nVCMPPS $3,Z0,Z1,K2\nRET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": {Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `"=x,0,x,`) || !strings.Contains(ir, `"=x,x,x,`) || !strings.Contains(ir, `"=r,v,v,`) { + t.Fatalf("mixed EVEX function widened a legacy/VEX physical register class:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "mixed-isa.ll", "mixed-isa.o", ir) +} + +func TestX86FloatCompareNativeValidatedPrivateFeatures(t *testing.T) { + for _, tc := range []struct{ row, want string }{ + {"CMPPS X0,X1,$0", "+sse"}, + {"CMPSD X0,X1,$0", "+sse2"}, + {"VCMPPS $0,Y0,Y1,Y2", "+avx"}, + {"VCMPPS $0,X0,X1,K2", "+avx512f,+avx512vl"}, + {"VCMPSS $0,X0,X1,K2", "+avx512f"}, + {"VCMPPS $0,Z0,Z1,K2", "+avx512f"}, + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+tc.row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", file.Funcs[0]); got != tc.want { + t.Errorf("%s features=%q want=%q", tc.row, got, tc.want) + } + } + for _, row := range []string{"VCMPPS $256,Z0,Z1,K2", "VCMPPS.Z $0,Z0,Z1,K2", "VCMPSS $0,X0,Y1,K2"} { + file, err := Parse(ArchAMD64, fmt.Sprintf("TEXT f(SB),4,$0-0\n%s\nRET\n", row)) + if err != nil { + t.Fatal(err) + } + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", file.Funcs[0]); got != "" { + t.Errorf("malformed source gained ISA permission: %s/%s", row, got) + } + } +} diff --git a/amd64_float_compare_numerical_test.go b/amd64_float_compare_numerical_test.go new file mode 100644 index 000000000..4bd3cf317 --- /dev/null +++ b/amd64_float_compare_numerical_test.go @@ -0,0 +1,257 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86FloatCompareOracleCase struct { + row string + bits, width, predicate int + scalar, legacy, maskResult bool + memory, broadcast, suppressed bool + destination int +} + +// Enumerate this matrix from explicit architectural axes, not the production +// grammar, predicate table, memory-shape classifier or caller effect registry. +func x86FloatCompareOracleCases(opcode string) []x86FloatCompareOracleCase { + bits := 32 + if strings.HasSuffix(opcode, "PD") || strings.HasSuffix(opcode, "SD") { + bits = 64 + } + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + legacy := !strings.HasPrefix(opcode, "V") + predicates := 32 + if legacy { + predicates = 8 + } + var cases []x86FloatCompareOracleCase + for predicate := 0; predicate < predicates; predicate++ { + for variant := 0; variant < 2; variant++ { + width := 16 + if !legacy && !scalar && predicate%2 != 0 { + width = 32 + } + bank := map[int]string{16: "X", 32: "Y"}[width] + first, destination := bank+"1", predicate%3 + if variant == 1 { + first = "(CX)" + } + row := fmt.Sprintf("%s $%d,%s,%s0,%s%d", opcode, predicate, first, bank, bank, destination) + if legacy { + destination = 0 + row = fmt.Sprintf("%s %s,X0,$%d", opcode, first, predicate) + } + cases = append(cases, x86FloatCompareOracleCase{row: row, bits: bits, width: width, + predicate: predicate, scalar: scalar, legacy: legacy, memory: variant == 1, destination: destination}) + } + if legacy { + continue + } + for variant := 0; variant < 4; variant++ { + width := []int{16, 32, 64}[predicate%3] + if scalar { + width = 16 + } + if variant == 3 && !scalar { + width = 64 // Go only permits packed SAE at Z width. + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + first, operation := bank+"1", opcode + memory, broadcast := variant == 1 || variant == 2, variant == 2 && !scalar + if memory { + first = "(CX)" + } + if broadcast { + operation += ".BCST" + } + if variant == 3 { + operation += ".SAE" + } + row := fmt.Sprintf("%s $%d,%s,%s0,K1,K2", operation, predicate, first, bank) + cases = append(cases, x86FloatCompareOracleCase{row: row, bits: bits, width: width, + predicate: predicate, scalar: scalar, maskResult: true, memory: memory, + broadcast: broadcast, suppressed: variant == 3}) + } + } + return cases +} + +func x86FloatCompareOracleAssembly(cases []x86FloatCompareOracleCase, dead bool) (string, map[string]FuncSig) { + var source strings.Builder + sigs := map[string]FuncSig{} + for index, c := range cases { + name := fmt.Sprintf("compare%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (BX),Z0\nVMOVUPS (BX),Z2\n", name) + if !c.memory { + source.WriteString("VMOVUPS (CX),Z1\n") + } + source.WriteString("KMOVQ DX,K1\nKMOVQ DX,K2\nSTC\n" + c.row + "\n") + if !dead { + if c.maskResult { + source.WriteString("KMOVQ K2,8(AX)\n") + } else { + fmt.Fprintf(&source, "VMOVUPS Z%d,8(AX)\n", c.destination) + } + } + source.WriteString("SETCS 72(AX)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + } + return source.String(), sigs +} + +func x86FloatCompareOracleMain(cases []x86FloatCompareOracleCase, dead bool) string { + var declarations, descriptors strings.Builder + for index, c := range cases { + fmt.Fprintf(&declarations, "extern void compare%d(void*,const void*,const void*,uint64_t);\n", index) + fmt.Fprintf(&descriptors, "{compare%d,%d,%d,%d,%d,%d,%d,%d,%d,%d},\n", index, c.bits, c.width, c.predicate, + x86FloatAddSubBoolInt(c.scalar), x86FloatAddSubBoolInt(c.legacy), x86FloatAddSubBoolInt(c.maskResult), + x86FloatAddSubBoolInt(c.memory), x86FloatAddSubBoolInt(c.broadcast), x86FloatAddSubBoolInt(c.suppressed)) + } + return x86FloatCompareIndependentC + declarations.String() + ` +typedef void (*compare_fn)(void*,const void*,const void*,uint64_t); +struct comparison {compare_fn call;unsigned bits,width,predicate,scalar,legacy,mask_result,memory,broadcast,sae;}; +static const struct comparison comparisons[]={ +` + descriptors.String() + fmt.Sprintf(` +}; +int main(void){ + const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX}; + unsigned saved;__asm__ volatile("stmxcsr %%0":"=m"(saved)); + for(unsigned index=0;indexbits/8; + const uint64_t *edges=c->bits==32?edges32:edges64; + for(unsigned pair=0;pair<18;pair++) for(unsigned m=0;m<4;m++) for(unsigned control=0;control<6;control++){ + _Alignas(64) uint8_t aStorage[192],bStorage[192]; + uint8_t *a=aStorage+64,*b=bStorage+64,out[81],want[64]; + memset(aStorage,0xa7,sizeof(aStorage));memset(bStorage,0xb9,sizeof(bStorage));memset(out,0xc3,sizeof(out)); + for(unsigned lane=0;lane<64/bytes;lane++){ + uint64_t av=edges[(pair+lane)%%18],bv=edges[(pair*7+lane*3+1)%%18]; + // Extended predicates have no actual AVX512 execution oracle here. Keep + // denormal and NaN coverage, but do not mix them in a single extended + // comparison: the diagnostic SSE class representative is not a proof of + // that exception-priority corner (the original legacy matrix retains it). + if(c->predicate>=8 && ((is_denormal(c->bits,av)&&is_nan(c->bits,bv)) || + (is_nan(c->bits,av)&&is_denormal(c->bits,bv)))) { + if(is_denormal(c->bits,av))av=0; + if(is_denormal(c->bits,bv))bv=0; + } + memcpy(a+lane*bytes,&av,bytes);memcpy(b+lane*bytes,&bv,bytes); + } + uint8_t beforeA[192],beforeB[192];memcpy(beforeA,aStorage,192);memcpy(beforeB,bStorage,192); + unsigned csr=controls[control],after,expectedCSR=csr;uint64_t expectedMask=0; + memset(want,0,64); + if(c->legacy)memcpy(want,a,64);else if(c->scalar)memcpy(want,a,16); + unsigned lanes=c->scalar?1:c->width/bytes; + for(unsigned lane=0;lanemask_result || ((masks[m]>>lane)&1); + if(!active)continue; + uint64_t av=read_bits(a+lane*bytes,bytes),bv=read_bits(b+(c->broadcast?0:lane*bytes),bytes); + int truth=independent_compare(c->bits,av,bv,c->predicate,csr); + if(truth)expectedMask|=UINT64_C(1)<sae)expectedCSR|=original_effect(c->bits,av,bv,c->predicate,csr)&63; + } + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + const uint8_t *input=c->mask_result&&c->memory&&m==0?0:b; + c->call(out,a,input,masks[m]); + __asm__ volatile("stmxcsr %%0":"=m"(after)); + __asm__ volatile("ldmxcsr %%0"::"m"(saved)); + if(after!=expectedCSR){fprintf(stderr,"compare%%u pred=%%u mxcsr=%%x want=%%x mask=%%llx\n",index,c->predicate,after,expectedCSR,(unsigned long long)masks[m]);return 10;} + if(out[72]!=1||memcmp(aStorage,beforeA,192)||memcmp(bStorage,beforeB,192))return 11; + for(unsigned byte=0;byte<81;byte++){ + uint8_t expected=0xc3; + if(byte==72)expected=1; + if(!%d&&byte>=8&&byte<72){ + if(c->mask_result){if(byte<16)expected=(uint8_t)(expectedMask>>(8*(byte-8)));} + else expected=want[byte-8]; + } + if(out[byte]!=expected){fprintf(stderr,"compare%%u pred=%%u byte=%%u got=%%x want=%%x\n",index,c->predicate,byte,out[byte],expected);return 12;} + } + } + } + __asm__ volatile("ldmxcsr %%0"::"m"(saved));return 0; +} +`, x86FloatAddSubBoolInt(dead)) +} + +// Keep optimized-away results separate: a compare still has MXCSR/trap and +// memory effects. Both optimization levels execute the same independent C. +func x86FloatCompareRunOptimization(t *testing.T, execution x86FloatCompareExecution, ir, main, level string) { + t.Helper() + execution.compileAndRunLLVM(t, "compare-effects", ir, main, level) +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareIndependentIEEEAndMXCSR(t *testing.T) { + x86FloatCompareIndependentRuntime(t, x86FloatCompareSSE) +} + +func x86FloatCompareIndependentRuntime(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + cases := x86FloatCompareRuntimeCases(opcode, axis) + if len(cases) == 0 { + continue + } + t.Run(opcode, func(t *testing.T) { + assembly, sigs := x86FloatCompareOracleAssembly(cases, false) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86FloatCompareOracleMain(cases, false) + execution.compileAndRunLLVM(t, "compare-ieee", ir, main, "-O2") + t.Logf("%s ISA-axis=%d: %d independently enumerated programs ×432 IEEE/DAZ/full-MXCSR/mask/upper-alias/canary inputs; extended NaN+denormal priority is not covered; actual transport=%v", opcode, axis, len(cases), execution.prefix) + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareDeadResultEffects(t *testing.T) { + x86FloatCompareDeadResultRuntime(t, x86FloatCompareSSE) +} + +func x86FloatCompareDeadPredicate(predicate int) bool { + return predicate == 0 || predicate == 1 || predicate == 11 || predicate == 15 || predicate == 27 || predicate == 31 +} + +func x86FloatCompareDeadResultRuntime(t *testing.T, axis x86FloatCompareRuntimeISA) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, axis) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + var cases []x86FloatCompareOracleCase + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + for _, c := range x86FloatCompareRuntimeCases(opcode, axis) { + if x86FloatCompareDeadPredicate(c.predicate) { + cases = append(cases, c) + } + } + } + assembly, sigs := x86FloatCompareOracleAssembly(cases, true) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + x86FloatCompareRunOptimization(t, execution, ir, x86FloatCompareOracleMain(cases, true), level) + t.Logf("dead-result ISA-axis=%d: %d complete programs ×432 inputs, %s, actual transport=%v", axis, len(cases), level, execution.prefix) + }) + } +} diff --git a/amd64_float_compare_oracle_test.go b/amd64_float_compare_oracle_test.go new file mode 100644 index 000000000..b70309cee --- /dev/null +++ b/amd64_float_compare_oracle_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +// Intel SDM Vol.2A Table 3-1 supplies the relation truth columns and signaling +// column. The relation reference uses only integer IEEE encodings; neither it +// nor the native original scalar CMP exception oracle calls a production helper. +// COMIS/UCOMIS and quiet imm0 are not substituted for legacy predicates: their +// observed exception priority differs for NaN/denormal pairs under Rosetta. +const x86FloatCompareIndependentC = ` +#include +#include +#include +static const unsigned relation_truth[32]={ + 4,2,6,8,11,13,9,7,12,10,14,0,3,5,1,15, + 4,2,6,8,11,13,9,7,12,10,14,0,3,5,1,15}; +static int is_nan(unsigned bits,uint64_t a) { + uint64_t fraction=bits==32?0x7fffff:UINT64_C(0xfffffffffffff); + uint64_t exponent=bits==32?0x7f800000:UINT64_C(0x7ff0000000000000); + return (a&exponent)==exponent && (a&fraction)!=0; +} +static uint64_t normalized(unsigned bits,uint64_t a,unsigned csr) { + uint64_t sign=bits==32?0x80000000:UINT64_C(0x8000000000000000); + uint64_t exponent=bits==32?0x7f800000:UINT64_C(0x7ff0000000000000); + if(bits==32) a&=UINT64_C(0xffffffff); + if((csr&64) && !(a&exponent)) a&=sign; + return a; +} +static int independent_compare(unsigned bits,uint64_t a,uint64_t b,unsigned pred,unsigned csr) { + a=normalized(bits,a,csr); b=normalized(bits,b,csr); + unsigned relation; + if(is_nan(bits,a)||is_nan(bits,b)) relation=8; + else { + uint64_t sign=bits==32?0x80000000:UINT64_C(0x8000000000000000); + uint64_t mask=bits==32?UINT64_C(0xffffffff):UINT64_MAX; + if(a==b || (!(a&~sign)&&!(b&~sign))) relation=4; + else { + uint64_t ak=(a&sign)?(~a&mask):(a^sign); + uint64_t bk=(b&sign)?(~b&mask):(b^sign); + relation=ak>(pred&31))&1); +#define CMP_EFFECT(op) do { \ + switch(native_pred) { \ + case 0: __asm__ volatile(op " $0,%1,%0":"+x"(x):"x"(y));break; \ + case 1: __asm__ volatile(op " $1,%1,%0":"+x"(x):"x"(y));break; \ + case 2: __asm__ volatile(op " $2,%1,%0":"+x"(x):"x"(y));break; \ + case 3: __asm__ volatile(op " $3,%1,%0":"+x"(x):"x"(y));break; \ + case 4: __asm__ volatile(op " $4,%1,%0":"+x"(x):"x"(y));break; \ + case 5: __asm__ volatile(op " $5,%1,%0":"+x"(x):"x"(y));break; \ + case 6: __asm__ volatile(op " $6,%1,%0":"+x"(x):"x"(y));break; \ + case 7: __asm__ volatile(op " $7,%1,%0":"+x"(x):"x"(y));break; \ + } \ +} while(0) + if(bits==32) { + uint32_t av=(uint32_t)a,bv=(uint32_t)b; float x,y; + memcpy(&x,&av,4); memcpy(&y,&bv,4); + CMP_EFFECT("cmpss"); + } else { + double x,y; memcpy(&x,&a,8); memcpy(&y,&b,8); + CMP_EFFECT("cmpsd"); + } +#undef CMP_EFFECT + __asm__ volatile("stmxcsr %0":"=m"(after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + return after; +} +static int is_denormal(unsigned bits,uint64_t a) { + uint64_t fraction=bits==32?0x7fffff:UINT64_C(0xfffffffffffff); + uint64_t exponent=bits==32?0x7f800000:UINT64_C(0x7ff0000000000000); + return !(a&exponent) && (a&fraction)!=0; +} +static const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x800000, + 0x3f800000,0xbf800000,0x3f800001,0x7f7fffff,0xff7fffff,0x7f800000, + 0xff800000,0x7fc12345,0xffc45678,0x7f812345,0xff845678,0x7fffffff}; +static const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1, + UINT64_C(0x8000000000000001),UINT64_C(0xfffffffffffff),UINT64_C(0x10000000000000), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000001), + UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff),UINT64_C(0x7ff0000000000000), + UINT64_C(0xfff0000000000000),UINT64_C(0x7ff8123456789abc),UINT64_C(0xfff8fedcba987654), + UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff0fedcba987654),UINT64_C(0x7fffffffffffffff)}; +static const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1fbf,0x7fc5}; +static uint64_t read_bits(const uint8_t *p,unsigned bytes) { + uint64_t value=0; memcpy(&value,p,bytes); return value; +} +` + +// An actual SSE instruction has a compile-time immediate. This switch covers +// all eight legacy predicates, independently of LLVM's predicate selection. +func x86FloatCompareOriginalC() string { + var source strings.Builder + source.WriteString(x86FloatCompareIndependentC) + for index, op := range []string{"cmpps", "cmppd", "cmpss", "cmpsd"} { + fmt.Fprintf(&source, "static void original%d(uint8_t *out,const uint8_t *a,const uint8_t *b,unsigned pred,unsigned csr){\n unsigned saved,after; __asm__ volatile(\"stmxcsr %%0\":\"=m\"(saved));\n __asm__ volatile(\"ldmxcsr %%0\"::\"m\"(csr));\n typedef uint8_t v16 __attribute__((vector_size(16))); v16 x,y; memcpy(&x,a,16); memcpy(&y,b,16);\n switch(pred){\n", index) + for pred := 0; pred < 8; pred++ { + fmt.Fprintf(&source, " case %d: __asm__ volatile(\"%s $%d,%%1,%%0\":\"+x\"(x):\"x\"(y)); break;\n", pred, op, pred) + } + source.WriteString(" }\n __asm__ volatile(\"stmxcsr %0\":\"=m\"(after));\n __asm__ volatile(\"ldmxcsr %0\"::\"m\"(saved)); memcpy(out,&x,16); memcpy(out+16,&after,4);\n}\n") + } + source.WriteString("typedef void (*original_fn)(uint8_t*,const uint8_t*,const uint8_t*,unsigned,unsigned);\n static original_fn originals[]={original0,original1,original2,original3};\n") + return source.String() +} + +func x86FloatCompareCaptureCommand(t *testing.T, command *exec.Cmd) []byte { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + // Keep one lifecycle/deadline per actual command, including the original-Go + // driver and the independently compiled native SSE reference. + bounded := exec.CommandContext(ctx, command.Path, command.Args[1:]...) + bounded.Dir, bounded.Env = command.Dir, command.Env + output, err := runtimeOracleCombinedOutput(ctx, bounded) + if err != nil { + t.Fatalf("compare oracle command %q: %v\n%s", command.Args, err, output) + } + return output +} + +func TestCrossLinuxRuntimeMatrixX86FloatingCompareOriginalLegacyPredicates(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + var assembly, declarations strings.Builder + assembly.WriteString("#include \"textflag.h\"\n") + for index, op := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD"} { + for pred := 0; pred < 8; pred++ { + name := fmt.Sprintf("legacy%d_%d", index, pred) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),NOSPLIT,$8-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nSTMXCSR 0(SP)\nLDMXCSR csr+24(FP)\nMOVOU (BX),X0\nMOVOU (CX),X1\n%s X1,X0,$%d\nMOVOU X0,(AX)\nSTMXCSR 16(AX)\nLDMXCSR 0(SP)\nRET\n", name, op, pred) + fmt.Fprintf(&declarations, "func %s(out,a,b *[20]byte,csr uint32)\n", name) + } + } + var functions strings.Builder + for index := 0; index < 4; index++ { + for pred := 0; pred < 8; pred++ { + fmt.Fprintf(&functions, "legacy%d_%d,", index, pred) + } + } + const goDriver = `package main +import("encoding/binary";"fmt") +func main(){ + functions:=[]func(*[20]byte,*[20]byte,*[20]byte,uint32){FUNCTIONS} + edges32:=[]uint64{0,0x80000000,1,0x80000001,0x7fffff,0x800000,0x3f800000,0xbf800000,0x3f800001,0x7f7fffff,0xff7fffff,0x7f800000,0xff800000,0x7fc12345,0xffc45678,0x7f812345,0xff845678,0x7fffffff} + edges64:=[]uint64{0,0x8000000000000000,1,0x8000000000000001,0xfffffffffffff,0x10000000000000,0x3ff0000000000000,0xbff0000000000000,0x3ff0000000000001,0x7fefffffffffffff,0xffefffffffffffff,0x7ff0000000000000,0xfff0000000000000,0x7ff8123456789abc,0xfff8fedcba987654,0x7ff0123456789abc,0xfff0fedcba987654,0x7fffffffffffffff} + controls:=[]uint32{0x1f80,0x1fc0,0x9f80,0x9fc0,0x1fbf,0x7fc5} + hash:=uint64(1469598103934665603) + for index,fn:=range functions{ + bytes,edges:=4,edges32 + if index/8==1||index/8==3{bytes,edges=8,edges64} + for _,av:=range edges{for _,bv:=range edges{for _,csr:=range controls{ + var a,b [20]byte + out:=struct{before byte;data [20]byte;after byte}{before:0xa7,after:0xb9} + for i:=0;i<16;i+=bytes{ + if bytes==4{binary.LittleEndian.PutUint32(a[i:],uint32(av));binary.LittleEndian.PutUint32(b[i:],uint32(bv))}else{binary.LittleEndian.PutUint64(a[i:],av);binary.LittleEndian.PutUint64(b[i:],bv)} + } + beforeA,beforeB:=a,b;fn(&out.data,&a,&b,csr) + if a!=beforeA||b!=beforeB||out.before!=0xa7||out.after!=0xb9{panic("input/output canary mutation")} + for _,v:=range out.data{hash^=uint64(v);hash*=1099511628211} + }}} + } + fmt.Printf("%016x\n",hash) +} +` + for name, source := range map[string]string{ + "go.mod": "module originalcompare\n\ngo 1.20\n", + "main.go": strings.Replace(goDriver, "FUNCTIONS", functions.String(), 1) + declarations.String(), + "original_amd64.s": assembly.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + goExecutable := filepath.Join(dir, gotoolchain.HostExecutable("original-go")) + command := exec.Command("go", "build", "-p=2", "-o", goExecutable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + x86FloatCompareCaptureCommand(t, command) + actualGo := execution.run(t, goExecutable) + const main = ` +int main(void){ + uint64_t hash=UINT64_C(1469598103934665603); + for(unsigned op=0;op<4;op++) for(unsigned pred=0;pred<8;pred++){ + unsigned bytes=(op==1||op==3)?8:4; const uint64_t *edges=bytes==4?edges32:edges64; + for(unsigned i=0;i<18;i++) for(unsigned j=0;j<18;j++) for(unsigned c=0;c<6;c++){ + uint8_t a[20]={0},b[20]={0},storage[22],want[16]; + uint8_t *out=storage+1;memset(storage,0xc3,sizeof(storage)); + for(unsigned lane=0;lane<16;lane+=bytes){memcpy(a+lane,edges+i,bytes);memcpy(b+lane,edges+j,bytes);} + uint8_t beforeA[20],beforeB[20];memcpy(beforeA,a,20);memcpy(beforeB,b,20); + originals[op](out,a,b,pred,controls[c]); memcpy(want,a,16); + if(storage[0]!=0xc3||storage[21]!=0xc3||memcmp(a,beforeA,20)||memcmp(b,beforeB,20))return 2; + unsigned compared=op>=2?bytes:16; + for(unsigned lane=0;lane=4?bytes:width; + for(unsigned lane=0;lane x86FloatCompareEVEX { + t.Fatal("unknown compare execution ISA", axis) + } + if axis != x86FloatCompareSSE && (runtime.GOOS != "linux" || runtime.GOARCH != "amd64" && runtime.GOARCH != "arm64") { + t.Fatal("VEX/EVEX comparison must consume its required Linux execution counterpart") + } + if axis == x86FloatCompareEVEX && runtime.GOARCH != "amd64" { + t.Fatal("EVEX requires actual native Linux AMD64, not an emulator or Rosetta") + } + execution := x86FloatCompareExecutionTools(t) + if axis == x86FloatCompareEVEX && (len(execution.prefix) != 0 || execution.preserveArgv0) { + t.Fatal("EVEX native counterpart cannot use a guest execution route") + } + x86FloatCompareVerifyArguments(t, execution) + if axis == x86FloatCompareVEX { + execution.requireAVX(t) + } + x86FloatCompareRequireOpcodeISA(t, execution, axis) + return execution +} + +// CPUID and enabled OS register state are both required. The original whole +// opcodes execute only after this check; their results are consumed. This is +// an execution preflight, not a production source/ISA authorization token. +const x86FloatCompareOpcodePreflightC = `#include +#include +#include +#if !defined(_WIN32) +#include +#endif + +static int required_state(unsigned axis, unsigned sse2, unsigned avx, + unsigned osxsave, unsigned f, unsigned vl, uint64_t xcr0) { + if (axis==1) return sse2!=0; + if (!sse2 || !avx || !osxsave || (xcr0&6)!=6) return 0; + if (axis==2) return 1; + return axis==3 && f && vl && (xcr0&0xe6)==0xe6; +} + +__attribute__((target("avx"))) static unsigned original_vex(void) { + unsigned mask; + __asm__ volatile("vxorps %%xmm0,%%xmm0,%%xmm0; " + "vcmpps $0,%%xmm0,%%xmm0,%%xmm1; " + "vmovmskps %%xmm1,%0" : "=r"(mask) :: "xmm0","xmm1"); + return mask; +} + +__attribute__((target("avx512f,avx512vl"))) static int original_evex(void) { + unsigned ps,pd,ss,sd,active; + __asm__ volatile("vxorps %%zmm0,%%zmm0,%%zmm0; " + "vcmpps $0,%%zmm0,%%zmm0,%%k1; kmovw %%k1,%0; " + "vcmppd $0,%%zmm0,%%zmm0,%%k1; kmovw %%k1,%1; " + "vcmpss $0,%%xmm0,%%xmm0,%%k1; kmovw %%k1,%2; " + "vcmpsd $0,%%xmm0,%%xmm0,%%k1; kmovw %%k1,%3; " + "movl $1,%4; kmovw %4,%%k2; " + "vcmpps $0,%%zmm0,%%zmm0,%%k1%{%%k2%}; kmovw %%k1,%4" + : "=r"(ps),"=r"(pd),"=r"(ss),"=r"(sd),"=r"(active) + :: "zmm0","k1","k2"); + return ps==0xffff && pd==0xff && ss==1 && sd==1 && active==1; +} + +int main(void) { +#if !defined(_WIN32) + struct rlimit no_core={0,0}; + if (setrlimit(RLIMIT_CORE,&no_core)) return 77; +#endif + unsigned a,b,c,d,sse2=0,avx=0,osxsave=0,f=0,vl=0; + uint64_t xcr0=0; + if (__get_cpuid(1,&a,&b,&c,&d)) { + sse2=(d>>26)&1; avx=(c>>28)&1; osxsave=(c>>27)&1; + if (osxsave) { + unsigned low,high; + __asm__ volatile("xgetbv" : "=a"(low),"=d"(high) : "c"(0)); + xcr0=((uint64_t)high<<32)|low; + } + } + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) { + f=(b>>16)&1; vl=(b>>31)&1; + } + unsigned axis=REQUIRED_AXIS; + printf("axis=%u SSE2=%u AVX=%u OSXSAVE=%u AVX512F=%u AVX512VL=%u XCR0=%llx\n", + axis,sse2,avx,osxsave,f,vl,(unsigned long long)xcr0); + fflush(stdout); + if (!required_state(axis,sse2,avx,osxsave,f,vl,xcr0)) return 78; + if (axis==2 && original_vex()!=15) return 79; + if (axis==3 && !original_evex()) return 80; + return 0; +} +` + +func x86FloatCompareRequireOpcodeISA(t *testing.T, execution x86FloatCompareExecution, axis x86FloatCompareRuntimeISA) { + t.Helper() + dir := t.TempDir() + source, executable := filepath.Join(dir, "preflight.c"), filepath.Join(dir, "preflight") + program := fmt.Sprintf("#define REQUIRED_AXIS %d\n", axis) + x86FloatCompareOpcodePreflightC + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + output := execution.run(t, executable) + t.Logf("actual CPU/OS-state and original-opcode preflight: %s", output) +} + +func TestX86FloatingCompareRuntimeCaseUnion(t *testing.T) { + counts, dead := [3]int{}, [3]int{} + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + complete := x86FloatCompareOracleCases(opcode) + // Scalar variant 1/2 intentionally have identical memory spellings. + // Preserve their original multiplicity instead of dropping a fixture. + remaining := make(map[x86FloatCompareOracleCase]int) + for _, fixture := range complete { + remaining[fixture]++ + } + for _, axis := range []x86FloatCompareRuntimeISA{x86FloatCompareSSE, x86FloatCompareVEX, x86FloatCompareEVEX} { + for _, fixture := range x86FloatCompareRuntimeCases(opcode, axis) { + if remaining[fixture] == 0 { + t.Fatal("duplicated runtime case", fixture.row) + } + remaining[fixture]-- + counts[axis-1]++ + if x86FloatCompareDeadPredicate(fixture.predicate) { + dead[axis-1]++ + } + } + } + for fixture, count := range remaining { + if count != 0 { + t.Fatal("missing runtime case", opcode, fixture.row, count) + } + } + } + if counts != [3]int{64, 256, 512} || dead != [3]int{16, 48, 96} { + t.Fatalf("changed complete fixture denominators: IEEE=%v dead=%v", counts, dead) + } + var fp [3]int + for _, fixture := range x86FloatCompareFPOracles(t) { + fp[fixture.axis-1]++ + } + if fp != [3]int{16, 16, 16} { + t.Fatal("changed FP byte-field runtime denominator", fp) + } +} + +func TestX86FloatingCompareCompleteRuntimeOriginalGoInventory(t *testing.T) { + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + t.Run(opcode, func(t *testing.T) { + assembly, _ := x86FloatCompareOracleAssembly(x86FloatCompareOracleCases(opcode), false) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + }) + } + var dead []x86FloatCompareOracleCase + for _, opcode := range []string{"CMPPS", "CMPPD", "CMPSS", "CMPSD", "VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"} { + for _, fixture := range x86FloatCompareOracleCases(opcode) { + if x86FloatCompareDeadPredicate(fixture.predicate) { + dead = append(dead, fixture) + } + } + } + assembly, _ := x86FloatCompareOracleAssembly(dead, true) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + assembly, _ = x86FloatCompareFPAssembly(x86FloatCompareFPOracles(t)) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + t.Log("actual stock Go objects bind 832 independent IEEE, 160 dead-result and all 48 scalar FP-byte-field programs; object evidence does not imply ISA execution") +} + +func TestX86FloatingCompareOpcodePreflightCapabilities(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, executable := filepath.Join(dir, "capabilities.c"), filepath.Join(dir, "capabilities") + // Execute the exact preflight predicate against independent missing-feature + // cases, without substituting synthetic capabilities for an actual ISA run. + const checks = ` +#undef main +int main(void) { + if (!required_state(1,1,0,0,0,0,0) || required_state(1,0,1,1,1,1,0xe6)) return 1; + if (!required_state(2,1,1,1,0,0,6) || !required_state(3,1,1,1,1,1,0xe6)) return 2; + if (required_state(0,1,1,1,1,1,0xe6) || required_state(4,1,1,1,1,1,0xe6)) return 3; + for (unsigned axis=2;axis<=3;axis++) { + if (required_state(axis,0,1,1,1,1,0xe6) || required_state(axis,1,0,1,1,1,0xe6) || + required_state(axis,1,1,0,1,1,0xe6)) return 4; + if (required_state(axis,1,1,1,1,1,0xe6&~2) || + required_state(axis,1,1,1,1,1,0xe6&~4)) return 5; + } + if (required_state(3,1,1,1,0,1,0xe6) || required_state(3,1,1,1,1,0,0xe6)) return 6; + for (unsigned bit=5;bit<=7;bit++) { + if (required_state(3,1,1,1,1,1,0xe6&~(1u< +int main(void) { + __builtin_cpu_init(); + int avx=!!__builtin_cpu_supports("avx"); + int avx2=!!__builtin_cpu_supports("avx2"); + int avx512=!!__builtin_cpu_supports("avx512f"); + printf("AVX=%d AVX2=%d AVX512F=%d\n",avx,avx2,avx512); + return avx?0:1; +} +` + if err := os.WriteFile(source, []byte(preflight), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("actual guest CPUID/XCR0 preflight: %s", strings.TrimSpace(string(execution.run(t, executable)))) +} + +func TestX86FloatingCompareExecutionTransport(t *testing.T) { + for _, test := range []struct { + name string + execution x86FloatCompareExecution + want []string + }{ + {"native", x86FloatCompareExecution{}, []string{"/guest", "numeric", "with space"}}, + {"rosetta", x86FloatCompareExecution{prefix: []string{"arch", "-x86_64"}}, + []string{"arch", "-x86_64", "/guest", "numeric", "with space"}}, + {"pinned", x86FloatCompareExecution{prefix: []string{"qemu", "-cpu", "max"}, preserveArgv0: true}, + []string{"qemu", "-cpu", "max", "/guest", "/guest", "numeric", "with space"}}, + } { + t.Run(test.name, func(t *testing.T) { + arguments := []string{"numeric", "with space"} + before := append([]string(nil), test.execution.prefix...) + if actual := test.execution.command("/guest", arguments...); !reflect.DeepEqual(actual, test.want) { + t.Fatalf("guest arguments: got %q, want %q", actual, test.want) + } + if !reflect.DeepEqual(before, test.execution.prefix) || !reflect.DeepEqual(arguments, []string{"numeric", "with space"}) { + t.Fatal("execution argument construction mutated its inputs") + } + }) + } +} + +func x86FloatCompareVerifyArguments(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module comparearguments\n\ngo 1.20\n", + "main.go": "package main\nimport(\"fmt\";\"os\")\nfunc main(){fmt.Printf(\"%q\\n\",os.Args)}\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, gotoolchain.HostExecutable("arguments")) + command := exec.Command("go", "build", "-p=2", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", + "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + x86FloatCompareCaptureCommand(t, command) + for _, arguments := range [][]string{nil, {"numeric", "with space"}} { + want := fmt.Sprintf("%q\n", append([]string{executable}, arguments...)) + if actual := string(execution.run(t, executable, arguments...)); actual != want { + t.Fatalf("actual guest argument witness: got %q, want %q", actual, want) + } + } +} diff --git a/amd64_float_compare_vex_numerical_linux_test.go b/amd64_float_compare_vex_numerical_linux_test.go new file mode 100644 index 000000000..7fbf00d0b --- /dev/null +++ b/amd64_float_compare_vex_numerical_linux_test.go @@ -0,0 +1,11 @@ +//go:build linux && (amd64 || arm64) + +package plan9asm + +import "testing" + +// This mandatory axis executes the translated actual whole VEX compare against +// the independent original opcode. QEMU has AVX/AVX2 but not native AVX512. +func TestCrossLinuxRuntimeMatrixX86FloatingCompareVEXWholeIEEEAndMXCSR(t *testing.T) { + x86FloatCompareVEXWholeRuntime(t) +} diff --git a/amd64_float_compare_vex_numerical_test.go b/amd64_float_compare_vex_numerical_test.go new file mode 100644 index 000000000..081d393e0 --- /dev/null +++ b/amd64_float_compare_vex_numerical_test.go @@ -0,0 +1,357 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// This matrix is independent of the production operand/predicate registries. +// All six native VEX widths have every predicate and both first-source roles; +// destination aliases rotate through both sources and an independent register. +func x86FloatCompareVEXOracleCases() []x86FloatCompareOracleCase { + var cases []x86FloatCompareOracleCase + for _, fixture := range x86FloatCompareOriginalVectors() { + bank := "X" + if fixture.width == 32 { + bank = "Y" + } + for predicate := 0; predicate < 32; predicate++ { + for _, memory := range []bool{false, true} { + first := bank + "1" + if memory { + first = "(CX)" + } + cases = append(cases, x86FloatCompareOracleCase{ + row: fmt.Sprintf("%s $%d,%s,%s0,%s%d", fixture.opcode, predicate, first, bank, bank, predicate%3), + bits: fixture.bits, width: fixture.width, predicate: predicate, + scalar: fixture.scalar, memory: memory, destination: predicate % 3, + }) + } + } + } + return cases +} + +func x86FloatCompareVEXOracleAssembly(cases []x86FloatCompareOracleCase) (string, map[string]FuncSig) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, fixture := range cases { + name := fmt.Sprintf("vex_compare%d", index) + fmt.Fprintf(&source, `TEXT %s(SB),4,$0-24 +MOVQ out+0(FP),AX +MOVQ a+8(FP),BX +MOVQ b+16(FP),CX +VMOVUPS (BX),Y0 +VMOVUPS (BX),Y2 +`, name) + if !fixture.memory { + source.WriteString("VMOVUPS (CX),Y1\n") + } + source.WriteString("STC\n" + fixture.row + "\n") + // Z is a translated shadow-view observation, not a native AVX512 + // instruction in the original-opcode reference. The expected high bytes + // are independently zero for VEX X/Y writes. No EVEX compare is run. + fmt.Fprintf(&source, "VMOVUPS Z%d,8(AX)\nSETCS 72(AX)\nRET\n", fixture.destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + } + return source.String(), sigs +} + +// Emit the actual requested whole VCMP opcode and full native memory tuple. +// Its MXCSR is an independent original-instruction reference, not a union of +// per-lane quiet/signaling representatives. IEEE truth remains integer-only. +func x86FloatCompareVEXWholeReferenceC() string { + var source strings.Builder + source.WriteString(x86FloatCompareIndependentC) + for index, fixture := range x86FloatCompareOriginalVectors() { + for variant := 0; variant < 2; variant++ { + fmt.Fprintf(&source, ` +__attribute__((target("avx"))) static void reference%d(uint8_t *out,const uint8_t *a,const uint8_t *b,unsigned predicate,unsigned csr,unsigned *after){ + typedef uint8_t vector __attribute__((vector_size(%d))); + vector left,right,result; + memcpy(&left,a,sizeof(left));memcpy(&right,b,sizeof(right)); + unsigned saved;uint8_t carry; + __asm__ volatile("stmxcsr %%0":"=m"(saved)); + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + switch(predicate){ +`, index*2+variant, fixture.width) + operand := "\"x\"(right)" + if variant == 1 { + bytes := fixture.width + if fixture.scalar { + bytes = fixture.bits / 8 + } + operand = fmt.Sprintf("\"m\"(*(const uint8_t(*)[%d])b)", bytes) + } + for predicate := 0; predicate < 32; predicate++ { + fmt.Fprintf(&source, " case %d: __asm__ volatile(\"stc; %s $%d,%%3,%%2,%%0; setc %%1\":\"=x\"(result),\"=qm\"(carry):\"x\"(left),%s:\"cc\");break;\n", + predicate, strings.ToLower(fixture.opcode), predicate, operand) + } + source.WriteString(` } + __asm__ volatile("stmxcsr %0":"=m"(*after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + memset(out,0,64);memcpy(out,&result,sizeof(result));out[64]=carry; +} +`) + } + } + source.WriteString("typedef void(*reference_fn)(uint8_t*,const uint8_t*,const uint8_t*,unsigned,unsigned,unsigned*);\nstatic reference_fn references[]={") + for index := 0; index < 12; index++ { + fmt.Fprintf(&source, "reference%d,", index) + } + source.WriteString("};\n") + return source.String() +} + +func x86FloatCompareVEXWholeMain(cases []x86FloatCompareOracleCase) string { + var declarations, descriptors strings.Builder + for index, fixture := range cases { + fmt.Fprintf(&declarations, "extern void vex_compare%d(void*,const void*,const void*);\n", index) + fmt.Fprintf(&descriptors, "{vex_compare%d,%d,%d,%d,%d,%d},\n", index, index/64*2+x86FloatAddSubBoolInt(fixture.memory), + fixture.bits, fixture.width, fixture.predicate, x86FloatAddSubBoolInt(fixture.scalar)) + } + return x86FloatCompareVEXWholeReferenceC() + declarations.String() + ` +typedef void(*translated_fn)(void*,const void*,const void*); +struct comparison{translated_fn call;unsigned reference,bits,width,predicate,scalar;}; +static const struct comparison comparisons[]={ +` + descriptors.String() + ` +}; +int main(void){ + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)); + uint64_t samples=0; + for(unsigned index=0;indexbits/8;const uint64_t *edges=bytes==4?edges32:edges64; + for(unsigned mode=0;mode<2;mode++)for(unsigned i=0;i<18;i++)for(unsigned j=0;j<18;j++)for(unsigned control=0;control<6;control++){ + uint8_t aStorage[192],bStorage[192],out[81],expected[65],truth[64]; + uint8_t *a=aStorage+65,*b=bStorage+65; + memset(aStorage,0xa7,sizeof(aStorage));memset(bStorage,0xb9,sizeof(bStorage));memset(out,0xc3,sizeof(out)); + for(unsigned lane=0;lane<64/bytes;lane++){ + uint64_t av=edges[(i+(mode?lane:0))%18],bv=edges[(j+(mode?lane*5:0))%18]; + memcpy(a+lane*bytes,&av,bytes);memcpy(b+lane*bytes,&bv,bytes); + } + uint8_t beforeA[192],beforeB[192];memcpy(beforeA,aStorage,192);memcpy(beforeB,bStorage,192); + unsigned csr=controls[control],expectedCSR,after; + references[c->reference](expected,a,b,c->predicate,csr,&expectedCSR); + memset(truth,0,64);if(c->scalar)memcpy(truth,a,16); + unsigned compared=c->scalar?bytes:c->width; + for(unsigned lane=0;lanebits,read_bits(a+lane,bytes),read_bits(b+lane,bytes),c->predicate,csr); + memset(truth+lane,value?255:0,bytes); + } + if(memcmp(truth,expected,64)||expected[64]!=1){ + fprintf(stderr,"original VEX truth/CF index=%u pred=%u mode=%u i=%u j=%u csr=%x\n",index,c->predicate,mode,i,j,csr);return 10; + } + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + c->call(out,a,b); + __asm__ volatile("stmxcsr %0":"=m"(after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + if(after!=expectedCSR||memcmp(aStorage,beforeA,192)||memcmp(bStorage,beforeB,192)){ + fprintf(stderr,"translated VEX MXCSR/input index=%u pred=%u mode=%u i=%u j=%u csr=%x got=%x want=%x\n",index,c->predicate,mode,i,j,csr,after,expectedCSR);return 11; + } + for(unsigned byte=0;byte<81;byte++){ + uint8_t want=byte>=8&&byte<=72?expected[byte-8]:0xc3; + if(out[byte]!=want){ + fprintf(stderr,"translated VEX alias/canary index=%u pred=%u mode=%u i=%u j=%u byte=%u got=%x want=%x\n",index,c->predicate,mode,i,j,byte,out[byte],want);return 12; + } + } + samples++; + } + } + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + printf("384 VEX forms, %llu full IEEE/MXCSR/CF/64-alias/canary samples\n",(unsigned long long)samples); + return samples==UINT64_C(1492992)?0:13; +} +` +} + +func TestX86FloatingCompareVEXWholeOracleInventory(t *testing.T) { + cases := x86FloatCompareVEXOracleCases() + if len(cases) != 384 { + t.Fatalf("VEX forms=%d want384", len(cases)) + } + source, _ := x86FloatCompareVEXOracleAssembly(cases) + if strings.Contains(source, "KMOV") || strings.Contains(source, ".SAE") || strings.Contains(source, ".BCST") { + t.Fatal("non-VEX compare prerequisites entered native VEX oracle") + } + goAssemblerOracleOutput(t, "amd64", source, "", true) + if strings.Contains(x86FloatCompareVEXWholeMain(cases), "original_effect(") { + // The shared relation source declares this older diagnostic helper but + // the whole-vector driver must not call it or substitute sample values. + main := strings.Split(x86FloatCompareVEXWholeMain(cases), "int main(void){")[1] + if strings.Contains(main, "original_effect(") || strings.Contains(main, "is_denormal(") { + t.Fatal("whole native effect reference was replaced by a class/lane policy") + } + } +} + +func TestX86FloatingCompareVEXWholeEffectsIR(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + cases := x86FloatCompareVEXOracleCases() + var physicalSource strings.Builder + var physicalBytes strings.Builder + physicalSigs := make(map[string]FuncSig) + for index, fixture := range cases { + name := fmt.Sprintf("physical_vex%d", index) + fmt.Fprintf(&physicalSource, "TEXT %s(SB),4,$0-0\n", name) + for _, b := range x86FloatCompareVEXOriginalBytes(t, fixture.row) { + fmt.Fprintf(&physicalSource, "BYTE $%d\n", b) + fmt.Fprintf(&physicalBytes, "0x%02x ", b) + } + physicalBytes.WriteString("\n") + // This is only a physical mode32 format harness. Unknown incoming + // pointers cannot borrow a native return contract from that byte marker. + physicalSource.WriteString("UNDEF\n") + physicalSigs[name] = FuncSig{Name: name, Ret: Void} + } + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 llvm-mc required") + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=i386-unknown-linux-gnu", "--mattr=+avx") + command.Stdin = strings.NewReader(physicalBytes.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go mode32 byte validation: %v\n%s", err, output) + } + physical := regexp.MustCompile(`(?m)^\s+vcmp[a-z0-9_]*(ps|pd|ss|sd)\s+`).FindAll(output, -1) + if len(physical) != 384 || strings.Contains(string(output), "warning:") { + t.Fatalf("original AMD64 Go bytes lack complete physical mode32 LLVM-MC identity:\n%s", output) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-macosx"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + source, sigs := x86FloatCompareVEXOracleAssembly(cases) + if target.arch == "386" { + source, sigs = physicalSource.String(), physicalSigs + } + goAssemblerOracleOutput(t, target.arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir := func() string { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + return module.String() + }() + if strings.Contains(ir, " = fcmp ") || strings.Count(ir, "asm sideeffect") < 384 { + t.Fatal("full native VEX predicate/width side effects were replaced by relation-only fcmp") + } + compileLLVMToObject(t, llc, target.triple, "vex-whole.ll", "vex-whole.o", ir) + }) + } +} + +func x86FloatCompareVEXOriginalBytes(t *testing.T, row string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatalf("original Go VEX encoding: %v\n%s", err, listing) + } + length := 0 + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) >= 4 && fields[len(fields)-1] == "RET" { + length, err = strconv.Atoi(fields[1]) + if err != nil || length < 1 || length > 15 { + t.Fatalf("original VEX native width: %v\n%s", err, listing) + } + } + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if length == 0 || len(fields) <= length || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + code := make([]byte, length) + for index := range code { + value, err := strconv.ParseUint(fields[index+1], 16, 8) + if err != nil { + t.Fatal(err) + } + code[index] = byte(value) + } + return code + } + t.Fatalf("original VEX machine bytes unavailable:\n%s", listing) + return nil +} + +func TestX86FloatingCompareVEXWholeOriginalCObject(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, object := filepath.Join(dir, "original.c"), filepath.Join(dir, "original.o") + contents := x86FloatCompareVEXWholeMain(x86FloatCompareVEXOracleCases()) + // These two headers belong to older fault fixtures, not this numerical + // reference. Keep the ordinary C object portable to Windows as well. + contents = strings.NewReplacer("#include \n", "", "#include \n", "").Replace(contents) + if err := os.WriteFile(source, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + arguments := append(append([]string(nil), execution.compiler...), "-O2", "-c", source, "-o", object) + command := exec.Command(arguments[0], arguments[1:]...) + x86FloatCompareCaptureCommand(t, command) +} + +func x86FloatCompareVEXWholeRuntime(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.requireAVX(t) + x86FloatCompareVerifyArguments(t, execution) + cases := x86FloatCompareVEXOracleCases() + source, sigs := x86FloatCompareVEXOracleAssembly(cases) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir := func() string { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + return module.String() + }() + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "vex-whole-compare", ir, x86FloatCompareVEXWholeMain(cases), level) + const expected = "384 VEX forms, 1492992 full IEEE/MXCSR/CF/64-alias/canary samples" + if strings.TrimSpace(string(output)) != expected { + t.Fatalf("whole native VEX oracle output: %q", output) + } + t.Logf("%s: %s transport=%v", level, expected, execution.prefix) + }) + } + t.Log("384 register/memory VEX forms ×3888 full Cartesian/mixed-lane samples per optimization level; no EVEX execution") +} diff --git a/amd64_float_mul_div_forms.go b/amd64_float_mul_div_forms.go new file mode 100644 index 000000000..598d3872b --- /dev/null +++ b/amd64_float_mul_div_forms.go @@ -0,0 +1,52 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86FloatMulDivSpec struct { + bits int + legacy, scalar, divide bool +} + +// Every legacy yxm and vector _yvaddpd/_yvaddsd MUL/DIV opcode. Sharing +// operand shapes does not share ADD/SUB's special-value or rounding rules. +var x86FloatMulDivSpecs = map[Op]x86FloatMulDivSpec{ + "MULPS": {32, true, false, false}, "MULPD": {64, true, false, false}, + "MULSS": {32, true, true, false}, "MULSD": {64, true, true, false}, + "DIVPS": {32, true, false, true}, "DIVPD": {64, true, false, true}, + "DIVSS": {32, true, true, true}, "DIVSD": {64, true, true, true}, + "VMULPS": {32, false, false, false}, "VMULPD": {64, false, false, false}, + "VMULSS": {32, false, true, false}, "VMULSD": {64, false, true, false}, + "VDIVPS": {32, false, false, true}, "VDIVPD": {64, false, false, true}, + "VDIVSS": {32, false, true, true}, "VDIVSD": {64, false, true, true}, +} + +type x86FloatMulDivForm struct { + spec x86FloatMulDivSpec + x86FloatBinaryForm +} + +func x86FloatMulDivProperties(ins Instr, goarch string) (x86FloatMulDivForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86FloatMulDivSpecs[Op(base)] + if !recognized { + return x86FloatMulDivForm{}, false, nil + } + literal := x86FloatArithmeticLiteral(ins, spec.legacy, spec.scalar) + shape, err := x86FloatBinaryOperandProperties(ins, goarch, spec.bits, spec.legacy, spec.scalar, literal, "mul/div") + form := x86FloatMulDivForm{spec: spec, x86FloatBinaryForm: shape} + if err != nil { + return form, true, err + } + if goarch == "386" && !spec.legacy { + for _, operand := range ins.Args { + width := amd64VectorByteWidth(operand.Reg) + if index, valid := amd64VectorRegisterIndex(operand.Reg, width); operand.Kind == OpReg && valid && index >= 8 { + return form, true, fmt.Errorf("mode32 floating mul/div has no physical SIMD register %s: %w", operand.Reg, ErrProbeNeedsContext) + } + } + } + return form, true, nil +} diff --git a/amd64_float_mul_div_forms_test.go b/amd64_float_mul_div_forms_test.go new file mode 100644 index 000000000..0bf7f466c --- /dev/null +++ b/amd64_float_mul_div_forms_test.go @@ -0,0 +1,239 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86FloatMulDivOpcodes() []string { + return []string{ + "MULPS", "MULPD", "MULSS", "MULSD", + "DIVPS", "DIVPD", "DIVSS", "DIVSD", + "VMULPS", "VMULPD", "VMULSS", "VMULSD", + "VDIVPS", "VDIVPD", "VDIVSS", "VDIVSD", + } +} + +func x86FloatMulDivFormats(arch string) []string { + // All eight opcode attributes are independently checked below. Go uses + // the same yxm/_yvaddpd/_yvaddsd operand grammars as ADD/SUB, not merely + // the same vector widths. Reuse the independent shape-axis inventory. + rows := x86FloatAddSubFormats(arch) + for i, row := range rows { + rows[i] = strings.NewReplacer("ADD", "MUL", "SUB", "DIV").Replace(row) + } + return rows +} + +func TestX86FloatMulDivEveryGoOpcodeAttribute(t *testing.T) { + if len(x86FloatMulDivSpecs) != 16 { + t.Fatal("incomplete floating MUL/DIV family") + } + for _, opcode := range x86FloatMulDivOpcodes() { + bits := 32 + if strings.HasSuffix(opcode, "PD") || strings.HasSuffix(opcode, "SD") { + bits = 64 + } + want := x86FloatMulDivSpec{bits, !strings.HasPrefix(opcode, "V"), + strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD"), strings.Contains(opcode, "DIV")} + if got, found := x86FloatMulDivSpecs[Op(opcode)]; !found || got != want { + t.Fatalf("%s typed specification differs from independent mnemonic: %+v/%+v", opcode, got, want) + } + } + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return // The current-toolchain gate owns encoder-table completeness. + } + dir := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + for _, filename := range []string{"asm6.go", "avx_optabs.go"} { + data, err := os.ReadFile(filepath.Join(dir, filename)) + if err != nil { + t.Fatal(err) + } + for _, opcode := range x86FloatMulDivOpcodes() { + legacy := !strings.HasPrefix(opcode, "V") + if (filename == "asm6.go") != legacy { + continue + } + if legacy { + if !strings.Contains(string(data), "{A"+opcode+", yxm,") { + t.Fatalf("Go legacy grammar changed: %s", opcode) + } + continue + } + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + bits := 32 + if strings.HasSuffix(opcode, "PD") || strings.HasSuffix(opcode, "SD") { + bits = 64 + } + prefix := "{as: A" + opcode + ", ytab: " + start := strings.Index(string(data), prefix) + if start < 0 { + t.Fatalf("Go omits %s", opcode) + } + row := strings.SplitN(string(data)[start:], "\n\t}},", 2)[0] + table := "_yvaddpd" + if scalar { + table = "_yvaddsd" + } + if !strings.HasPrefix(row, prefix+table+",") { + t.Fatalf("Go grammar changed: %s", row) + } + seen := 0 + for _, line := range strings.Split(row, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + seen++ + if !strings.Contains(line, "evexZeroingEnabled") || strings.Contains(line, "evexSaeEnabled") || + strings.Contains(line, "evexRoundingEnabled") != (scalar || strings.Contains(line, "evex512")) || + strings.Contains(line, fmt.Sprintf("evexBcstN%d", bits/8)) != !scalar { + t.Fatalf("%s opcode attribute changed: %s", opcode, line) + } + } + if scalar && seen != 1 || !scalar && seen != 3 { + t.Fatalf("%s incomplete EVEX row count %d", opcode, seen) + } + } + } +} + +func TestX86FloatMulDivCompleteGoGrammarAndFiveLLVMTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + var declarations strings.Builder + declarations.WriteString("package shapes\n") + sigs := map[string]FuncSig{} + for index, row := range x86FloatMulDivFormats(arch) { + name := fmt.Sprintf("muldiv%d", index) + entry := fmt.Sprintf("TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + file, err := Parse(ArchAMD64, entry) + if err != nil { + t.Fatal(err) + } + form, recognized, err := x86FloatMulDivProperties(file.Funcs[0].Instrs[1], arch) + if !recognized || err != nil || form.bits == 0 || form.byteWidth == 0 { + t.Fatalf("%s/%s: %v", arch, row, err) + } + source.WriteString(entry) + fmt.Fprintf(&declarations, "func %s()\n", name) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + options := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + func() { + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + goSource := strings.ReplaceAll(source.String(), "TEXT muldiv", "TEXT ·muldiv") + result, err := translateGoModuleInContext(context, x86FloatMulDivShapesPackage(t, declarations.String(), arch), + []byte(goSource), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + for api, body := range []string{ir, module.String(), result.Module.String()} { + name := fmt.Sprintf("muldiv-formats-api%d", api) + compileLLVMToObject(t, llc, triple, name+".ll", name+".o", body) + } + }() + } + t.Logf("%s: %d independent Go MUL/DIV formats", arch, len(sigs)) + } +} + +func x86FloatMulDivShapesPackage(t *testing.T, source, arch string) GoPackage { + t.Helper() + set := token.NewFileSet() + file, err := parser.ParseFile(set, "shapes.go", source, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("shapes", set, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: pkg.Path(), Types: pkg, Syntax: []*ast.File{file}} +} + +func TestX86FloatMulDivRejectsMalformedBeforeEmission(t *testing.T) { + for _, row := range []string{ + "MULPS X0,Y1", "MULSD X0,AX", "DIVSS X0,X16", "DIVPD.SAE X0,X1", + "VMULPS X0,X1", "VMULPD X0,Y1,Y2", "VMULSS Y0,Y1,Y2", "VDIVSD X0,X1,K0,X2", + "VMULPS.Z X0,X1,X2", "VMULPS.BCST X0,X1,X2", "VMULSS.BCST (AX),X1,X2", + "VDIVPS.SAE Z0,Z1,Z2", "VMULPS.RD_SAE X0,X1,X2", "VDIVSD.RU_SAE (AX),X1,X2", + "VMULPS. X0,X1,X2", "VDIVPS.Z.BCST (AX),Z1,K1,Z2", + "VMULPS $data(SB),X1,X2", "VDIVSS (AX)(SP*2),X1,X2", "VMULPD 2147483648(AX),X1,X2", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + oldReservedZeroing := row == "VMULPS.Z X0,X1,X2" && !goToolchainAtLeast(runtime.Version(), 1, 21) + goAssemblerOracleOutput(t, "amd64", source, "", oldReservedZeroing) + if oldReservedZeroing { + listing := x86InsertExtractGoListing(t, "amd64", source) + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if !bytes.Equal(code, []byte{0x62, 0xf1, 0x74, 0x88, 0x59, 0xd0, 0xc3}) { + t.Fatalf("old Go reserved EVEX.z/aaa=0 observation changed: %x", code) + } + if _, _, known, err := decodedX86EVEXBinaryFloatInstruction(code, 0, 64, "", false); !known || err == nil { + t.Fatal("old Go reserved zeroing bytes acquired valid raw semantics") + } + t.Log("old Go accepted reserved EVEX.z/aaa=0; both named and raw validators still reject it") + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + return // Parser rejection also occurs before IR emission. + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + var known, term bool + if strings.HasPrefix(row, "V") { + known, term, err = ctx.lowerBinaryFloating(ins.Op, ins) + } else { + known, term, err = ctx.lowerFP(ins.Op, ins) + } + if !known || term || err == nil || output.Len() != before { + t.Fatalf("invalid MUL/DIV emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} diff --git a/amd64_float_mul_div_literal_runtime_test.go b/amd64_float_mul_div_literal_runtime_test.go new file mode 100644 index 000000000..5d7a6f535 --- /dev/null +++ b/amd64_float_mul_div_literal_runtime_test.go @@ -0,0 +1,256 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86MulDivLiteralNumericalSource() (declarations, assembly string) { + var goSource, machine strings.Builder + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + for _, value := range x86MulDivLiteralValues { + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&goSource, "func %s(input *[32]byte, output *[80]byte, csr uint32)\n", name) + fmt.Fprintf(&machine, `TEXT ·%s(SB),4,$8-20 +STMXCSR saved-4(SP) +LDMXCSR csr+16(FP) +MOVQ input+0(FP),AX +MOVQ output+8(FP),BX +VMOVUPS (AX),Y0 +MOVQ $0x7fffffffffffffff,DX +ADDQ $1,DX +STC +%s $%s,X0 +MOVUPS X0,8(BX) +VMOVUPS Y0,24(BX) +STMXCSR 56(BX) +SETCS 60(BX) +SETOS 61(BX) +LDMXCSR saved-4(SP) +RET +`, name, opcode, value.text) + } + } + return goSource.String(), machine.String() +} + +func x86MulDivLiteralOptimizedObject(t *testing.T, llc, triple, ir string) { + t.Helper() + directory := t.TempDir() + source, object := filepath.Join(directory, "literal.ll"), filepath.Join(directory, "literal.o") + if err := os.WriteFile(source, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=obj", source, "-o", object) + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("LLVM22 optimized literal object: %v\n%s", err, output) + } + info, err := os.Stat(object) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatalf("LLVM22 optimized literal object missing/nonregular/empty: %v", err) + } +} + +func x86MulDivLiteralCOracle() string { + var declarations, checks strings.Builder + index := 0 + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86MulDivLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + name := strings.ToLower(opcode) + value.name + fmt.Fprintf(&declarations, "extern void %s(void*,void*,unsigned);\n", name) + fmt.Fprintf(&checks, " if (check_literal(%s,%q,%d,UINT64_C(0x%x),%d)) return %d;\n", name, name, bits, literal, x86FloatAddSubBoolInt(strings.HasPrefix(opcode, "DIV")), index+1) + index++ + } + } + return x86FloatMulDivSSEOracle + declarations.String() + ` +typedef void (*literal_fn)(void*,void*,unsigned); +static int check_literal(literal_fn fn,const char *name,unsigned bits,uint64_t literal,int divide) { + const uint64_t edge32[]={0,0x80000000,1,0x80000001,0x3f800000,0xbf800000, + 0x7f7fffff,0x7f800000,0xff800000,0x7fc12345,0x7f812345}; + const uint64_t edge64[]={0,UINT64_C(0x8000000000000000),1,UINT64_C(0x8000000000000001), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x7fefffffffffffff), + UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0x7ff0123456789abc)}; + const unsigned controls[]={0x1f80,0x1f85,0x1fc0,0x9f80,0x9fc0,0x3f80,0x5f80,0x7f80}; + for(unsigned sample=0;sample<11;sample++) { + unsigned char input[32],before[32],output[80],want[80]; + for(unsigned byte=0;byte<32;byte++) input[byte]=(unsigned char)(byte*37+19); + uint64_t left=bits==32?edge32[sample]:edge64[sample]; + memcpy(input,&left,bits/8); + memcpy(before,input,32); + for(unsigned mode=0;mode<8;mode++) { + unsigned ambient=0x3f85,after; + memset(output,0xa7,80); + memset(want,0xa7,80); + uint64_t result=sse_result(bits,left,literal,divide,controls[mode]); + unsigned expectedCSR=last_sse_csr; + memcpy(want+8,input,16); + memcpy(want+24,input,32); + memcpy(want+8,&result,bits/8); + memcpy(want+24,&result,bits/8); + memcpy(want+56,&expectedCSR,4); + want[60]=1; + want[61]=1; + __asm__ volatile("ldmxcsr %0"::"m"(ambient)); + fn(input,output,controls[mode]); + __asm__ volatile("stmxcsr %0":"=m"(after)); + if(memcmp(input,before,32)||memcmp(output,want,80)||after!=ambient) { + fprintf(stderr,"%s bits=%u sample=%u csr=%x after=%x\n",name,bits,sample,controls[mode],after); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + ` + unsigned csr=0x1f80; + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + return 0; +} +` +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivLiteralNumerical(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86MulDivLiteralNumericalSource() + x86ScalarLiteralGoListing(t, "amd64", assembly) + pkg := x86MulDivLiteralPackage(t, "amd64", declarations) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.TrimPrefix(sym, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var translation *GoModuleTranslation + translation, err = translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{GOARCH: "amd64", TargetTriple: triple, ResolveSym: resolve}) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + ir = translation.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "muldiv-literal-"+api, triple, ir, x86MulDivLiteralCOracle(), prefix) + }) + } + t.Logf("%d original literal forms × 11 independent SSE bit inputs × 8 full MXCSR controls × 3 translation APIs; whole X/Y bytes, CF/OF=1 and canaries", 4*len(x86MulDivLiteralValues)) +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivLiteralOriginalGo(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + declarations, assembly := x86MulDivLiteralNumericalSource() + var checks strings.Builder + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + bits := 32 + if strings.HasSuffix(opcode, "SD") { + bits = 64 + } + for _, value := range x86MulDivLiteralValues { + literal := value.bits64 + if bits == 32 { + literal = uint64(value.bits32) + } + fmt.Fprintf(&checks, " check(t,%s,%d,0x%x,%t)\n", strings.ToLower(opcode)+value.name, bits, literal, strings.HasPrefix(opcode, "DIV")) + } + } + goTest := `package literal +import("encoding/binary";"math";"testing") +type literalFn func(*[32]byte,*[80]byte,uint32) +func check(t *testing.T,fn literalFn,bits int,literal uint64,divide bool) { + t.Helper() + for _,negative:=range []bool{false,true} { + var input [32]byte + var output [80]byte + for i:=range input {input[i]=byte(i*37+19)} + for i:=range output {output[i]=0xa7} + var result uint64 + if bits==32 { + left:=float32(4) + if negative {left=-left} + binary.LittleEndian.PutUint32(input[:4],math.Float32bits(left)) + right:=math.Float32frombits(uint32(literal)) + if divide {left/=right}else{left*=right} + result=uint64(math.Float32bits(left)) + }else{ + left:=float64(4) + if negative {left=-left} + binary.LittleEndian.PutUint64(input[:8],math.Float64bits(left)) + right:=math.Float64frombits(literal) + if divide {left/=right}else{left*=right} + result=math.Float64bits(left) + } + before:=input + fn(&input,&output,0x1f80) + if input!=before {t.Fatal("input changed")} + for i,got:=range output { + if i>=56&&i<60 {continue} // CSR is checked independently by the C SSE oracle. + want:=byte(0xa7) + if i>=8&&i<24 {want=input[i-8]} + if i>=24&&i<56 {want=input[i-24]} + if i>=8&&i<8+bits/8 {want=byte(result>>(8*(i-8)))} + if i>=24&&i<24+bits/8 {want=byte(result>>(8*(i-24)))} + if i==60||i==61 {want=1} + if got!=want {t.Fatalf("bits=%d literal=%x byte=%d got=%x want=%x",bits,literal,i,got,want)} + } + } +} +func TestOriginalLiteralNumerical(t *testing.T) { +` + checks.String() + "}\n" + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module literal\n\ngo 1.20\n", "literal.go": "package literal\n" + declarations, + "literal_amd64.s": assembly, "literal_test.go": goTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go original literal numerical: %v\n%s", err, output) + } + t.Logf("actual Go original %d literal kernels × 2 initialized finite inputs; X/Y aliases, CF/OF=1 and whole output canaries (host execution may be Rosetta)", 4*len(x86MulDivLiteralValues)) +} diff --git a/amd64_float_mul_div_literal_test.go b/amd64_float_mul_div_literal_test.go new file mode 100644 index 000000000..75f91dbb3 --- /dev/null +++ b/amd64_float_mul_div_literal_test.go @@ -0,0 +1,506 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86MulDivLiteralValues = []x86ScalarLiteralValue{ + {name: "reduce", text: "0.0625", bits32: 0x3d800000, bits64: 0x3fb0000000000000}, + {name: "one", text: "1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "plus", text: "+1.0", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "minus", text: "-1.0", bits32: 0xbf800000, bits64: 0xbff0000000000000}, + {name: "zero", text: "0.0"}, + {name: "negativeZero", text: "-0.0", bits32: 0x80000000, bits64: 0x8000000000000000}, + {name: "hex", text: "0x1.4p+0", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "scientific", text: "6.25e-1", bits32: 0x3f200000, bits64: 0x3fe4000000000000}, + {name: "parentheses", text: "(1.25)", bits32: 0x3fa00000, bits64: 0x3ff4000000000000}, + {name: "signedParentheses", text: "(-2.5)", bits32: 0xc0200000, bits64: 0xc004000000000000}, + {name: "roundEven", text: "1.000000059604644775390625", bits32: 0x3f800000, bits64: 0x3ff0000010000000}, + {name: "underflow32", text: "1e-50", bits64: 0x358dee7a4ad4b81f}, + {name: "overflow32", text: "1e40", bits32: 0x7f800000, bits64: 0x483d6329f1c35ca5}, + {name: "underscore", text: "1_000.0", bits32: 0x447a0000, bits64: 0x408f400000000000}, + {name: "nestedSign", text: "+(-(+1.25))", bits32: 0xbfa00000, bits64: 0xbff4000000000000}, + {name: "leadingDot", text: ".5", bits32: 0x3f000000, bits64: 0x3fe0000000000000}, + {name: "trailingDot", text: "1.", bits32: 0x3f800000, bits64: 0x3ff0000000000000}, + {name: "exponent", text: "1E+2", bits32: 0x42c80000, bits64: 0x4059000000000000}, + {name: "hexExponent", text: "0x1p-4", bits32: 0x3d800000, bits64: 0x3fb0000000000000}, + {name: "doubleNegativeZero", text: "--0.0"}, + {name: "unaryNegativeZero", text: "-(0.0)", bits32: 0x80000000, bits64: 0x8000000000000000}, + {name: "negativeUnderflow32", text: "-1e-50", bits32: 0x80000000, bits64: 0xb58dee7a4ad4b81f}, + {name: "negativeOverflow32", text: "-1e40", bits32: 0xff800000, bits64: 0xc83d6329f1c35ca5}, +} + +func TestX86FloatMulDivLiteralMathArchExp(t *testing.T) { + x86ScalarLiteralActualGoRewriteSet(t) + path := filepath.Join(runtime.GOROOT(), "src/math/exp_amd64.s") + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + before := sha256.Sum256(original) + row := "MULSD $0.0625, X0" + if bytes.Count(original, []byte(row)) != 2 { + t.Fatal("original math.archExp no longer contains both literal reductions") + } + // The source row is original. The surrounding first-party function has + // an explicit initialized scalar parameter/result, not guessed math ABI. + source := "TEXT ·Reduce(SB),4,$0-16\nMOVSD input+0(FP),X0\n" + row + "\nMOVSD X0,ret+8(FP)\nRET\n" + x86ScalarLiteralGoListing(t, "amd64", source) + pkg := x86MulDivLiteralPackage(t, "amd64", "func Reduce(input float64) float64\n") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}); err != nil { + t.Fatal(err) + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(after) != before { + t.Fatalf("actual original math source changed: %v", err) + } + t.Logf("original math.archExp source SHA256=%x; both reduction rows retain initialized scalar semantics", before) +} + +func x86MulDivLiteralPackage(t *testing.T, arch, declarations string) GoPackage { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "literal.go", "package literal\n"+declarations, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("literal", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "literal", Types: pkg, Syntax: []*ast.File{file}} +} + +func TestX86FloatMulDivLiteralActualGoObjects(t *testing.T) { + // This shared oracle authenticates the invoked SDK/assembler and hashes + // obj6/asm6/AVX tables before and after these original-source objects. + x86ScalarLiteralActualGoRewriteSet(t) + for _, name := range []string{"obj6.go", "asm6.go"} { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86", name) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if name == "obj6.go" { + if err := x86MulDivLiteralRewriteSource(data); err != nil { + t.Fatal(err) + } + } else { + for _, op := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + if !bytes.Contains(data, []byte("{A"+op+", yxm,")) { + t.Fatalf("actual Go no longer uses yxm for %s", op) + } + } + } + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + for _, op := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + for _, value := range x86MulDivLiteralValues { + for _, register := range []int{0, 7, 8, 15} { + row := fmt.Sprintf("%s $%s,X%d", op, value.text, register) + entry := fmt.Sprintf("TEXT literal%d(SB),4,$0-0\n%s\nRET\n", source.Len(), row) + if arch == "386" && register >= 8 { + x86ScalarLiteralExpectedGoListing(t, arch, entry, false) + continue + } + source.WriteString(entry) + } + } + } + listing, err := x86ScalarLiteralGoListing(t, arch, source.String()) + if err != nil { + t.Fatal(err) + } + for _, value := range x86MulDivLiteralValues { + for _, pool := range []struct { + name string + bits uint64 + width int + }{ + {fmt.Sprintf("$f32.%08x", value.bits32), uint64(value.bits32), 4}, + {fmt.Sprintf("$f64.%016x", value.bits64), value.bits64, 8}, + } { + if !strings.Contains(listing, pool.name) { + t.Fatalf("actual Go pool rounding/sign missing %s", pool.name) + } + start := strings.Index(listing, pool.name+" SRODATA") + if start < 0 { + t.Fatalf("actual Go pool has no original DATA payload: %s", pool.name) + } + var payload []byte + for _, line := range strings.Split(listing[start:], "\n")[1:] { + data := x86InsertExtractListedBytes(line) + if len(data) == 0 { + break + } + payload = append(payload, data...) + } + var want [8]byte + binary.LittleEndian.PutUint64(want[:], pool.bits) + if !bytes.Equal(payload, want[:pool.width]) { + t.Fatalf("actual original pool %s bytes=%x want=%x", pool.name, payload, want[:pool.width]) + } + } + } + if strings.Contains(listing, "XORPS") { + t.Fatal("arithmetic zero literal incorrectly acquired MOV zero rewrite") + } + } +} + +func TestX86FloatMulDivLiteralCallerSourceContracts(t *testing.T) { + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + move, result := "MOVSS", "float32" + if strings.HasSuffix(opcode, "SD") { + move, result = "MOVSD", "float64" + } + body := fmt.Sprintf("%s (DI),X15\n%s $0.0625,X15\n%s X15,(DX)", move, opcode, move) + for _, target := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(opcode+"/"+target, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, strings.ReplaceAll(x86CallerFixtureSource, "float32", result), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, target); err != nil { + t.Fatal(err) + } + }) + } + for name, unsafe := range map[string]string{ + "undefined low bytes": fmt.Sprintf("%s $1.0,X0\n%s X0,(DX)", opcode, move), + "pointer input": fmt.Sprintf("MOVQ DI,X0\n%s $1.0,X0\n%s X0,(DX)", opcode, move), + "private stack bits": fmt.Sprintf("MOVQ SP,X0\n%s $1.0,X0\n%s X0,(DX)", opcode, move), + "read-only input store": strings.Replace(body, "(DX)", "(DI)", 1), + "one-byte result overflow": strings.Replace(body, "(DX)", "1(DX)", 1), + "unproved source memory": strings.Replace(body, "(DI)", "(BX)", 1), + "FP pointer contents": fmt.Sprintf("%s a+0(FP),X0\n%s $1.0,X0\n%s X0,(DX)", move, opcode, move), + "join lacks input": fmt.Sprintf("CMPQ AX,$0\nJE joined\n%s (DI),X15\njoined:\n%s $1.0,X15\n%s X15,(DX)", move, opcode, move), + } { + t.Run(opcode+"/"+name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", unsafe, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, strings.ReplaceAll(x86CallerFixtureSource, "float32", result), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("literal invented source/extent/escape evidence: %v", err) + } + }) + } + } +} + +// Independently bind the actual encoder's complete scalar pool-rewrite body, +// not just the presence of an opcode name somewhere in the source file. +func x86MulDivLiteralRewriteSource(source []byte) error { + file, err := parser.ParseFile(token.NewFileSet(), "obj6.go", source, 0) + if err != nil { + return err + } + seen := make(map[string]int) + var progedit *ast.FuncDecl + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == "progedit" { + if progedit != nil || function.Body == nil { + return fmt.Errorf("missing or duplicate progedit body") + } + progedit = function + } + } + if progedit == nil { + return fmt.Errorf("actual Go progedit missing") + } + allowed := make(map[*ast.CaseClause]bool) + for _, statement := range progedit.Body.List { + switchStatement, ok := statement.(*ast.SwitchStmt) + if !ok || switchStatement.Init != nil { + continue + } + selector, ok := switchStatement.Tag.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "As" { + continue + } + base, ok := selector.X.(*ast.Ident) + if !ok || base.Name != "p" { + continue + } + for _, entry := range switchStatement.Body.List { + if clause, ok := entry.(*ast.CaseClause); ok { + allowed[clause] = true + } + } + } + var failure error + ast.Inspect(progedit.Body, func(node ast.Node) bool { + clause, ok := node.(*ast.CaseClause) + if !ok { + return true + } + for _, expression := range clause.List { + name, ok := expression.(*ast.Ident) + if !ok || name.Name != "AMULSS" && name.Name != "AMULSD" && name.Name != "ADIVSS" && name.Name != "ADIVSD" { + continue + } + seen[name.Name]++ + if !allowed[clause] { + failure = fmt.Errorf("nested or conditional literal rewrite %s", name.Name) + } + bits := 64 + if strings.HasSuffix(name.Name, "SS") { + bits = 32 + } + expected := fmt.Sprintf(`package p +func f() { +if p.From.Type == obj.TYPE_FCONST { + f%d := %s + p.From.Type = obj.TYPE_MEM + p.From.Name = obj.NAME_EXTERN + p.From.Sym = ctxt.Float%dSym(f%d) + p.From.Offset = 0 +} +}`, bits, map[int]string{32: "float32(p.From.Val.(float64))", 64: "p.From.Val.(float64)"}[bits], bits, bits) + fixture, err := parser.ParseFile(token.NewFileSet(), "expected.go", expected, 0) + if err != nil { + failure = err + continue + } + body, err := x86ScalarLiteralBodySyntax(clause.Body) + want, wantErr := x86ScalarLiteralBodySyntax(fixture.Decls[0].(*ast.FuncDecl).Body.List) + if err != nil || wantErr != nil || body != want { + failure = fmt.Errorf("unrecognized actual Go %s literal rewrite body", name.Name) + } + } + return true + }) + for _, name := range []string{"AMULSS", "AMULSD", "ADIVSS", "ADIVSD"} { + if seen[name] != 1 { + return fmt.Errorf("actual Go literal rewrite registration %s count=%d", name, seen[name]) + } + } + return failure +} + +func TestX86FloatMulDivLiteralRewriteSourceRejectsMutation(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/obj6.go") + source, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := x86MulDivLiteralRewriteSource(source); err != nil { + t.Fatal(err) + } + before := sha256.Sum256(source) + for _, replacement := range [][2]string{ + {"func progedit(", "func unrelated("}, + {"AMULSS,", "AMULPS,"}, {"ADIVSD,", "ADIVPD,"}, + {"ctxt.Float32Sym(f32)", "ctxt.Float64Sym(f32)"}, {"p.From.Offset = 0", "p.From.Offset = 4"}, + } { + mutated := bytes.ReplaceAll(source, []byte(replacement[0]), []byte(replacement[1])) + if bytes.Equal(mutated, source) || x86MulDivLiteralRewriteSource(mutated) == nil { + t.Fatalf("unknown source rewrite accepted: %q", replacement) + } + } + after, err := os.ReadFile(path) + if err != nil || sha256.Sum256(after) != before { + t.Fatalf("source mutation oracle changed the real SDK: %v", err) + } +} + +func TestX86FloatMulDivLiteralOriginalInstructionBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []struct { + opcode string + prefix, opcodeByte byte + }{ + {"MULSS", 0xf3, 0x59}, {"MULSD", 0xf2, 0x59}, + {"DIVSS", 0xf3, 0x5e}, {"DIVSD", 0xf2, 0x5e}, + } { + for _, literal := range []string{"0.0", "-0.0", "1.0", "0x1.4p0"} { + code := x86PackedFirstGoInstructionBytes(t, arch, row.opcode+" $"+literal+",X0") + if !bytes.Equal(code[:4], []byte{row.prefix, 0x0f, row.opcodeByte, 0x05}) { + t.Fatalf("%s %s literal %s original bytes=%x", arch, row.opcode, literal, code) + } + } + } + } +} + +func TestX86FloatMulDivLiteralCallerKeepsDefinedBytes(t *testing.T) { + for _, opcode := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + file, err := Parse(ArchAMD64, "TEXT literal(SB),4,$0-0\n"+opcode+" $-0.0,X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatMulDivProperties(ins, "amd64") + if !recognized || err != nil || !form.literal || form.memory != -1 || form.memoryBytes != 0 { + t.Fatalf("literal confused with caller memory: %+v / %v", form, err) + } + low := x86CallerVectorByteMask(int64(form.bits / 8)) + for _, old := range []uint64{0, low &^ 1, low, low | 0xf0000000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": old}, masks: map[Reg]bool{}} + before, _ := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFloatBinaryStep(&ins, form.x86FloatBinaryForm, &state, nil) + if accepted != (old&low == low) || !accepted && !reflect.DeepEqual(before, state.vectors) || accepted && state.vectors["Z0"] != old { + t.Fatalf("%s invented destination/upper knowledge: old=%x state=%x accepted=%t", opcode, old, state.vectors["Z0"], accepted) + } + } + // Source spelling is not permission to use a different typed value. + ins.Args[0].Imm++ + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + _, _, err = ctx.lowerFloatMulDiv(ins.Op, ins) + if err == nil || output.String() != before { + t.Fatal("mismatched literal/source bits emitted IR") + } + } +} + +func TestX86FloatMulDivLiteralThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var declarations, source strings.Builder + for _, op := range []string{"MULSS", "MULSD", "DIVSS", "DIVSD"} { + for _, value := range x86MulDivLiteralValues { + name := strings.ToLower(op) + value.name + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\nPXOR X0,X0\n%s $%s,X0\nRET\n", name, op, value.text) + } + } + pkg := x86MulDivLiteralPackage(t, target.arch, declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(sym string) string { return strings.Replace(sym, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var result *GoModuleTranslation + result, err = translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + ir = result.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mul-div-literal.ll", "mul-div-literal.o", ir) + x86MulDivLiteralOptimizedObject(t, llc, target.triple, ir) + }) + } + }) + } +} + +func TestX86FloatMulDivLiteralRejectsNonGoFormats(t *testing.T) { + for _, row := range []string{ + "MULSD $1,X0", "MULSS $0,X0", "DIVSD $0x3ff0000000000000,X0", + "MULSD $(1.0+2.0),X0", "DIVSS $(1.0/2.0),X0", "MULSS $1e999,X0", + "MULPS $1.0,X0", "MULPD $1.0,X0", "DIVPS $1.0,X0", "DIVPD $1.0,X0", + "VMULSS $1.0,X0,X1", "VMULSD $1.0,X0,X1", "VDIVSS $1.0,X0,X1", "VDIVSD $1.0,X0,X1", + "MULSD.Z $1.0,X0", "MULSS $1.0,Y0", "DIVSD $1.0,X16", "MULSD X0,$1.0", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT literal(SB),4,$0-0\n" + row + "\nRET\n" + x86ScalarLiteralExpectedGoListing(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerFloatMulDiv(ins.Op, ins) + if !known || term || err == nil || before != output.String() { + t.Fatalf("non-Go literal emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} + +// Keep this consumer's lifecycle inventory independent of the shared file, +// which has another writer. The actual original-Go test still uses the same +// bounded owned command helper as all other native numerical oracles. +func TestX86FloatMulDivLiteralCommandLifecycle(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_float_mul_div_literal_runtime_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + commands, consumers := 0, 0 + ast.Inspect(file, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if function, ok := call.Fun.(*ast.Ident); ok && function.Name == "runtimeOracleCombinedOutput" { + consumers++ + } + if method, ok := call.Fun.(*ast.SelectorExpr); ok { + if owner, ok := method.X.(*ast.Ident); ok && owner.Name == "exec" && method.Sel.Name == "CommandContext" { + commands++ + } + switch method.Sel.Name { + case "CombinedOutput", "Output", "Start", "Run": + if owner, ok := method.X.(*ast.Ident); !ok || owner.Name != "t" { + t.Errorf("literal oracle bypasses owned lifecycle: %s", method.Sel.Name) + } + } + } + return true + }) + if commands != 2 || consumers != 2 { + t.Fatalf("original-Go literal command lifecycle missing: commands=%d consumers=%d", commands, consumers) + } +} diff --git a/amd64_float_mul_div_mode32_test.go b/amd64_float_mul_div_mode32_test.go new file mode 100644 index 000000000..a8123ca5e --- /dev/null +++ b/amd64_float_mul_div_mode32_test.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86FloatMulDivMode32HighNamesStayUnlowered(t *testing.T) { + for _, opcode := range x86FloatMulDivOpcodes() { + if !strings.HasPrefix(opcode, "V") { + continue + } + widths := []string{"X", "Y", "Z"} + if strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") { + widths = widths[:1] + } + for _, width := range widths { + for _, index := range []int{8, 15, 16, 29, 31} { + row := fmt.Sprintf("%s %s%d,%s1,%s2", opcode, width, index, width, width) + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAccepted := width != "Z" + goAssemblerOracleOutput(t, "386", source, "", goAccepted) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + context, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + context.goarch = "386" + before := output.Len() + known, term, err := context.lowerFloatMulDiv(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !known || term || err == nil || goAccepted && !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("mode32 high bank acquired a physical value: %t/%t/%v", known, term, err) + } + }) + } + } + } +} + +func TestX86FloatMulDivMode32KeepsActualMXCSRInstructions(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 llc/llvm-objdump required") + } + var source strings.Builder + sigs := map[string]FuncSig{} + for _, opcode := range x86FloatMulDivOpcodes() { + if strings.HasSuffix(opcode, "PS") || strings.HasSuffix(opcode, "PD") { + continue + } + bits, typ := 32, I32 + load, resultOffset := "MOVSS", 8 + if strings.HasSuffix(opcode, "SD") { + bits, typ, load, resultOffset = 64, I64, "MOVSD", 16 + } + name := strings.ToLower(opcode) + row, destination := opcode+" X1,X0", "X0" + if strings.HasPrefix(opcode, "V") { + row, destination = opcode+" X1,X0,X2", "X2" + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-%d\n%s a+0(FP),X0\n%s b+%d(FP),X1\n%s\n%s %s,result+%d(FP)\nRET\n", + name, resultOffset+bits/8, load, load, bits/8, row, load, destination, resultOffset) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{typ, typ}, Ret: typ, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}, {Offset: int64(bits / 8), Type: typ, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: int64(resultOffset), Type: typ, Field: -1}}, + }} + } + goAssemblerOracleOutput(t, "386", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + input, object := filepath.Join(dir, "mode32.ll"), filepath.Join(dir, "mode32.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=obj", input, "-o", object) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual mode32 object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + command = exec.CommandContext(ctx, disassembler, "-d", object) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual mode32 machine instructions: %v\n%s", err, output) + } + for _, opcode := range []string{"mulss", "mulsd", "divss", "divsd"} { + if strings.Count(string(output), "\t"+opcode+"\t") != 2 { + t.Fatalf("%s lost %s/MXCSR to another FP environment:\n%s", triple, opcode, output) + } + } + if strings.Contains(string(output), "fmul") || strings.Contains(string(output), "fdiv") { + t.Fatalf("mode32 MUL/DIV used x87 instead of source MXCSR:\n%s", output) + } + } +} diff --git a/amd64_float_mul_div_numerical_test.go b/amd64_float_mul_div_numerical_test.go new file mode 100644 index 000000000..8f042cb27 --- /dev/null +++ b/amd64_float_mul_div_numerical_test.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The arithmetic reference is actual independent legacy SSE. No production +// opcode/rounding helper supplies expected numerical bits or MXCSR flags. +var x86FloatMulDivSSEOracle = strings.NewReplacer("subss", "divss", "subsd", "divsd", "addss", "mulss", "addsd", "mulsd").Replace(x86FloatAddSubSSEOracle) + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivDefaultPhysicalSemantics(t *testing.T) { + testX86FloatMulDivPhysicalSemantics(t, false) +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivFixedRoundPhysicalSemantics(t *testing.T) { + testX86FloatMulDivPhysicalSemantics(t, true) +} + +func testX86FloatMulDivPhysicalSemantics(t *testing.T, fixed bool) { + t.Helper() + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, row := range x86FloatMulDivFormats("amd64") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + form, _, err := x86FloatMulDivProperties(file.Funcs[0].Instrs[1], "amd64") + if err != nil { + t.Fatal(err) + } + if (form.rounding != "") != fixed { + continue + } + fixedMode := -1 + for mode, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + if form.rounding == suffix { + fixedMode = mode + } + } + name := fmt.Sprintf("binary%d", count) + count++ + row = strings.Replace(row, "8(AX)", "(BX)", 1) + row = strings.Replace(row, "(AX)", "(BX)", 1) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", name, row, map[bool]int{true: 1, false: 2}[form.spec.legacy]) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t);\n", name) + arithmeticBytes := form.byteWidth + if form.spec.scalar { + arithmeticBytes = form.spec.bits / 8 + } + opcode := strings.Split(strings.Fields(row)[0], ".")[0] + independentDivide := strings.Contains(opcode, "DIV") + aligned := form.legacy && !form.scalar && form.memory >= 0 + fmt.Fprintf(&checks, ` + for(int round=0;round<4;round++) for(int m=0;m<4;m++) for(int control=0;control<6;control++) for(int ambient=0;ambient<4;ambient++) { + uint8_t aStorage[96] __attribute__((aligned(16))),bStorage[68],out[67];uint8_t *a=aStorage+(%[13]d?16:1),*b=bStorage+3; + memset(aStorage,0xa7,96);memset(bStorage,0xb9,68);memset(out,0xc3,67); + for(int i=0;i<64/%[2]d;i++) { uint64_t av=%[3]d==32?UINT64_C(0x3f000001)+i*0x12345+round*0x10000:UINT64_C(0x3fe0000000000001)+i*0x123456789; + uint64_t bv=%[3]d==32?UINT64_C(0xbf800001)+i*0x12345:UINT64_C(0xbff0000000000001)+i*0x2345678; + if(round==1){av=%[3]d==32?UINT64_C(0x7fc12345)+i:UINT64_C(0x7ff8123456789abc)+i;bv=%[3]d==32?UINT64_C(0xffc45678)+i:UINT64_C(0xfff8fedcba987654)+i;} + if(round==2){av=%[3]d==32?UINT64_C(0x7f812345)+i:UINT64_C(0x7ff0123456789abc)+i;bv=%[3]d==32?UINT64_C(0xff800000):UINT64_C(0xfff0000000000000);} + if(round==3){av=1+i;bv=(%[3]d==32?UINT64_C(0x80000000):UINT64_C(0x8000000000000000))|i;} + memcpy(a+i*%[2]d,&av,%[2]d);memcpy(b+i*%[2]d,&bv,%[2]d); } + unsigned csr=controls[control]|(ambient<<13),observed,expectedFlags=0; + __asm__ volatile("ldmxcsr %%0"::"m"(csr)); + uint64_t mask=masks[m];%[1]s(out+1,a,b,mask); + __asm__ volatile("stmxcsr %%0":"=m"(observed)); + for(int i=0;i<64/%[2]d;i++) { + uint64_t av=0,bv=0,got=0;memcpy(&av,a+(%[4]d?0:i*%[2]d),%[2]d);memcpy(&bv,b+i*%[2]d,%[2]d);memcpy(&got,out+1+i*%[2]d,%[2]d); + int active=%[5]d<0 || ((mask>>i)&1); + unsigned operationCSR=%[12]d<0?csr:((csr&~0x6000u)|((unsigned)%[12]d<<13)); + uint64_t want=i*%[2]d<%[6]d?(active?sse_result(%[3]d,bv,av,%[7]d,operationCSR):(%[8]d?0:bv)):(%[9]d || (%[10]d && i*%[2]d<16)?bv:0); + if(i*%[2]d<%[6]d && active && %[12]d<0)expectedFlags|=last_sse_csr&63; + if(got!=want){fprintf(stderr,"%[1]s lane=%%d got=%%llx want=%%llx\n",i,(unsigned long long)got,(unsigned long long)want);return %[11]d;} + } + if(observed!=(csr|expectedFlags)){fprintf(stderr,"%[1]s sticky observed=%%x expected=%%x\n",observed,csr|expectedFlags);return 3;} + if(out[0]!=0xc3 || out[65]!=1 || out[66]!=0xc3 || a[-1]!=0xa7 || a[64]!=0xa7 || bStorage[2]!=0xb9 || bStorage[67]!=0xb9)return 2; + } +`, name, form.spec.bits/8, form.spec.bits, x86FloatAddSubBoolInt(form.broadcast), form.mask, arithmeticBytes, x86FloatAddSubBoolInt(independentDivide), x86FloatAddSubBoolInt(form.zeroing), x86FloatAddSubBoolInt(form.spec.legacy), x86FloatAddSubBoolInt(form.spec.scalar), count+10, fixedMode, x86FloatAddSubBoolInt(aligned)) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := x86FloatMulDivSSEOracle + declarations.String() + "int main(void){const uint64_t masks[]={0,1,0x55aa55aa55aa55aaULL,~0ULL}; const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0};\n" + checks.String() + "unsigned csr=0x1f80;__asm__ volatile(\"ldmxcsr %0\"::\"m\"(csr));return 0;}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "mul-div-default-semantics", triple, ir, main, prefix) + t.Logf("%d independent opcode/width/memory/mask programs x384 IEEE/MXCSR/canary inputs", count) +} diff --git a/amd64_float_mul_div_regression_test.go b/amd64_float_mul_div_regression_test.go new file mode 100644 index 000000000..57df156f8 --- /dev/null +++ b/amd64_float_mul_div_regression_test.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86GoCallerFloatMulDivOriginalSourceReachesObjects(t *testing.T) { + for _, opcode := range []string{"MULSS", "DIVSS", "VMULSS", "VDIVSS"} { + t.Run(opcode, func(t *testing.T) { + row := opcode + " X0,X1" + if strings.HasPrefix(opcode, "V") { + row += ",X2" + } + destination := "X1" + if strings.HasPrefix(opcode, "V") { + destination = "X2" + } + body := "VMOVSS (DI),X0\nVMOVSS (SI),X1\n" + row + "\nVMOVSS " + destination + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivConstantMXCSR(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for _, opcode := range []string{"MULSS", "DIVSS", "VMULSS", "VDIVSS"} { + name := strings.ToLower(opcode) + left, right := uint32(0x3f800001), uint32(0x3f800001) + if strings.Contains(opcode, "DIV") { + left, right = 0x3f800000, 0x40400000 + } + row, destination := opcode+" X1,X0", "X0" + if strings.HasPrefix(opcode, "V") { + row, destination = opcode+" X1,X0,X2", "X2" + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-4\nMOVL $0x%x,AX\nMOVD AX,X0\nMOVL $0x%x,AX\nMOVD AX,X1\n%s\nMOVSS %s,ret+0(FP)\nRET\n", name, left, right, row, destination) + sigs[name] = FuncSig{Name: name, Ret: I32, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Field: -1}}}} + fmt.Fprintf(&declarations, "extern uint32_t %s(void);\n", name) + fmt.Fprintf(&checks, "failed|=check(%s,0x%x,0x%x,%d,%q);\n", name, left, right, x86FloatAddSubBoolInt(strings.Contains(opcode, "DIV")), opcode) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const oracle = `#include +#include +#include +typedef uint32_t(*binary_constant)(void); +static int check(binary_constant fn,uint32_t left,uint32_t right,int divide,const char *name) { + unsigned saved; __asm__ volatile("stmxcsr %0":"=m"(saved)); + int failed=0; + for(unsigned mode=0;mode<4;mode++) { + unsigned csr=0x1f80|(mode<<13),wantCSR,gotCSR; + float a,b;memcpy(&a,&left,4);memcpy(&b,&right,4); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + if(divide)__asm__ volatile("divss %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("mulss %1,%0":"+x"(a):"x"(b)); + uint32_t want;memcpy(&want,&a,4); + __asm__ volatile("stmxcsr %0":"=m"(wantCSR)); + __asm__ volatile("ldmxcsr %0"::"m"(csr)); + uint32_t got=fn();__asm__ volatile("stmxcsr %0":"=m"(gotCSR)); + if(got!=want||gotCSR!=wantCSR) { + fprintf(stderr,"%s RC%u got=%x/%x original-SSE=%x/%x\n",name,mode,got,gotCSR,want,wantCSR); + failed=1; + } + } + __asm__ volatile("ldmxcsr %0"::"m"(saved));return failed; +} +` + main := oracle + declarations.String() + "int main(void){int failed=0;\n" + checks.String() + "return failed;}\n" + t.Run("original-ir", func(t *testing.T) { + compileAndRunRuntimeTestForTarget(t, llc, clang, "mul-div-constant-mxcsr", triple, ir, main, prefix) + }) + t.Run("optimized-ir", func(t *testing.T) { + opt := findLLVM22Tool("opt") + if opt == "" { + t.Fatal("LLVM22 opt missing") + } + dir := t.TempDir() + input, output := filepath.Join(dir, "source.ll"), filepath.Join(dir, "optimized.ll") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, opt, "-S", "-passes=default", input, "-o", output) + message, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("LLVM22 O2: %v\n%s", err, message) + } + optimized, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "mul-div-constant-mxcsr-o2", triple, string(optimized), main, prefix) + }) +} diff --git a/amd64_float_zero_mask_epoch_test.go b/amd64_float_zero_mask_epoch_test.go new file mode 100644 index 000000000..144f6120b --- /dev/null +++ b/amd64_float_zero_mask_epoch_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// The Go encoder tables do not express this helper rule. Go 1.20 could +// assemble EVEX.z=1/aaa=0; issue 57952 closed that bug in Go 1.21. Those +// historical objects are observations of invalid bytes, never legal forms. +func TestX86BinaryFloatZeroingRequiresNonzeroMaskAcrossAssemblerEpochs(t *testing.T) { + oldAssembler := !goToolchainAtLeast(runtime.Version(), 1, 21) + for _, arch := range []string{"386", "amd64"} { + for operation, opcode := range map[string]byte{ + "ADD": 0x58, "SUB": 0x5c, "MUL": 0x59, "DIV": 0x5e, + } { + for pp, suffix := range []string{"PS", "PD", "SS", "SD"} { + widths := []string{"X", "Y", "Z"} + if pp >= 2 { + widths = widths[:1] + } + for width, bank := range widths { + name := "V" + operation + suffix + row := fmt.Sprintf("%s.Z %s0,%s1,%s2", name, bank, bank, bank) + t.Run(arch+"/"+row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", oldAssembler) + + // Independent register encoding: source0, source1, + // destination2; EVEX.v'=1 and the requested vector length. + p1 := byte(0x74 | pp) + if pp&1 != 0 { + p1 |= 0x80 + } + code := []byte{0x62, 0xf1, p1, 0x88 | byte(width<<5), opcode, 0xd0} + if oldAssembler { + listing := x86InsertExtractGoListing(t, arch, source) + var observed []byte + for _, line := range strings.Split(string(listing), "\n") { + observed = append(observed, x86InsertExtractListedBytes(line)...) + } + want := append(append([]byte(nil), code...), 0xc3) + if !bytes.Equal(observed, want) { + t.Fatalf("historical assembler observation changed: got %x want %x", observed, want) + } + t.Logf("actual old Go emitted reserved z=1/aaa=0 bytes %x", code) + } + mode := 64 + if arch == "386" { + mode = 32 + } + if _, _, known, err := decodedX86EVEXBinaryFloatInstruction(code, 0, mode, "", false); !known || err == nil || + !strings.Contains(err.Error(), "zeroing requires a nonzero mask") { + t.Fatalf("reserved zeroing acquired raw semantics: known=%t err=%v", known, err) + } + x86RequireZeroMaskAPIOutcome(t, arch, source, false) + x86RequireZeroMaskAPIOutcome(t, arch, x86RawINSERTPSSource("f", code), false) + + explicit := fmt.Sprintf("%s.Z %s0,%s1,K0,%s2", name, bank, bank, bank) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+explicit+"\nRET\n", "", false) + x86RequireZeroMaskAPIOutcome(t, arch, "TEXT f(SB),4,$0-0\n"+explicit+"\nRET\n", false) + + if arch == "amd64" { + valid := fmt.Sprintf("%s.Z %s0,%s1,K1,%s2", name, bank, bank, bank) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+valid+"\nRET\n", "", true) + control := append([]byte(nil), code...) + control[3] |= 1 + ins, size, known, err := decodedX86EVEXBinaryFloatInstruction(control, 0, 64, "", false) + if !known || err != nil || size != len(control) || ins.Op != Op(name+".Z") || + len(ins.Args) != 4 || ins.Args[2].Reg != "K1" { + t.Fatalf("nonzero-mask control rejected: %+v size=%d known=%t err=%v", ins, size, known, err) + } + x86RequireZeroMaskAPIOutcome(t, arch, "TEXT f(SB),4,$0-0\n"+valid+"\nRET\n", true) + x86RequireZeroMaskAPIOutcome(t, arch, x86RawINSERTPSSource("f", control), true) + } + }) + } + } + } + } +} + +func x86RequireZeroMaskAPIOutcome(t *testing.T, arch, source string, wantSuccess bool) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal("fixture must reach the semantic validators", err) + } + triples := []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + if arch == "amd64" { + triples = []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + } + pkg := x86FloatMulDivShapesPackage(t, "package shapes\nfunc f()\n", arch) + for _, triple := range triples { + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}} + if ir, err := Translate(file, options); (err == nil) != wantSuccess || err == nil && ir == "" { + t.Fatalf("%s text API outcome wantSuccess=%t err=%v", triple, wantSuccess, err) + } + func() { + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if (err == nil) != wantSuccess || err == nil && module.IsNil() { + t.Fatalf("%s module API outcome wantSuccess=%t err=%v", triple, wantSuccess, err) + } + goSource := strings.Replace(source, "TEXT f(SB)", "TEXT ·f(SB)", 1) + result, err := translateGoModuleInContext(context, pkg, []byte(goSource), + GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if (err == nil) != wantSuccess || err == nil && (result == nil || result.Module.IsNil()) { + t.Fatalf("%s Go API outcome wantSuccess=%t err=%v", triple, wantSuccess, err) + } + }() + } +} diff --git a/amd64_fma3_forms.go b/amd64_fma3_forms.go new file mode 100644 index 000000000..c8c0d54b9 --- /dev/null +++ b/amd64_fma3_forms.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FMA3Form struct { + spec amd64FMA3Spec + suffix amd64FMA3Suffix + byteWidth int + memoryBytes int + destination int + mask int // -1 when unmasked + memory int // -1 for a register first source +} + +// The complete _yvaddpd/_yvaddsd axes are independent of numerical input, +// caller-memory, initialized-mask and source-stack continuation proofs. FP16 +// shares the architectural grammar but keeps its raw-encoding-only boundary. +func amd64FMA3Properties(ins Instr, goarch string) (amd64FMA3Form, bool, error) { + base, suffix, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := amd64FMA3Specs[Op(base)] + if !recognized { + return amd64FMA3Form{}, false, nil + } + form := amd64FMA3Form{ + spec: spec, destination: len(ins.Args) - 1, mask: -1, memory: -1, + } + fail := func() (amd64FMA3Form, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix are outside the FMA3 grammar: %q", goarch, base, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || spec.rawOnly && !ins.x86Encoded || + len(ins.Args) != 3 && len(ins.Args) != 4 { + return fail() + } + properties, valid := parseAMD64FMA3Suffix(suffix) + if !valid || spec.scalar && properties.broadcast { + return fail() + } + form.suffix = properties + if len(ins.Args) == 4 { + if goarch == "386" && !ins.x86Encoded || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + form.mask = 2 + } else if properties.zeroing { + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if spec.scalar && form.byteWidth != 16 || + !amd64GoPackedVectorMoveRegister(ins.Args[form.destination], form.byteWidth, goarch) || + !amd64GoPackedVectorMoveRegister(ins.Args[1], form.byteWidth, goarch) || + properties.rounding != "" && !spec.scalar && form.byteWidth != 64 { + return fail() + } + first := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(first) { + return fail() + } + } else if first.Kind == OpReg { + if !amd64GoPackedVectorMoveRegister(first, form.byteWidth, goarch) { + return fail() + } + } else if properties.rounding != "" || !isAMD64MemoryOperand(first) { + return fail() + } + if isAMD64MemoryOperand(first) { + form.memory = 0 + form.memoryBytes = form.byteWidth + if spec.scalar || properties.broadcast { + form.memoryBytes = spec.laneBits / 8 + } + } + return form, true, x86Mode32VectorRegisterContext(ins, goarch) +} diff --git a/amd64_fma3_forms_test.go b/amd64_fma3_forms_test.go new file mode 100644 index 000000000..72d74612c --- /dev/null +++ b/amd64_fma3_forms_test.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86FMA3GrammarMatchesEveryCurrentGoTableMember(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go"), nil, 0) + if err != nil { + t.Fatal(err) + } + rows := make(map[string]int) + wanted := make(map[Op]bool) + ast.Inspect(file, func(node ast.Node) bool { + if value, ok := node.(*ast.ValueSpec); ok && len(value.Names) == 1 && len(value.Values) == 1 { + name := value.Names[0].Name + if name == "_yvaddpd" || name == "_yvaddsd" { + literal, ok := value.Values[0].(*ast.CompositeLit) + if !ok { + t.Fatalf("Go FMA3 grammar changed shape: %s", name) + } + rows[name] = len(literal.Elts) + } + } + literal, ok := node.(*ast.CompositeLit) + if !ok { + return true + } + var opcode, table string + for _, item := range literal.Elts { + pair, ok := item.(*ast.KeyValueExpr) + if !ok { + continue + } + key, keyOK := pair.Key.(*ast.Ident) + value, valueOK := pair.Value.(*ast.Ident) + if !keyOK || !valueOK { + continue + } + if key.Name == "as" { + opcode = value.Name + } else if key.Name == "ytab" { + table = value.Name + } + } + if strings.HasPrefix(opcode, "AVFM") || strings.HasPrefix(opcode, "AVFNM") { + if table != "_yvaddpd" && table != "_yvaddsd" { + t.Fatalf("Go FMA3 acquired an unreviewed grammar: %s/%s", opcode, table) + } + wanted[Op(strings.TrimPrefix(opcode, "A"))] = table == "_yvaddsd" + } + return true + }) + if len(wanted) != 60 || !reflect.DeepEqual(rows, map[string]int{"_yvaddpd": 8, "_yvaddsd": 3}) { + t.Fatalf("current Go FMA3 family changed: opcodes=%d rows=%v", len(wanted), rows) + } + named := 0 + for opcode, spec := range amd64FMA3Specs { + if spec.rawOnly { + if _, exists := wanted[opcode]; exists { + t.Fatalf("raw-only opcode is now a Go named format: %s", opcode) + } + continue + } + scalar, exists := wanted[opcode] + if !exists || scalar != spec.scalar { + t.Fatalf("Go opcode omitted or changed: %s/%+v", opcode, spec) + } + named++ + } + if named != len(wanted) { + t.Fatalf("named family count mismatch: %d/%d", named, len(wanted)) + } +} + +// Independent opcode enumeration and format axes. Every form has a separate +// entry, so an earlier trap/return cannot remove a later LLVM object witness. +func TestX86FMA3FixedRoundingDefaultArchitecture(t *testing.T) { + const source = "TEXT defaultFMA(SB),4,$0-0\nVFMADD132SD.RD_SAE X0,X1,X2\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"defaultFMA": {Name: "defaultFMA", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc, _, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "default-fma.ll", "default-fma.o", ir) +} + +func x86FMA3NamedFormats(arch string) []string { + var instructions []string + ops := append(go127FMA3PackedOps(), go127FMA3ScalarOps()...) + for _, opcode := range ops { + scalar := strings.HasSuffix(opcode, "SS") || strings.HasSuffix(opcode, "SD") + banks := []string{"X", "Y", "Z"} + if scalar { + banks = banks[:1] + } + for _, bank := range banks { + for _, mode := range []string{"", "merge", "zero"} { + if arch == "386" && mode != "" { + continue + } + mask, zero := "", "" + if mode != "" { + mask = "K1," + } + if mode == "zero" { + zero = ".Z" + } + operands := bank + "0," + bank + "1," + mask + bank + "2" + memory := "(AX)," + bank + "1," + mask + bank + "2" + instructions = append(instructions, opcode+zero+" "+operands, opcode+zero+" "+memory) + if !scalar { + instructions = append(instructions, opcode+".BCST"+zero+" "+memory) + } + if scalar || bank == "Z" { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + instructions = append(instructions, opcode+"."+rounding+zero+" "+operands) + } + } + } + } + } + return instructions +} + +func TestX86FMA3GrammarAxesActualGoAndFiveLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, instruction := range x86FMA3NamedFormats(arch) { + name := fmt.Sprintf("fmaformat%d", index) + entry := fmt.Sprintf("TEXT %s(SB),4,$0-0\n%s\nRET\n", name, instruction) + file, err := Parse(ArchAMD64, entry) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := amd64FMA3Properties(ins, arch) + if err != nil || !recognized || form.spec.rawOnly || form.destination != len(ins.Args)-1 { + t.Fatalf("%s %s: incomplete grammar %+v/%t/%v", arch, instruction, form, recognized, err) + } + if form.memory >= 0 { + want := form.byteWidth + if strings.Contains(instruction, ".BCST") || strings.HasSuffix(strings.Split(string(ins.Op), ".")[0], "SS") || + strings.HasSuffix(strings.Split(string(ins.Op), ".")[0], "SD") { + want = 4 + if strings.Contains(string(ins.Op), "PD") || strings.Contains(string(ins.Op), "SD") { + want = 8 + } + } + if form.memory != 0 || form.memoryBytes != want { + t.Fatalf("%s changed native read extent: %+v; want %d", instruction, form, want) + } + } + source.WriteString(entry) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "fma-formats.ll", "fma-formats.o", ir) + } + t.Logf("%s: %d independently emitted original-Go format entries", arch, len(sigs)) + } +} + +func TestX86FMA3DispatchPreservesOriginalSourceIdentity(t *testing.T) { + for opcode, spec := range amd64FMA3Specs { + for _, identity := range []Op{"", opcode, Op(strings.ToLower(string(opcode)))} { + ins := Instr{Op: identity, x86Encoded: spec.rawOnly, Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}, + }} + original := ins + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + recognized, terminated, err := ctx.lowerFMA3(opcode, ins) + if !recognized || terminated || err != nil || !reflect.DeepEqual(ins, original) { + t.Fatalf("%s/%s legacy dispatch changed source/failed: %t/%t/%v", opcode, identity, recognized, terminated, err) + } + } + for _, conflict := range []Op{"VMOVUPS", "VFMADD132PS.BCST", opcode + ".Z"} { + ins := Instr{Op: conflict, x86Encoded: spec.rawOnly, Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}, + }} + original := ins + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + recognized, _, err := ctx.lowerFMA3(opcode, ins) + if !recognized || err == nil || output.String() != before || !reflect.DeepEqual(ins, original) { + t.Fatalf("%s/%s conflicting dispatch changed source/emitted IR: %t/%v", opcode, conflict, recognized, err) + } + } + } +} diff --git a/amd64_fma3_round_runtime_test.go b/amd64_fma3_round_runtime_test.go new file mode 100644 index 000000000..3adcddfd2 --- /dev/null +++ b/amd64_fma3_round_runtime_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// This separately exercises the production numerical helper on every host. +// Original instruction acceptance and caller/LLVM objects are independently +// covered by the complete Go-format oracle, not inferred from this helper. +func TestCrossLinuxRuntimeMatrixX86FMA3FixedRoundingExtremes(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + var ir strings.Builder + ir.WriteString("declare i128 @llvm.ctlz.i128(i128, i1)\n") + for _, bits := range []int{32, 64} { + floatType := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(&ir, "declare %s @llvm.fma.f%d(%s, %s, %s)\n", floatType, bits, floatType, floatType, floatType) + emitAMD64FMA3RoundingHelper(&ir, bits) + fmt.Fprintf(&ir, "define i%d @fma_round%d(i%d %%a, i%d %%b, i%d %%c, i32 %%mode, i32 %%mxcsr) {\n", bits, bits, bits, bits, bits) + for _, operand := range []string{"a", "b", "c"} { + fmt.Fprintf(&ir, " %%%s_float = bitcast i%d %%%s to %s\n", operand, bits, operand, floatType) + } + fmt.Fprintf(&ir, " %%value = call %s @__plan9asm_fma3_round_f%d(%s %%a_float, %s %%b_float, %s %%c_float, i32 %%mode, i32 %%mxcsr)\n", floatType, bits, floatType, floatType, floatType) + fmt.Fprintf(&ir, " %%bits = bitcast %s %%value to i%d\n ret i%d %%bits\n}\n", floatType, bits, bits) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + compileAndRunRuntimeTestForTarget(t, llc, clang, "fma3-fixed-round-extremes", triple, ir.String(), x86FMA3RoundingExtremeOracle, nil) +} + +const x86FMA3RoundingExtremeOracle = `#include +#include +#include +#include +#include +#pragma STDC FENV_ACCESS ON + +extern uint32_t fma_round32(uint32_t,uint32_t,uint32_t,uint32_t,uint32_t); +extern uint64_t fma_round64(uint64_t,uint64_t,uint64_t,uint32_t,uint32_t); + +static uint64_t state=UINT64_C(0x71b32dd678098deb); +static uint64_t random_bits(void) { + state^=state<<13; state^=state>>7; state^=state<<17; + return state; +} + +static uint64_t flush_input(uint64_t bits, uint64_t exponent, uint64_t sign, unsigned csr) { + return (csr&64) && !(bits&exponent) ? bits&sign : bits; +} + +static int check(unsigned width, uint64_t a, uint64_t b, uint64_t c, + unsigned mode, unsigned ambient, unsigned csr) { + const int modes[]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + uint64_t sign=width==32 ? UINT64_C(0x80000000) : UINT64_C(0x8000000000000000); + uint64_t exponent=width==32 ? UINT64_C(0x7f800000) : UINT64_C(0x7ff0000000000000); + uint64_t got, expected=0; + if (fesetround(modes[ambient])) return 1; + if (feclearexcept(FE_ALL_EXCEPT)) return 5; + if (width==32) got=fma_round32((uint32_t)a,(uint32_t)b,(uint32_t)c,mode,csr); + else got=fma_round64(a,b,c,mode,csr); + if (fegetround()!=modes[ambient]) return 2; + if (fetestexcept(FE_ALL_EXCEPT)) return 6; + a=flush_input(a,exponent,sign,csr); + b=flush_input(b,exponent,sign,csr); + c=flush_input(c,exponent,sign,csr); + if (fesetround(modes[mode])) return 3; + if (width==32) { + uint32_t aa=(uint32_t)a, bb=(uint32_t)b, cc=(uint32_t)c, result_bits; + float first, second, addend; + memcpy(&first,&aa,4); memcpy(&second,&bb,4); memcpy(&addend,&cc,4); + float result=fmaf(first,second,addend); + memcpy(&result_bits,&result,4); expected=result_bits; + } else { + double first, second, addend; + memcpy(&first,&a,8); memcpy(&second,&b,8); memcpy(&addend,&c,8); + double result=fma(first,second,addend); + memcpy(&expected,&result,8); + } + if ((csr&32768) && !(expected&exponent)) expected&=sign; + if (got!=expected) { + fprintf(stderr,"FMA rounding width/mode/ambient/csr %u/%u/%u/%x inputs=%llx,%llx,%llx got=%llx expected=%llx\n", + width,mode,ambient,csr,(unsigned long long)a,(unsigned long long)b, + (unsigned long long)c,(unsigned long long)got,(unsigned long long)expected); + return 4; + } + return 0; +} + +int main(void) { + const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x800000, + 0x800001,0x7f7fffff,0xff7fffff,0x3f000000,0xbf000000,0x3f800000, + 0xbf800000,0x3f800001,0xbf800002}; + const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1, + UINT64_C(0x8000000000000001),UINT64_C(0xfffffffffffff), + UINT64_C(0x10000000000000),UINT64_C(0x10000000000001), + UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff), + UINT64_C(0x3fe0000000000000),UINT64_C(0xbfe0000000000000), + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000), + UINT64_C(0x3ff0000000000001),UINT64_C(0xbff0000000000002)}; + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0}; + for (unsigned width=32; width<=64; width+=32) { + const uint64_t *edges=width==32 ? edges32 : edges64; + const uint64_t exponent=width==32 ? UINT64_C(0x7f800000) : UINT64_C(0x7ff0000000000000); + const uint64_t exponent_low=width==32 ? UINT64_C(0x800000) : UINT64_C(0x10000000000000); + for (unsigned i=0; i<5000; i++) { + uint64_t a, b, c; + if (i<3375) { + a=edges[i%15]; b=edges[(i/15)%15]; c=edges[(i/225)%15]; + } else { + a=random_bits(); b=random_bits(); c=random_bits(); + if (width==32) { a=(uint32_t)a; b=(uint32_t)b; c=(uint32_t)c; } + if ((a&exponent)==exponent) a^=exponent_low; + if ((b&exponent)==exponent) b^=exponent_low; + if ((c&exponent)==exponent) c^=exponent_low; + } + for (unsigned mode=0; mode<4; mode++) { + for (unsigned ambient=0; ambient<4; ambient++) { + for (unsigned control=0; control<4; control++) { + if (check(width,a,b,c,mode,ambient,controls[control])) return 1; + } + } + } + } + } + return fesetround(FE_TONEAREST) ? 1 : 0; +} +` diff --git a/amd64_fp_address.go b/amd64_fp_address.go new file mode 100644 index 000000000..0724d17be --- /dev/null +++ b/amd64_fp_address.go @@ -0,0 +1,56 @@ +package plan9asm + +import "fmt" + +// boundFPAddress addresses a declared field's backing storage, never its +// current value. An unbound source address needs a physical frame contract; +// inventing a zero pointer or extending an alloca into padding is not one. +func (c *amd64Ctx) boundFPAddress(off int64) (string, error) { + slot, result, ok := c.fpVectorSlot(off) + if !ok { + return "", fmt.Errorf("%w: %s FP address at +%d(FP) requires bound typed frame storage", ErrProbeNeedsContext, c.goarch, off) + } + if x86FrameScalarBytes(slot.Type, c.goarch) == 0 { + // The vector lookup's conservative aggregate-size fallback is not a + // storage-width proof. Scalar bool fields use canonical i8 backing. + return "", fmt.Errorf("%w: %s FP address at +%d(FP) lacks a scalar byte-storage contract for %s", ErrProbeNeedsContext, c.goarch, off, slot.Type) + } + if result { + c.markFPResultAddrTaken(slot.Offset) + } + var pointer string + if c.classicFrame != "" { + pointer = c.classicFramePtr(off) + } else { + if result { + pointer, _, ok = c.fpResultAlloca(slot.Offset) + } else { + pointer = c.fpParamAlloca[slot.Offset] + ok = pointer != "" + } + if !ok { + return "", fmt.Errorf("%w: %s FP address at +%d(FP) has no backing storage", ErrProbeNeedsContext, c.goarch, off) + } + if off != slot.Offset { + adjusted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr inbounds i8, ptr %s, i64 %d\n", adjusted, pointer, off-slot.Offset) + pointer = "%" + adjusted + } + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", address, pointer) + return "%" + address, nil +} + +func x86FrameScalarBytes(typ LLVMType, goarch string) int64 { + switch typ { + case I1, I8, I16, I32, I64, Ptr, LLVMType("float"), LLVMType("double"): + pointerSize := int64(8) + if goarch == "386" { + pointerSize = 4 + } + return frameTypeSize(typ, pointerSize) + default: + return 0 + } +} diff --git a/amd64_fp_address_test.go b/amd64_fp_address_test.go new file mode 100644 index 000000000..bd5124e6f --- /dev/null +++ b/amd64_fp_address_test.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86LEAFPAddressesRequireBoundStorage(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + widths := []string{"W", "L"} + if target.arch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + for _, prefix := range []string{"", "$"} { + for _, offset := range []int64{0, 1, 8, 9, 4, 12} { + t.Run(fmt.Sprintf("%s/%s/%s/%d", target.triple, width, prefix, offset), func(t *testing.T) { + source := fmt.Sprintf("TEXT Address(SB),4,$0-16\nLEA%s %sr+%d(FP),AX\nRET\n", width, prefix, offset) + requireX86GoAssemblerResult(t, target.arch, source, prefix == "") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Address": { + Name: "Address", Args: []LLVMType{I32}, Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I32, Index: 0, Field: -1, Name: "r"}}, + }, + }}, + } + ir, err := Translate(file, opt) + if prefix == "$" { + if err == nil { + t.Fatal("accepted TYPE_ADDR outside Go's LEA Ym/Yrl row") + } + return + } + if offset == 4 || offset == 12 { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unbound FP address became success/zero: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "ptrtoint ptr") { + t.Fatalf("LEA used the parameter value instead of its storage:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "fp-address.ll", "fp-address.o", ir) + }) + } + } + } + } +} + +func TestX86LEAFPAddressCannotInventScalarBacking(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + lea, triple := "LEAQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + lea, triple = "LEAL", "i386-unknown-linux-gnu" + } + for _, typ := range []LLVMType{"{ i8 }", "{}", "<8 x i8>"} { + t.Run(arch+"/"+string(typ), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT Address(SB),4,$0-16\n"+lea+" x+0(FP),AX\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"Address": { + Name: "Address", Args: []LLVMType{typ}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}}}, + }}, + }) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("address lacks a scalar/canonical-byte contract, got %v", err) + } + }) + } + } +} diff --git a/amd64_fp_bool_storage_test.go b/amd64_fp_bool_storage_test.go new file mode 100644 index 000000000..1a8981bb1 --- /dev/null +++ b/amd64_fp_bool_storage_test.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// A Go bool occupies one canonical byte in ABI0 memory, even though its LLVM +// argument and result have type i1. These sources exercise the actual byte +// backing, including indirect writes through LEA, not just the SSA value. +func x86BoolFrameSource(arch string) string { + lea, word := "LEAQ", 8 + if arch == "386" { + lea, word = "LEAL", 4 + } + source := fmt.Sprintf(`TEXT ·ReadByte(SB),4,$0-%d +%s x+0(FP),AX +MOVB (AX),BX +MOVB BX,ret+%d(FP) +RET +TEXT ·WriteParam(SB),4,$0-%d +%s x+0(FP),AX +MOVB $1,(AX) +MOVB x+0(FP),BX +MOVB BX,ret+%d(FP) +RET +TEXT ·WriteResult(SB),4,$0-%d +MOVB x+0(FP),BX +%s ret+%d(FP),AX +MOVB BX,(AX) +RET +TEXT ·WriteDirect(SB),4,$0-%d +MOVB $0,x+0(FP) +MOVB x+0(FP),BX +MOVB BX,ret+%d(FP) +RET +`, word+1, lea, word, word+1, lea, word, word+1, lea, word, word+1, word) + source += fmt.Sprintf(`TEXT ·BoolCallee(SB),4,$0-%d +%s x+0(FP),AX +MOVB (AX),BX +MOVB BX,ret+%d(FP) +RET +TEXT ·BoolCaller(SB),4,$16-%d +MOVB x+0(FP),AX +MOVB AX,0(SP) +CALL ·BoolCallee(SB) +MOVB %d(SP),AX +MOVB AX,ret+%d(FP) +RET +TEXT ·BoolField(SB),4,$0-%d +%s p_flag+1(FP),AX +MOVB (AX),BX +MOVB BX,ret+%d(FP) +RET +`, word+1, lea, word, word+1, word, word, word+1, lea, word) + return source +} + +func x86BoolFrameIR(t *testing.T, arch, triple string) string { + t.Helper() + source := x86BoolFrameSource(arch) + requireX86GoAssemblerResult(t, arch, source, true) + pkg := mustGoPackage(t, "example.com/boolframe", `package boolframe +func ReadByte(x bool) uint8 +func WriteParam(x bool) bool +func WriteResult(x bool) bool +func WriteDirect(x bool) bool +func BoolCallee(x bool) bool +func BoolCaller(x bool) bool +func BoolField(p struct { Pad uint8; Flag bool }) bool +`) + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + ir := translation.Module.String() + for _, unsafeBacking := range []string{ + "%fp_arg_0 = alloca i1", "%fp_ret_0 = alloca i1", + } { + if strings.Contains(ir, unsafeBacking) { + t.Fatalf("Go bool uses a sub-byte backing store: %s", unsafeBacking) + } + } + // This LLVM caller keeps the exact aggregate type. A C struct parameter + // would use C's ABI coercions and is not an oracle for this LLVM boundary. + return ir + ` +define i1 @BoolFieldWrapper(i1 %flag) { +entry: + %pad = insertvalue { i8, i1 } undef, i8 17, 0 + %input = insertvalue { i8, i1 } %pad, i1 %flag, 1 + %result = call i1 @BoolField({ i8, i1 } %input) + ret i1 %result +} +` +} + +func TestX86BoolFrameCanonicalByteObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + ir := x86BoolFrameIR(t, target.arch, target.triple) + compileLLVMToObject(t, llc, target.triple, "bool-frame.ll", "bool-frame.o", ir) + }) + } +} + +func x86BoolFrameGoOracle(t *testing.T, arch string, runPrefix []string) { + t.Helper() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module example.com/boolframe\n\ngo 1.20\n", + "bool_" + arch + ".s": x86BoolFrameSource(arch), + "bool_test.go": `package boolframe +import "testing" +func ReadByte(x bool) uint8 +func WriteParam(x bool) bool +func WriteResult(x bool) bool +func WriteDirect(x bool) bool +func BoolCallee(x bool) bool +func BoolCaller(x bool) bool +func BoolField(p struct { Pad uint8; Flag bool }) bool +func TestCanonicalBytes(t *testing.T) { + for i := 0; i < 1024; i++ { + for _, x := range []bool{false, true} { + want := uint8(0) + if x { want = 1 } + if ReadByte(x) != want || !WriteParam(x) || WriteResult(x) != x || WriteDirect(x) { + t.Fatalf("canonical Go bool frame failed for %v", x) + } + if BoolCallee(x) != x || BoolCaller(x) != x || BoolField(struct { Pad uint8; Flag bool }{17, x}) != x { + t.Fatalf("canonical Go bool call/aggregate failed for %v", x) + } + } + } +} +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + args := []string{"test", "-vet=off", "-count=1"} + if len(runPrefix) != 0 { + args = append(args, "-exec", strings.Join(runPrefix, " ")) + } + cmd := exec.Command("go", append(args, ".")...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOWORK=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0", "GOARCH="+arch) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("Go %s bool byte oracle: %v\n%s", arch, err, out) + } +} + +func TestCrossLinuxRuntimeMatrixX86BoolFrameCanonicalBytes(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("x86 byte ABI oracle runs in the required Linux/amd64 cross-runtime gate") + } + x86BoolFrameGoOracle(t, "amd64", nil) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var runPrefix []string + if crossRosetta { + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + const main = `#include +#include +extern uint8_t ReadByte(bool); +extern bool WriteParam(bool), WriteResult(bool), WriteDirect(bool); +extern bool BoolCallee(bool), BoolCaller(bool), BoolFieldWrapper(bool); +int main(void) { + for (unsigned i = 0; i < 1024; i++) { + for (unsigned x = 0; x < 2; x++) { + volatile uint64_t canary = UINT64_C(0x123456789abcdef0); + if (ReadByte(x) != x || !WriteParam(x) || WriteResult(x) != x || WriteDirect(x)) return 1; + if (canary != UINT64_C(0x123456789abcdef0)) return 2; + if (BoolCallee(x) != x || BoolCaller(x) != x || BoolFieldWrapper(x) != x) return 3; + } + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bool_frame", triple, + x86BoolFrameIR(t, "amd64", triple), main, runPrefix) + if os.Getenv("PLAN9ASM_CROSS_EXEC") == "1" { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("PLAN9ASM_CROSS_EXEC bool matrix requires Linux/amd64") + } + prefix := []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"} + x86BoolFrameGoOracle(t, "386", prefix) + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc", "-no-pie"}, + "bool_frame386", "i386-unknown-linux-gnu", x86BoolFrameIR(t, "386", "i386-unknown-linux-gnu"), main, prefix) + } +} diff --git a/amd64_fp_storage.go b/amd64_fp_storage.go new file mode 100644 index 000000000..8d8991f9a --- /dev/null +++ b/amd64_fp_storage.go @@ -0,0 +1,33 @@ +package plan9asm + +import "fmt" + +// Go ABI0 bool fields occupy a whole canonical byte. LLVM's i1 describes the +// logical argument, not byte-addressable storage: writing i1 alone does not +// establish the other seven bits for subsequent assembly byte loads. +func x86FPStorageType(typ LLVMType) LLVMType { + if typ == I1 { + return I8 + } + return typ +} + +func (c *amd64Ctx) storeFPStorage(typ LLVMType, value, pointer, alignment, metadata string) { + if typ == I1 { + byteValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i8\n", byteValue, value) + value = "%" + byteValue + } + fmt.Fprintf(c.b, " store %s %s, ptr %s%s%s\n", x86FPStorageType(typ), value, pointer, alignment, metadata) +} + +func (c *amd64Ctx) loadFPStorage(typ LLVMType, pointer, alignment string) string { + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s%s\n", loaded, x86FPStorageType(typ), pointer, alignment) + if typ == I1 { + logical := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i8 %%%s to i1\n", logical, loaded) + return "%" + logical + } + return "%" + loaded +} diff --git a/amd64_frame_vector.go b/amd64_frame_vector.go new file mode 100644 index 000000000..b0522622d --- /dev/null +++ b/amd64_frame_vector.go @@ -0,0 +1,196 @@ +package plan9asm + +import "fmt" + +// fpVectorSlot locates the scalar ABI0 field containing one byte of a vector +// memory operand. Aggregate parameters have several independently allocated +// fields on amd64; treating the first field as a contiguous pointer is wrong. +func (c *amd64Ctx) fpVectorSlot(off int64) (FrameSlot, bool, bool) { + for _, slot := range c.sig.Frame.Params { + size := x86FrameTypeSize(slot.Type) + if slot.Type == Ptr { + size = int64(goWordSize(c.goarch)) + } + if off >= slot.Offset && off < slot.Offset+size { + return slot, false, true + } + } + for _, slot := range c.fpResults { + size := x86FrameTypeSize(slot.Type) + if slot.Type == Ptr { + size = int64(goWordSize(c.goarch)) + } + if off >= slot.Offset && off < slot.Offset+size { + return slot, true, true + } + } + return FrameSlot{}, false, false +} + +func (c *amd64Ctx) validateFPVectorSpan(off int64, byteWidth int) error { + for i := 0; i < byteWidth; i++ { + if _, _, ok := c.fpVectorSlot(off + int64(i)); !ok { + return fmt.Errorf("undeclared FP vector byte at +%d(FP)", off+int64(i)) + } + } + return nil +} + +func (c *amd64Ctx) loadFPVectorBytes(off int64, byteWidth int) (string, error) { + if err := c.validateFPVectorSpan(off, byteWidth); err != nil { + return "", err + } + value := "zeroinitializer" + words := make(map[int64]string) + for i := 0; i < byteWidth; i++ { + slot, result, _ := c.fpVectorSlot(off + int64(i)) + word, found := words[slot.Offset] + if !found { + var scalar string + var err error + if result { + scalar, err = c.loadFPResult(slot) + } else { + scalar, err = c.loadFPParamValue(slot) + } + if err != nil { + return "", err + } + word, err = c.fpVectorScalarBits(slot.Type, scalar) + if err != nil { + return "", err + } + words[slot.Offset] = word + } + shift := (off + int64(i) - slot.Offset) * 8 + if shift != 0 { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", shifted, word, shift) + word = "%" + shifted + } + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", part, word) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, value, part, i) + value = "%" + inserted + } + return value, nil +} + +func (c *amd64Ctx) fpVectorScalarBits(typ LLVMType, scalar string) (string, error) { + switch typ { + case I64: + return scalar, nil + case Ptr: + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", converted, scalar) + return "%" + converted, nil + case LLVMType("double"): + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", converted, scalar) + return "%" + converted, nil + case LLVMType("float"): + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", bits, scalar) + scalar = "%" + bits + typ = I32 + } + bits, ok := amd64IntegerTypeBits(typ) + if !ok { + return "", fmt.Errorf("unsupported FP vector field type %s", typ) + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i64\n", wide, bits, scalar) + return "%" + wide, nil +} + +func (c *amd64Ctx) storeFPVectorBytes(off int64, byteWidth int, value, metadata string) error { + if err := c.validateFPVectorSpan(off, byteWidth); err != nil { + return err + } + if c.classicFrame != "" { + for i := 0; i < byteWidth; i++ { + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", part, byteWidth, value, i) + fmt.Fprintf(c.b, " store i8 %%%s, ptr %s, align 1%s\n", part, c.classicFramePtr(off+int64(i)), metadata) + if slot, result, ok := c.fpVectorSlot(off + int64(i)); ok && result { + c.markFPResultWritten(slot.Offset) + } + } + return nil + } + + updated := make(map[int64]string) + order := make([]FrameSlot, 0, byteWidth/4) + for i := 0; i < byteWidth; i++ { + current := off + int64(i) + slot, result, _ := c.fpVectorSlot(current) + bits, found := updated[slot.Offset] + if !found { + var scalar string + var err error + if result { + scalar, err = c.loadFPResult(slot) + } else { + scalar, err = c.loadFPParamValue(slot) + } + if err != nil { + return err + } + bits, err = c.fpVectorScalarBits(slot.Type, scalar) + if err != nil { + return err + } + order = append(order, slot) + } + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", part, byteWidth, value, i) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", wide, part) + shift := (current - slot.Offset) * 8 + inserted := "%" + wide + if shift != 0 { + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i64 %s, %d\n", tmp, inserted, shift) + inserted = "%" + tmp + } + cleared := c.newTmp() + mask := ^(uint64(0xff) << uint(shift)) + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", cleared, bits, mask) + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %s\n", merged, cleared, inserted) + updated[slot.Offset] = "%" + merged + } + for _, slot := range order { + bits := updated[slot.Offset] + converted := bits + switch slot.Type { + case I64: + case Ptr: + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", tmp, bits) + converted = "%" + tmp + case LLVMType("double"): + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", tmp, bits) + converted = "%" + tmp + case LLVMType("float"): + narrow, tmp := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", narrow, bits) + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", tmp, narrow) + converted = "%" + tmp + default: + width, ok := amd64IntegerTypeBits(slot.Type) + if !ok || width > 64 { + return fmt.Errorf("unsupported FP vector field type %s", slot.Type) + } + tmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", tmp, bits, width) + converted = "%" + tmp + } + if err := c.storeFPResultWithMetadata(slot.Offset, slot.Type, converted, metadata); err != nil { + return err + } + } + return nil +} diff --git a/amd64_go_assembler_test.go b/amd64_go_assembler_test.go new file mode 100644 index 000000000..16f2a8485 --- /dev/null +++ b/amd64_go_assembler_test.go @@ -0,0 +1,18 @@ +package plan9asm + +import ( + "runtime" + "testing" +) + +// requireX86GoAssemblerResult keeps the Go 1.27 table oracle available to +// tests that must remain visible to llgo. Older compatibility toolchains still +// exercise parsing, translation, and LLVM compilation, but cannot define the +// current operand grammar. +func requireX86GoAssemblerResult(t *testing.T, goarch, src string, wantSuccess bool) { + t.Helper() + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return + } + goAssemblerOracleOutput(t, goarch, src, "", wantSuccess) +} diff --git a/amd64_go_caller_acyclic_domain_test.go b/amd64_go_caller_acyclic_domain_test.go new file mode 100644 index 000000000..8ef5a191e --- /dev/null +++ b/amd64_go_caller_acyclic_domain_test.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "math" + "reflect" + "testing" +) + +func x86CallerJoinDomainState() (x86SourceStackState, *x86GoCallerRegionProof) { + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{ + {kind: x86CallerSliceInput, width: 4}, {kind: x86CallerSliceInput, width: 4}, + {kind: x86CallerFreshResult, width: 4}, {kind: x86CallerLocalLength, width: 8}, + }} + state := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, numericFlags: true, gOriginal: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}, SI: {argument: 1}, DX: {argument: 2}, CX: {argument: 3}}, + known: map[Reg]uint8{AX: 0xff}, vectors: map[Reg]uint64{"Z0": ^uint64(0)}, masks: map[Reg]bool{"K1": true}, + } + return state, proof +} + +func TestX86GoCallerAcyclicJoinDomainIntersectionAndNoAliasing(t *testing.T) { + left, proof := x86CallerJoinDomainState() + left.cells = []x86SourceStackCell{{offset: -8, width: 8, + bp: &x86SourceStackAddress{original: true}, caller: &x86CallerPointerValue{argument: 0}}} + right := x86CloneCallerState(left) + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: 4} + right.known[AX] = 1 + right.known[BX] = 0xff + right.vectors["Z0"] = 0xf + right.vectors["Z1"] = ^uint64(0) + delete(right.masks, "K1") + right.masks["K2"] = true + beforeLeft, beforeRight := x86CloneCallerState(left), x86CloneCallerState(right) + merged, valid := x86CallerAcyclicMerge(left, right, proof) + if !valid || merged.callers[DI] != (x86CallerPointerValue{argument: 0, span: 4}) || + merged.known[AX] != 1 || merged.known[BX] != 0 || merged.vectors["Z0"] != 0xf || + merged.vectors["Z1"] != 0 || merged.masks["K1"] || merged.masks["K2"] { + t.Fatalf("not an all-path same-origin intersection: %+v, %t", merged, valid) + } + reversed, valid := x86CallerAcyclicMerge(right, left, proof) + if !valid || !reflect.DeepEqual(merged, reversed) { + t.Fatal("merge depends on predecessor arrival order") + } + merged.callers[DI] = x86CallerPointerValue{} + merged.known[AX] = 0 + merged.vectors["Z0"] = 0 + merged.masks["K1"] = true + merged.cells[0].bp.offset = 7 + merged.cells[0].caller.offset = 7 + if !reflect.DeepEqual(left, beforeLeft) || !reflect.DeepEqual(right, beforeRight) { + t.Fatal("merged state aliases a predecessor map, cell or saved pointer") + } +} + +func TestX86GoCallerAcyclicJoinDomainRejectsChangedFixedOrigins(t *testing.T) { + for _, change := range []string{ + "missing origin", "different argument", "result offset", "length offset", "SP", "BP", "g", "flags", "saved cell", + "span on result", "negative span", "overflow span", "past offset cap", "past end cap", + } { + t.Run(change, func(t *testing.T) { + left, proof := x86CallerJoinDomainState() + right := x86CloneCallerState(left) + switch change { + case "missing origin": + delete(right.callers, DI) + case "different argument": + right.callers[DI] = x86CallerPointerValue{argument: 1} + case "result offset": + right.callers[DX] = x86CallerPointerValue{argument: 2, offset: 1} + case "length offset": + right.callers[CX] = x86CallerPointerValue{argument: 3, offset: 1} + case "SP": + right.sp = -8 + case "BP": + right.bp.original = false + case "g": + right.gOriginal = false + case "flags": + right.numericFlags = false + case "saved cell": + right.cells = []x86SourceStackCell{{offset: -8, width: 8}} + case "span on result": + left.callers[DX] = x86CallerPointerValue{argument: 2, span: 1} + right.callers[DX] = left.callers[DX] + case "negative span": + right.callers[DI] = x86CallerPointerValue{argument: 0, span: -1} + case "overflow span": + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: 1, span: math.MaxInt64} + case "past offset cap": + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: -x86SourceStackLimit - 1} + case "past end cap": + right.callers[DI] = x86CallerPointerValue{argument: 0, offset: x86SourceStackLimit, span: 1} + } + if _, valid := x86CallerAcyclicMerge(left, right, proof); valid { + t.Fatal("changed fixed state or invalid span acquired a join") + } + }) + } +} + +func TestX86GoCallerAcyclicJoinPointerArithmeticBounds(t *testing.T) { + for _, test := range []struct { + pointer x86CallerPointerValue + delta int64 + valid bool + }{ + {x86CallerPointerValue{offset: -4, span: 8}, 4, true}, + {x86CallerPointerValue{offset: -x86SourceStackLimit, span: 2 * x86SourceStackLimit}, 0, true}, + {x86CallerPointerValue{offset: x86SourceStackLimit - 4, span: 4}, 1, false}, + {x86CallerPointerValue{offset: -x86SourceStackLimit, span: 4}, -1, false}, + {x86CallerPointerValue{offset: 1, span: 4}, math.MaxInt64, false}, + {x86CallerPointerValue{offset: -1, span: 4}, math.MinInt64, false}, + {x86CallerPointerValue{offset: math.MaxInt64, span: 1}, 0, false}, + } { + shifted, valid := x86CallerPointerShift(test.pointer, test.delta) + if valid != test.valid || valid && shifted.span != test.pointer.span { + t.Fatalf("shift %+v by %d: %+v/%t", test.pointer, test.delta, shifted, valid) + } + } +} + +func TestX86GoCallerAcyclicJoinDomainCannotRecoverUnknownFacts(t *testing.T) { + left, proof := x86CallerJoinDomainState() + unknown := x86CloneCallerState(left) + delete(unknown.known, AX) + delete(unknown.vectors, "Z0") + delete(unknown.masks, "K1") + joined, valid := x86CallerAcyclicMerge(left, unknown, proof) + if !valid { + t.Fatal("same origins with unknown numerical bytes should intersect") + } + joined, valid = x86CallerAcyclicMerge(joined, left, proof) + if !valid || joined.known[AX] != 0 || joined.vectors["Z0"] != 0 || joined.masks["K1"] { + t.Fatal("later predecessor recovered an erased all-path fact") + } +} + +func TestX86GoCallerAcyclicJoinDomainAllPredecessorPermutations(t *testing.T) { + first, proof := x86CallerJoinDomainState() + second, third := x86CloneCallerState(first), x86CloneCallerState(first) + second.callers[DI] = x86CallerPointerValue{argument: 0, offset: -4} + third.callers[DI] = x86CallerPointerValue{argument: 0, offset: 12} + second.known[AX] = 0xf + third.known[AX] = 3 + second.vectors["Z0"] = 0xff + third.vectors["Z0"] = 0xf + second.known[BX], third.known[BX] = 0xff, 0xff + second.vectors["Z1"], third.vectors["Z1"] = 0xff, 0xff + second.masks["K2"], third.masks["K2"] = true, true + third.masks["K1"] = false + states := []x86SourceStackState{first, second, third} + var expected *x86SourceStackState + for _, order := range [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}} { + joined, valid := x86CallerAcyclicMerge(states[order[0]], states[order[1]], proof) + if valid { + joined, valid = x86CallerAcyclicMerge(joined, states[order[2]], proof) + } + if !valid || joined.callers[DI] != (x86CallerPointerValue{argument: 0, offset: -4, span: 16}) || + joined.known[AX] != 3 || joined.known[BX] != 0 || joined.vectors["Z0"] != 0xf || + joined.vectors["Z1"] != 0 || joined.masks["K1"] || joined.masks["K2"] { + t.Fatalf("permutation %v recovered a missing fact: %+v/%t", order, joined, valid) + } + if expected != nil && !reflect.DeepEqual(*expected, joined) { + t.Fatalf("three-predecessor merge depends on order %v", order) + } + copy := x86CloneCallerState(joined) + expected = © + } +} + +func TestX86GoCallerAcyclicJoinGraphRequiresCompleteReachableDAG(t *testing.T) { + for _, test := range []struct { + name, body string + acyclic bool + }{ + {"fallthrough", "MOVL $1,AX\nRET", true}, + {"diamond", "CMPQ AX,$0\nJE other\nMOVL $1,AX\nJMP done\nother:\nMOVL $2,AX\ndone:\nRET", true}, + {"dead cycle", "JMP done\ndead:\nJMP dead\ndone:\nRET", true}, + {"reachable cycle", "loop:\nCMPQ AX,$0\nJNE loop\nRET", false}, + {"cycle beyond join", "CMPQ AX,$0\nJE joined\nMOVL $1,AX\njoined:\nloop:\nJMP loop", false}, + {"PC forward", "JMP 1(PC)\nRET", true}, + {"PC self", "JMP 0(PC)", false}, + {"PC out of source", "JMP 3(PC)\nRET", false}, + {"unknown indirect target", "JMP AX\nRET", false}, + {"unknown label", "JMP missing\nRET", false}, + {"fall off source", "MOVL $1,AX", false}, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·graph(SB),4,$0-0\n"+test.body+"\n") + if err != nil { + t.Fatal(err) + } + fn := file.Funcs[0] + labels := map[string]int{} + var instructions []int + for index, ins := range fn.Instrs { + if ins.Op == OpLABEL { + labels[ins.Args[0].Sym] = len(instructions) + } else { + instructions = append(instructions, index) + } + } + if got := x86CallerReachableAcyclic(fn, instructions, labels); got != test.acyclic { + t.Fatalf("reachable graph acyclic=%t, want %t", got, test.acyclic) + } + }) + } + if x86CallerReachableAcyclic(Func{}, nil, nil) || + x86CallerReachableAcyclic(Func{}, make([]int, 8193), nil) { + t.Fatal("empty or over-budget source acquired a new merge") + } +} diff --git a/amd64_go_caller_acyclic_join.go b/amd64_go_caller_acyclic_join.go new file mode 100644 index 000000000..82c687f7f --- /dev/null +++ b/amd64_go_caller_acyclic_join.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "math" + "reflect" +) + +// A span is an abstract set of offsets from the same authenticated input. +// It is not an allocation extent or a promise that an unsafe read cannot fault. +func x86CallerPointerEnd(pointer x86CallerPointerValue) (int64, bool) { + if pointer.span < 0 || pointer.offset > math.MaxInt64-pointer.span { + return 0, false + } + end := pointer.offset + pointer.span + return end, pointer.offset >= -x86SourceStackLimit && end <= x86SourceStackLimit +} + +func x86CallerPointerShift(pointer x86CallerPointerValue, delta int64) (x86CallerPointerValue, bool) { + if _, valid := x86CallerPointerEnd(pointer); !valid || + delta > 0 && pointer.offset > math.MaxInt64-delta || + delta < 0 && pointer.offset < math.MinInt64-delta { + return pointer, false + } + pointer.offset += delta + _, valid := x86CallerPointerEnd(pointer) + return pointer, valid +} + +func x86CloneCallerState(state x86SourceStackState) x86SourceStackState { + if state.cells != nil { + cells := make([]x86SourceStackCell, len(state.cells)) + copy(cells, state.cells) + state.cells = cells + } + for index := range state.cells { + if state.cells[index].bp != nil { + value := *state.cells[index].bp + state.cells[index].bp = &value + } + if state.cells[index].caller != nil { + value := *state.cells[index].caller + state.cells[index].caller = &value + } + } + state.callers = x86CloneCallerPointers(state.callers) + state.known = x86CloneCallerKnown(state.known) + state.vectors, state.masks = x86CloneCallerVectorState(state.vectors, state.masks) + return state +} + +// Only numerical definedness and same-origin read-only input offsets merge. +// Neither a missing origin nor a fixed local, stack or continuation can borrow +// this domain. Return a fresh deep clone, never edit either predecessor. +func x86CallerAcyclicMerge(left, right x86SourceStackState, proof *x86GoCallerRegionProof) (x86SourceStackState, bool) { + if proof == nil || left.sp != right.sp || left.bp != right.bp || + left.numericFlags != right.numericFlags || left.partialNumericFlags != right.partialNumericFlags || left.gOriginal != right.gOriginal || + !reflect.DeepEqual(left.cells, right.cells) || len(left.callers) != len(right.callers) { + return x86SourceStackState{}, false + } + merged := x86CloneCallerState(left) + for reg, a := range left.callers { + b, present := right.callers[reg] + if !present || a.argument != b.argument || a.argument < 0 || a.argument >= len(proof.arguments) { + return x86SourceStackState{}, false + } + aEnd, aValid := x86CallerPointerEnd(a) + bEnd, bValid := x86CallerPointerEnd(b) + if !aValid || !bValid { + return x86SourceStackState{}, false + } + if proof.arguments[a.argument].kind != x86CallerSliceInput { + if a != b || a.span != 0 { + return x86SourceStackState{}, false + } + continue + } + if b.offset < a.offset { + a.offset = b.offset + } + if bEnd > aEnd { + aEnd = bEnd + } + a.span = aEnd - a.offset // Both endpoints already have the 1 MiB bound. + merged.callers[reg] = a + } + // Include absent facts as zero, including on a later predecessor. Retain + // zero entries so replays do not depend on which predecessor arrived first. + for reg, value := range right.known { + merged.known[reg] = left.known[reg] & value + } + for reg, value := range left.known { + merged.known[reg] = value & right.known[reg] + } + for reg, value := range right.vectors { + merged.vectors[reg] = left.vectors[reg] & value + } + for reg, value := range left.vectors { + merged.vectors[reg] = value & right.vectors[reg] + } + for reg, value := range right.masks { + merged.masks[reg] = left.masks[reg] && value + } + for reg, value := range left.masks { + merged.masks[reg] = value && right.masks[reg] + } + return merged, true +} + +// A new merge cannot establish any cyclic invariant. This deliberately checks +// the entire reachable graph, not only the current edge or label's spelling. +func x86CallerReachableAcyclic(fn Func, instructions []int, labels map[string]int) bool { + if len(instructions) == 0 || len(instructions) > 8192 { + return false + } + edges := make(map[int][]int) + seen := make(map[int]bool) + work := []int{0} + for len(work) != 0 { + pc := work[0] + work = work[1:] + if pc < 0 || pc >= len(instructions) { + return false + } + if seen[pc] { + continue + } + seen[pc] = true + ins := fn.Instrs[instructions[pc]] + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpRET { + continue + } + next := []int{pc + 1} + if op == OpJMP || isAMD64ConditionalBranch(op) { + if len(ins.Args) != 1 { + return false + } + operand := ins.Args[0] + label := operand.Sym + if operand.Kind == OpIdent { + label = operand.Ident + } + target, valid := labels[label] + if operand.Kind == OpMem && operand.Mem.Base == PC && operand.Mem.Index == "" && + operand.Mem.Sym == "" && operand.Mem.Off >= -int64(len(instructions)) && operand.Mem.Off <= int64(len(instructions)) { + target, valid = pc+int(operand.Mem.Off), true + } else if operand.Kind != OpLabel && operand.Kind != OpIdent { + return false + } + if !valid { + return false + } + if op == OpJMP { + next = []int{target} + } else { + next = append(next, target) + } + } + edges[pc] = next + work = append(work, next...) + } + indegree := make(map[int]int, len(seen)) + for _, next := range edges { + for _, pc := range next { + indegree[pc]++ + } + } + work = work[:0] + for pc := range seen { + if indegree[pc] == 0 { + work = append(work, pc) + } + } + removed := 0 + for len(work) != 0 { + pc := work[0] + work = work[1:] + removed++ + for _, next := range edges[pc] { + indegree[next]-- + if indegree[next] == 0 { + work = append(work, next) + } + } + } + return removed == len(seen) +} diff --git a/amd64_go_caller_acyclic_join_test.go b/amd64_go_caller_acyclic_join_test.go new file mode 100644 index 000000000..b75d08d67 --- /dev/null +++ b/amd64_go_caller_acyclic_join_test.go @@ -0,0 +1,273 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerAcyclicJoinAssembly(body string) string { + return strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) +} + +func TestX86GoCallerAcyclicJoinActualSourceInputs(t *testing.T) { + for name, body := range map[string]string{ + "same input offsets": "MOVQ (CX),AX\nCMPQ AX,$1\nJE other\nADDQ $4,DI\nCMPQ AX,$1\nother:\nMOVL (DI),AX\nMOVL AX,(DX)", + "copy and affine range": "MOVQ (CX),AX\nCMPQ AX,$1\nJE other\nADDQ $4,DI\nCMPQ AX,$1\nother:\nMOVQ DI,BX\nLEAQ 4(BX),R11\nSUBQ $4,R11\nMOVL (R11),AX\nMOVL AX,(DX)", + "all path low GP bytes": "MOVQ (CX),AX\nCMPQ AX,$1\nJE narrow\nMOVL $42,BX\nJMP joined\nnarrow:\nMOVB $42,BL\njoined:\nMOVB BL,(DX)", + "unused SIMD definition": "VMOVUPS (DI),X0\nMOVQ (CX),AX\nCMPQ AX,$1\nJE joined\nVMOVUPS (SI),X1\njoined:\nVMOVSS X0,(DX)", + "unused mask definition": "MOVQ (CX),AX\nCMPQ AX,$1\nJE joined\nKMOVQ AX,K1\njoined:\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatalf("complete actual source acyclic join %s: %v", triple, err) + } + } + }) + } +} + +func TestX86GoCallerCountedLoopOriginalBackedgeNeedsFiniteProof(t *testing.T) { + // These are unchanged consecutive original directives and the actual + // backedge in Weaviate dot_avx256_amd64.s. The acyclic mechanism still cannot + // prove them; the new private finite replay proves only this bounded region, + // not the complete external function or its later unbounded loops. + body := `MOVQ (CX),R9 +LONG $0xff418d45 +LONG $0x03c1f641 +JE done +WORD $0x8944; BYTE $0xc8 +WORD $0xe083; BYTE $0x03 +LONG $0xc057f8c5 +WORD $0xc931 +LBB0_4: +LONG $0x0f10fac5 +LONG $0xb971e2c4; BYTE $0x06 +LONG $0x04c78348 +LONG $0x04c68348 +LONG $0x01c18348 +WORD $0xc839 +JNE LBB0_4 +done: +MOVL $7,AX +MOVL AX,(DX)` + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + var instructions []int + labels := map[string]int{} + for index, ins := range decoded.Instrs { + if ins.Op == OpLABEL { + labels[ins.Args[0].Sym] = len(instructions) + } else { + instructions = append(instructions, index) + } + } + if x86CallerReachableAcyclic(decoded, instructions, labels) { + t.Fatal("a natural backedge acquired DAG admission") + } + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("source-bound finite replay of only the first region: %v", err) + } + // Preserve the bounded but outside-budget variant as a real source Context, + // without hiding its original directives or adding a wrapper length check. + outsideBudget := strings.Replace(assembly, "WORD $0xe083; BYTE $0x03", "WORD $0xe083; BYTE $0x07", 1) + unsafePkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, outsideBudget) + if err := x86CallerCoreTranslate(t, unsafePkg, outsideBudget, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bound outside finite mechanism admitted: %v", err) + } +} + +func TestX86GoCallerAcyclicJoinRetainsUnknownPermissions(t *testing.T) { + for name, body := range map[string]string{ + "different base": "CMPQ AX,$0\nJE joined\nMOVQ SI,DI\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + "lost pointer": "CMPQ AX,$0\nJE joined\nMOVQ $0,DI\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + "readonly write": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVL AX,(DI)", + "result offset": "CMPQ AX,$0\nJE joined\nADDQ $1,DX\nCMPQ AX,$0\njoined:\nMOVB AX,(DX)", + "length offset": "CMPQ AX,$0\nJE joined\nADDQ $1,CX\nCMPQ AX,$0\njoined:\nMOVL (CX),AX\nMOVL AX,(DX)", + "upper range cap": "CMPQ AX,$0\nJE joined\nADDQ $1048576,DI\nCMPQ AX,$0\njoined:\nLEAQ 1(DI),BX\nMOVL (BX),AX\nMOVL AX,(DX)", + "partial pointer": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVW DI,BX\nMOVL (BX),AX\nMOVL AX,(DX)", + "truncated pointer": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nLEAL (DI),DI\nMOVL (DI),AX\nMOVL AX,(DX)", + "stack escape": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVQ DI,SP\nMOVL AX,(DX)", + "input escapes as data": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVQ DI,(DX)", + "unknown length": "MOVQ R11,AX\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nMOVL AX,(DX)", + "late unknown GP": "CMPQ AX,$0\nJE missing\nMOVQ $7,BX\nJMP joined\nmissing:\nMOVB $7,BL\njoined:\nMOVQ BX,AX\nMOVL AX,(DX)", + "late unknown vector": "CMPQ AX,$0\nJE missing\nVMOVUPS (DI),Y0\nJMP joined\nmissing:\nMOVSS (DI),X0\njoined:\nVEXTRACTF128 $1,Y0,X1\nVMOVSS X1,(DX)", + "late unknown mask": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nKMOVQ AX,K1\njoined:\nVPADDD.Z Z0,Z0,K1,Z1\nVMOVSS X1,(DX)", + "mask0 cannot grant source": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nVPADDD.Z (DX),Z0,K1,Z1\nVMOVSS X1,(DX)", + "stale address flags": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + "same state loop cannot borrow span": "CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nCMPQ AX,$0\nJNE joined\nMOVL (DI),AX\nMOVL AX,(DX)", + "unknown dead instruction": "JMP joined\nRDTSC\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown permission acquired a finite input span: %v", err) + } + }) + } +} + +func TestX86GoCallerAcyclicJoinRevokesProvisionalCertificatesForLateUnknown(t *testing.T) { + // The short predecessor can reach and certify the input load before the + // other predecessor arrives. A longer path then removes numerical bytes; + // neither the early certificate nor arrival order may admit the function. + for _, knownFirst := range []bool{true, false} { + fast, slow := "MOVQ $7,BX", "MOVB $7,BL" + if !knownFirst { + fast, slow = slow, fast + } + body := "CMPQ AX,$0\nJE slow\n" + fast + "\nJMP joined\nslow:\n" + + "MOVL $1,R10\nMOVL $2,R10\nMOVL $3,R10\nMOVL $4,R10\n" + slow + + "\njoined:\nMOVL (DI),AX\nMOVQ BX,AX\nMOVL AX,(DX)" + assembly := x86CallerAcyclicJoinAssembly(body) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + normalized := normalizeX86RawReturnSourceStack(decoded, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs, + }, true, file.Funcs[0]) + if normalized.x86SourceStack != nil || !strings.Contains(normalized.x86SourceStackFailure, "unproved caller-region effect: MOVQ") { + t.Fatalf("late unknown path acquired source permission: %s", normalized.x86SourceStackFailure) + } + for _, ins := range normalized.Instrs { + if len(ins.x86CallerMemory) != 0 || len(ins.x86SourceStackFP) != 0 { + t.Fatal("failed final source proof leaked a provisional instruction certificate") + } + } + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("late unknown predecessor was hidden by a prior proof: %v", err) + } + } +} + +func TestX86GoCallerAcyclicJoinFinalCertificatesAndSourceMutations(t *testing.T) { + body := "MOVQ (CX),AX\nCMPQ AX,$1\nJE joined\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)" + for _, change := range []string{"none", "access width", "operand", "missing caller", "actual caller bytes"} { + t.Run(change, func(t *testing.T) { + assembly := x86CallerAcyclicJoinAssembly(body) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil || normalized.Funcs[0].x86SourceStack == nil { + t.Fatalf("complete final source proof: %v", err) + } + found := false + for index := range normalized.Funcs[0].Instrs { + ins := &normalized.Funcs[0].Instrs[index] + if ins.Op != "MOVL" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem { + continue + } + found = true + if !x86CallerMemoryCertified(*ins, 0) { + t.Fatal("final merged input read lacks a recomputed complete-width certificate") + } + if change == "access width" { + access := ins.x86CallerMemory[0] + access.width++ + ins.x86CallerMemory[0] = access + if x86CallerMemoryCertified(*ins, 0) || x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("changed provisional access acquired final admission") + } + // Final normalization recaptures the unchanged original native + // read, rather than inheriting the changed private certificate. + } else if change == "operand" { + ins.Args[0].Mem.Base = DX + } + } + if !found { + t.Fatal("missing final original input load") + } + if change == "missing caller" { + sig := sigs["caller.kernel"] + sig.x86CallerRegions = nil + sigs[sig.Name] = sig + } + if change == "actual caller bytes" { + pkg.SourceSelection.GoSources["caller.go"] = []byte(x86CallerFixtureSource + "\nvar exposed = kernel\n") + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed producer caller source retained its original permission: %v", err) + } + return + } + _, err = Translate(normalized, options) + if change == "none" || change == "access width" { + if err != nil { + t.Fatal(err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("edited %s acquired a final caller proof: %v", change, err) + } + }) + } +} + +func TestX86GoCallerAcyclicJoinFiveTargetsRetain386Boundary(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + if target.arch == "amd64" { + assembly := x86CallerAcyclicJoinAssembly("CMPQ AX,$0\nJE joined\nADDQ $4,DI\nCMPQ AX,$0\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)") + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, target.triple); err != nil { + t.Fatal(err) + } + return + } + // The actual caller producer remains AMD64-only. A 386 object is + // the existing independent scalar ABI0 control, not a copied span. + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "386"); !errors.Is(err, ErrProbeNeedsContext) || proof != nil { + t.Fatal("AMD64 input-origin proof was borrowed by mode32") + } + source, scalarSig, _ := x86SourceStackNumericalFixture("386", "@MOV @WORD(SP),AX\nCMPL AX,$0\nJE done\nADDL $7,AX\ndone:\n@MOV AX,@RESULT(SP)\nBYTE $0xc3") + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions("386", target.triple, scalarSig)) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mode32-control.ll", "mode32-control.o", module.String()) + }) + } +} diff --git a/amd64_go_caller_acyclic_runtime_test.go b/amd64_go_caller_acyclic_runtime_test.go new file mode 100644 index 000000000..3a8290e32 --- /dev/null +++ b/amd64_go_caller_acyclic_runtime_test.go @@ -0,0 +1,144 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerAcyclicRuntimeSources() (goSource, assembly, goTest, cOracle string) { + forms := []struct { + name, body string + width int + }{ + {"inputjoin", "CMPQ AX,$1\nJE joined\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", 4}, + {"copyjoin", "CMPQ AX,$1\nJE joined\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVQ DI,BX\nLEAQ 8(BX),R11\nSUBQ $8,R11\nMOVL (R11),AX\nMOVL AX,(DX)", 4}, + {"reversedjoin", "CMPQ AX,$1\nJNE changed\nJMP joined\nchanged:\nADDQ $4,DI\nCMPQ AX,$1\njoined:\nMOVL (DI),AX\nMOVL AX,(DX)", 4}, + {"lowjoin", "CMPQ AX,$1\nJE narrow\nMOVL $42,BX\nJMP joined\nnarrow:\nMOVB $42,BL\njoined:\nMOVB BL,(DX)", 1}, + } + var declarations, machine, tests, c strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + tests.WriteString(`package caller +import "testing" +func TestOriginalAcyclicCallerValues(t *testing.T) { + values := []uint32{0,1,7,0x80000000,0xffffffff,0x12345678} + for _, value := range values { + for length:=1; length<=4; length++ { + x := []uint32{value,value^0xaabbccdd,19,23} + y := []uint32{31,37,41,43} + originalX, originalY := [4]uint32{x[0],x[1],x[2],x[3]}, [4]uint32{31,37,41,43} + canary := [4]uint64{11,17,29,53} + want := x[0] + if length!=1 { want=x[1] } +`) + c.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range forms { + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint32") + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", form.name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+form.name+"(", 1) + declarations.WriteString(declaration) + source := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+form.name) + source = strings.Replace(source, "MOVL $7,AX", "MOVQ (CX),AX", 1) + machine.WriteString(strings.Replace(source, "MOVL AX,(DX)", form.body, 1)) + want := "want" + if form.width == 1 { + want = "uint32(42)" + } + fmt.Fprintf(&tests, ` + if got:=Call%s(x[:length],y); got!=%s || + [4]uint32{x[0],x[1],x[2],x[3]}!=originalX || [4]uint32{y[0],y[1],y[2],y[3]}!=originalY || + canary!=[4]uint64{11,17,29,53} { t.Fatalf("%s length=%%d got=%%#x",length,got) } +`, form.name, want, form.name) + fmt.Fprintf(&c, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + } + tests.WriteString(" }\n }\n}\n") + c.WriteString(`int main(void) { + const uint32_t values[]={0,1,7,0x80000000u,0xffffffffu,0x12345678u}; + for (unsigned i=0;i= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := int64(form.byteWidth) + if form.scalar { + width = int64(form.bits / 8) + } + required := x86CallerVectorByteMask(width) + if form.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + for _, index := range []int{form.first, form.second} { + if form.literal && index == form.first { + continue + } + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + // A merely initialized mask does not prove any lane active. Merge needs + // the old numerical lane(s), zeroing does not; neither invents inputs. + if form.mask >= 0 && !form.zeroing && old&required != required { + return false + } + known := ^uint64(0) + if form.legacy { + known = old | required + } else if form.scalar { + upperX := x86CallerVectorByteMask(16) &^ required + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] + known = ^x86CallerVectorByteMask(16) | required | second&upperX + } + state.vectors[destination] = known + return true +} diff --git a/amd64_go_caller_add_sub_test.go b/amd64_go_caller_add_sub_test.go new file mode 100644 index 000000000..f085f46a6 --- /dev/null +++ b/amd64_go_caller_add_sub_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerFloatAddSubOriginalSourceReachesObjects(t *testing.T) { + for _, fixture := range []struct { + name string + body string + }{ + {"original VADDPS LONG", "VMOVUPS (DI),Y2\nVMOVUPS (SI),Y3\nLONG $0xd358ecc5\nVMOVSS X2,(DX)"}, + {"named VADDPS", "VMOVUPS (DI),Y2\nVMOVUPS (SI),Y3\nVADDPS Y3,Y2,Y2\nVMOVSS X2,(DX)"}, + {"original VSUBSS family", "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVSUBSS X0,X1,X2\nVMOVSS X2,(DX)"}, + } { + t.Run(fixture.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", fixture.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerFloatAddSubWholeFamilyByteEffects(t *testing.T) { + for _, row := range x86FloatAddSubFormats("amd64") { + if strings.Contains(row, "(AX)") || strings.Contains(row, "8(AX)") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatAddSubProperties(ins, "amd64") + if !recognized || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("missing shared family effect %s: %v", row, err) + } + for _, missing := range []int{-1, form.first, form.second, form.destination, 4} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + if missing >= 0 && missing < len(ins.Args) && ins.Args[missing].Kind == OpReg { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= 1 + } + if missing == 4 { + state.masks["K1"] = false + } + before, beforeMasks := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFloatAddSubStep(&ins, form, &state, nil) + want := missing == -1 || missing == form.destination && !form.spec.legacy && (form.mask < 0 || form.zeroing) || missing == 4 && form.mask < 0 + if accepted != want || !reflect.DeepEqual(state.masks, beforeMasks) || !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing %d: %t want %t, invented numerical/mask evidence", row, missing, accepted, want) + } + } + width := int64(form.byteWidth) + if form.spec.scalar { + width = int64(form.spec.bits / 8) + } + low := x86CallerVectorByteMask(width) + for _, old := range []uint64{0, 0xff00, 0xff00000000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": low, "Z1": low | old, "Z2": low | old}, masks: map[Reg]bool{"K1": true}} + if !x86CallerFloatAddSubStep(&ins, form, &state, nil) { + t.Fatalf("low arithmetic requires unrelated unknown bytes: %s", row) + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + want := ^uint64(0) + if form.spec.legacy { + want = low | old + } else if form.spec.scalar { + want = ^x86CallerVectorByteMask(16) | low | old&(x86CallerVectorByteMask(16)&^low) + } + if state.vectors[destination] != want { + t.Fatalf("%s physical byte definitions %x want %x", row, state.vectors[destination], want) + } + } + } +} + +func TestX86GoCallerFloatAddSubDoesNotInventValuesOrCallerPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown arithmetic source": "VADDPS X0,X1,X2\nVMOVSS X2,(DX)", + "SUB is not an integer zero idiom": "VSUBPS X0,X0,X2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVUPS (DI),X0\nADDPS X0,X1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ AX,K1\nVADDPS X0,X1,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVADDPS.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "mask pointer bits": "VMOVUPS (DI),X0\nKMOVQ DI,K1\nVSUBPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "unknown memory": "VMOVUPS (DI),X0\nVADDPS (BX),X0,X2\nVMOVSS X2,(DX)", + "physical stack": "VMOVUPS (DI),X0\nVADDPS 0(SP),X0,X2\nVMOVSS X2,(DX)", + "FP pointer contents": "VMOVUPS (DI),X0\nVADDPS a+0(FP),X0,X2\nVMOVSS X2,(DX)", + "read length as vector": "VMOVUPS (DI),X0\nVADDPS (CX),X0,X2\nVMOVSS X2,(DX)", + "read g": "VMOVUPS (DI),X0\nVADDPS (R14),X0,X2\nVMOVSS X2,(DX)", + "wide output": "VMOVUPS (DI),X0\nVADDPS X0,X0,X2\nVMOVUPS X2,(DX)", + "write input": "VMOVUPS (DI),X0\nVADDPS X0,X0,X2\nVMOVSS X2,(DI)", + "unknown CFG join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVADDPS X0,X0,X2\nVMOVSS X2,(DX)", + "scalar unknown upper reused": "VMOVSS (DI),X0\nVADDSS X0,X3,X2\nVADDPS X2,X2,X2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved add/sub source passed: %v", err) + } + }) + } +} + +func TestX86GoCallerFloatAddSubAllOpcodesInitializedMemory(t *testing.T) { + for _, opcode := range x86FloatAddSubOpcodes() { + t.Run(opcode, func(t *testing.T) { + args := "(DI),X1" + destination := "X1" + if strings.HasPrefix(opcode, "V") { + args += ",X2" + destination = "X2" + } + body := fmt.Sprintf("VMOVUPS (SI),Z1\n%s %s\nVMOVSS %s,(DX)", opcode, args, destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_aes.go b/amd64_go_caller_aes.go new file mode 100644 index 000000000..f502016f5 --- /dev/null +++ b/amd64_go_caller_aes.go @@ -0,0 +1,24 @@ +package plan9asm + +func x86CallerAESByteStep(ins *Instr, f x86AESForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + width := x86CallerVectorByteMask(int64(f.byteWidth)) + source := state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] & width + if f.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.byteWidth), false) { + return false + } + source = width + } + input := uint64(0) + if f.state >= 0 { + input = state.vectors[x86CallerVectorRoot(ins.Args[f.state].Reg)] & width + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + defined := x86AESDefinedBytes(f, source, input) + if !f.spec.vector { + state.vectors[destination] = state.vectors[destination]&^width | defined + } else { + state.vectors[destination] = ^width | defined + } + return true +} diff --git a/amd64_go_caller_binding.go b/amd64_go_caller_binding.go new file mode 100644 index 000000000..339cf9377 --- /dev/null +++ b/amd64_go_caller_binding.go @@ -0,0 +1,48 @@ +package plan9asm + +import "fmt" + +// GoX86RawReturnCallerBindingNeeded is only a loader cost predicate. A true +// result proves neither a caller region nor a valid source continuation. +func GoX86RawReturnCallerBindingNeeded(file *File, goarch string, resolve func(string) string, sigs map[string]FuncSig) bool { + if file == nil || goarch != "amd64" || resolve == nil { + return false + } + for _, fn := range file.Funcs { + if x86GoCallerRegionCandidate(fn, sigs[resolve(goTextSymbolForResolution(fn.Sym))]) { + return true + } + } + return false +} + +// BindGoX86RawReturnCallerRegions consumes the compiler's complete actual +// typed package selection. It returns new signatures with private source +// evidence; the whole-source normalizer must still prove every stack, memory +// effect and return. Pointer/noescape types or JSON hashes are insufficient. +func BindGoX86RawReturnCallerRegions(pkg GoPackage, file *File, goarch string, resolve func(string) string, sigs map[string]FuncSig) (map[string]FuncSig, error) { + if !GoX86RawReturnCallerBindingNeeded(file, goarch, resolve, sigs) { + return sigs, nil + } + if pkg.SourceSelection == nil { + return nil, fmt.Errorf("%w: bounded raw-return caller binding requires actual selected source", ErrProbeNeedsContext) + } + bound := make(map[string]FuncSig, len(sigs)) + for name, sig := range sigs { + bound[name] = sig + } + for _, fn := range file.Funcs { + name := resolve(goTextSymbolForResolution(fn.Sym)) + sig := bound[name] + if !x86GoCallerRegionCandidate(fn, sig) { + continue + } + proof, err := deriveX86GoCallerRegions(pkg, *pkg.SourceSelection, fn, sig, goarch) + if err != nil { + return nil, err + } + sig.x86CallerRegions = proof + bound[name] = sig + } + return bound, nil +} diff --git a/amd64_go_caller_binding_test.go b/amd64_go_caller_binding_test.go new file mode 100644 index 000000000..30b05c53a --- /dev/null +++ b/amd64_go_caller_binding_test.go @@ -0,0 +1,43 @@ +package plan9asm + +import ( + "errors" + "go/types" + "strings" + "testing" +) + +func TestX86GoCallerRegionBindingRequiresCompleteTypedSelection(t *testing.T) { + for _, change := range []string{"none", "missing selection", "missing sizes", "wrong sizes", "changed source", "missing assembly"} { + t.Run(change, func(t *testing.T) { + pkg, selected, _, sig := x86CallerFixture(t, x86CallerFixtureSource) + file := selected.Assembly["kernel_amd64.s"] + pkg.SourceSelection = &selected + sigs := map[string]FuncSig{sig.Name: sig} + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + switch change { + case "missing selection": + pkg.SourceSelection = nil + case "missing sizes": + selected.Sizes = nil + case "wrong sizes": + selected.Sizes = types.SizesFor("gc", "386") + case "changed source": + selected.GoSources["caller.go"] = []byte(x86CallerFixtureSource + "\nvar exposed = kernel\n") + case "missing assembly": + selected.SelectedAssemblyFiles = nil + } + bound, err := BindGoX86RawReturnCallerRegions(pkg, file, "amd64", resolve, sigs) + if change == "none" { + if err != nil || bound[sig.Name].x86CallerRegions == nil { + t.Fatalf("actual selection acquired no private binding: %v", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || bound != nil { + t.Fatalf("%s obtained complete/partial binding: %+v, %v", change, bound, err) + } + if sigs[sig.Name].x86CallerRegions != nil { + t.Fatal("caller-owned signature map was mutated") + } + }) + } +} diff --git a/amd64_go_caller_counted_loop.go b/amd64_go_caller_counted_loop.go new file mode 100644 index 000000000..776c2ecd0 --- /dev/null +++ b/amd64_go_caller_counted_loop.go @@ -0,0 +1,477 @@ +package plan9asm + +import ( + "fmt" + "math" + "reflect" + "strings" +) + +// This private plan partitions numerical low bits, not allocation bounds. It +// never supplies a pointer, initial GP value, public option or serialized proof. +type x86CallerCountedPlan struct { + edges [][]int + loops []x86CallerCountedLoop + headers, latches map[int]int + original, normalized [32]byte +} + +type x86CallerCountedLoop struct { + body map[int]bool + header, latch, update int + counter, bound Reg +} + +func x86CallerCountedEdges(fn Func, instructions []int, labels map[string]int) ([][]int, bool) { + if len(instructions) == 0 || len(instructions) > 8192 { + return nil, false + } + edges := make([][]int, len(instructions)) + for pc, index := range instructions { + ins := fn.Instrs[index] + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpRET { + continue + } + edges[pc] = []int{pc + 1} + if op == OpJMP || isAMD64ConditionalBranch(op) { + if len(ins.Args) != 1 { + return nil, false + } + arg := ins.Args[0] + label := arg.Sym + if arg.Kind == OpIdent { + label = arg.Ident + } + target, valid := labels[label] + if arg.Kind == OpMem && arg.Mem.Base == PC && arg.Mem.Index == "" && arg.Mem.Sym == "" && + arg.Mem.Off >= -int64(len(instructions)) && arg.Mem.Off <= int64(len(instructions)) { + target, valid = pc+int(arg.Mem.Off), true + } else if arg.Kind != OpLabel && arg.Kind != OpIdent { + return nil, false + } + if !valid { + return nil, false + } + if op == OpJMP { + edges[pc] = []int{target} + } else { + edges[pc] = append(edges[pc], target) + } + } + for _, next := range edges[pc] { + if next < 0 || next >= len(instructions) { + return nil, false + } + } + } + return edges, true +} + +// Each SCC must retain the original one-loop rule: one entry edge, one exact +// adjacent comparison/latch, an immutable bound and one unavoidable +1 update. +// Its remaining edges must be a DAG, not an inferred cyclic merge invariant. +func deriveX86CallerCountedLoop(fn Func, instructions []int, edges [][]int, reachable []bool, + body map[int]bool, header, latchPC int) *x86CallerCountedLoop { + plan := &x86CallerCountedLoop{body: body, header: header, latch: latchPC, update: -1} + if plan.latch == 0 || !plan.body[plan.latch-1] { + return nil + } + latch := fn.Instrs[instructions[plan.latch]] + compare := fn.Instrs[instructions[plan.latch-1]] + if latch.Op != "JNE" || len(edges[plan.latch]) != 2 || edges[plan.latch][1] != plan.header || + (compare.Op != "CMPL" && compare.Op != "CMPQ") || len(compare.Args) != 2 || + compare.Args[0].Kind != OpReg || compare.Args[1].Kind != OpReg { + return nil + } + left, leftOK := x86CallerGP(compare.Args[0].Reg) + right, rightOK := x86CallerGP(compare.Args[1].Reg) + if !leftOK || !rightOK || left == right || left != compare.Args[0].Reg || right != compare.Args[1].Reg { + return nil + } + entries, updates := 0, 0 + for pc, nexts := range edges { + if !reachable[pc] { + continue + } + for _, next := range nexts { + if next == plan.latch && pc != plan.latch-1 { + return nil // The latch must consume this exact adjacent comparison. + } + if !plan.body[pc] && plan.body[next] { + if next != plan.header { + return nil + } + entries++ + } + if plan.body[pc] && !plan.body[next] && pc != plan.latch { + return nil + } + } + if !plan.body[pc] { + continue + } + ins := fn.Instrs[instructions[pc]] + if (ins.Op == "ADDL" || ins.Op == "ADDQ") && len(ins.Args) == 2 && + ins.Args[0].Kind == OpImm && ins.Args[0].ImmRaw == "" && ins.Args[0].Imm == 1 && + ins.Args[1].Kind == OpReg && (ins.Args[1].Reg == left || ins.Args[1].Reg == right) { + updates++ + plan.update, plan.counter = pc, ins.Args[1].Reg + } + } + if entries != 1 || updates != 1 { + return nil + } + plan.bound = left + if plan.counter == left { + plan.bound = right + } + // Every body path must pass the one update; both live values stay immutable + // otherwise. Conservatively reject even an unmodeled read of these registers. + for pc := range plan.body { + ins := fn.Instrs[instructions[pc]] + if pc == plan.update || ins.Op == "CMPL" || ins.Op == "CMPQ" || isAMD64ConditionalBranch(ins.Op) || ins.Op == OpJMP { + continue + } + for _, arg := range ins.Args { + if arg.Kind == OpReg { + root, _ := x86CallerGP(arg.Reg) + if root == plan.counter || root == plan.bound { + return nil + } + } + } + } + seen := map[int]bool{} + work := []int{plan.header} + for len(work) != 0 { + pc := work[0] + work = work[1:] + if pc == plan.update || seen[pc] { + continue + } + if pc == plan.latch { + return nil + } + seen[pc] = true + work = append(work, edges[pc]...) + } + return plan +} + +type x86CallerCountedNumber struct { + source, version int + mask, value uint64 + known bool +} + +type x86CallerCountedFlags struct { + pc int + op Op + regs [2]Reg + versions [2]int + equal, known bool +} + +type x86CallerCountedNumbers struct { + regs map[Reg]x86CallerCountedNumber + flags x86CallerCountedFlags + source, sequence int +} + +func (facts x86CallerCountedNumbers) clone() x86CallerCountedNumbers { + copy := facts + copy.regs = make(map[Reg]x86CallerCountedNumber, len(facts.regs)) + for reg, value := range facts.regs { + copy.regs[reg] = value + } + return copy +} + +// Relational evidence is independent of numericFlags. Unknown upper bits stay +// unknown; only a real readonly local-length load supplies the partition source. +func (facts *x86CallerCountedNumbers) step(pc, residue int, ins Instr, before x86SourceStackState, proof *x86GoCallerRegionProof) { + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpTEXT || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) { + return + } + facts.sequence++ + if (op == "CMPL" || op == "CMPQ" || op == "TESTB" || op == "TESTW" || op == "TESTL" || op == "TESTQ") && len(ins.Args) == 2 { + facts.flags = x86CallerCountedFlags{pc: pc, op: op} + values := [2]x86CallerCountedNumber{} + for index, arg := range ins.Args { + if arg.Kind == OpImm && arg.ImmRaw == "" { + values[index] = x86CallerCountedNumber{source: -1, value: uint64(arg.Imm), known: true} + } else if arg.Kind == OpReg { + root, valid := x86CallerGP(arg.Reg) + if !valid || arg.Reg != root { + return + } + var present bool + values[index], present = facts.regs[root] + if !present { + return + } + facts.flags.regs[index], facts.flags.versions[index] = root, values[index].version + } else { + return + } + } + if strings.HasPrefix(string(op), "TEST") { + for index := range values { + other := values[1-index] + if !values[index].known && values[index].source == facts.source && facts.source >= 0 && other.known && other.value <= 3 { + values[index].value, values[index].known = uint64(residue), true + } + } + mask := uint64(math.MaxUint64) + switch op { + case "TESTB": + mask = math.MaxUint8 + case "TESTW": + mask = math.MaxUint16 + case "TESTL": + mask = math.MaxUint32 + } + facts.flags.equal = values[0].value&values[1].value&mask == 0 + } else { + if op == "CMPL" { + values[0].value, values[1].value = uint64(uint32(values[0].value)), uint64(uint32(values[1].value)) + } + facts.flags.equal = values[0].value == values[1].value + } + facts.flags.known = values[0].known && values[1].known + return + } + move := op == "MOVL" || op == "MOVQ" + _, conditionalMove, conditionalMoveError := x86ConditionalMoveProperties(op, ins, "amd64") + if !move && !(conditionalMove && conditionalMoveError == nil) && x86LEAWidth(op, "amd64") == 0 && op != "PUSHQ" && op != "POPQ" && + !strings.HasPrefix(string(op), "MOV") && !strings.HasPrefix(string(op), "VMOV") { + facts.flags = x86CallerCountedFlags{} + } + if len(ins.Args) == 0 || ins.Args[len(ins.Args)-1].Kind != OpReg { + return + } + arg := ins.Args[len(ins.Args)-1] + reg, valid := x86CallerGP(arg.Reg) + if !valid { + return + } + old, oldPresent := facts.regs[reg] + delete(facts.regs, reg) + if arg.Reg != reg || len(ins.Args) != 2 { + return + } + source := ins.Args[0] + value := x86CallerCountedNumber{source: -1, version: facts.sequence} + if move { + switch source.Kind { + case OpImm: + if source.ImmRaw != "" || source.ImmIsFloat { + return + } + value.value, value.known = uint64(source.Imm), true + case OpReg: + root, ok := x86CallerGP(source.Reg) + if !ok || root != source.Reg { + return + } + if root == reg { + value = old + } else { + var present bool + value, present = facts.regs[root] + if !present { + return + } + } + if !oldPresent && root == reg { + return + } + case OpMem: + pointer, present := before.callers[source.Mem.Base] + if op != "MOVQ" || !present || pointer.offset != 0 || pointer.span != 0 || + pointer.argument < 0 || pointer.argument >= len(proof.arguments) || proof.arguments[pointer.argument].kind != x86CallerLocalLength || + source.Mem.Off != 0 || source.Mem.Index != "" || source.Mem.Sym != "" || source.Mem.Segment != "" { + return + } + value.source = pc + if facts.source < 0 { + facts.source = pc + } + default: + return + } + if op == "MOVL" && value.known { + value.value = uint64(uint32(value.value)) + } + } else if (op == "XORL" || op == "XORQ") && source.Kind == OpReg && source.Reg == reg { + value.value, value.known = 0, true + } else if (op == "ANDL" || op == "ANDQ") && oldPresent && source.Kind == OpImm && source.ImmRaw == "" { + value = old + if old.known { + value.value &= uint64(source.Imm) + } else if old.source == facts.source && facts.source >= 0 && source.Imm > 0 && source.Imm <= 3 { + value.mask, value.value, value.known = uint64(source.Imm), uint64(residue)&uint64(source.Imm), true + } else { + return + } + } else if (op == "ADDL" || op == "ADDQ") && oldPresent && old.known && source.Kind == OpImm && source.ImmRaw == "" { + limit := uint64(math.MaxUint64) + if op == "ADDL" { + limit = math.MaxUint32 + } + if source.Imm < 0 || old.value > limit || uint64(source.Imm) > limit-old.value { + return + } + value.value, value.known = old.value+uint64(source.Imm), true + } else { + return + } + if (op == "ANDL" || op == "ADDL") && value.known { + value.value = uint64(uint32(value.value)) + } + value.version = facts.sequence + facts.regs[reg] = value +} + +type x86CallerCountedVisit struct { + pc, residue, iteration, loop int + state x86SourceStackState + numbers x86CallerCountedNumbers +} + +type x86CallerCountedWork struct { + x86CallerCountedVisit + bound x86CallerCountedNumber + entry x86SourceStackState +} + +func x86CallerCountedOrigins(entry, current x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if entry.sp != current.sp || entry.bp != current.bp || entry.gOriginal != current.gOriginal || + !reflect.DeepEqual(entry.cells, current.cells) || len(entry.callers) != len(current.callers) { + return false + } + for reg, pointer := range entry.callers { + other, present := current.callers[reg] + if !present || pointer.argument != other.argument || pointer.argument < 0 || pointer.argument >= len(proof.arguments) { + return false + } + if _, valid := x86CallerPointerEnd(other); !valid { + return false + } + if proof.arguments[pointer.argument].kind != x86CallerSliceInput && pointer != other { + return false + } + } + return true +} + +type x86CallerCountedTransfer func(int, *Instr, *x86SourceStackState) (bool, string) + +// Each feasible path keeps a fresh state, so no predecessor can supply another +// path's permission. The finite visit cap fails; it never widens a loop state. +func replayX86CallerCountedPlan(plan *x86CallerCountedPlan, fn *Func, instructions []int, initial x86SourceStackState, + proof *x86GoCallerRegionProof, transfer x86CallerCountedTransfer) ([]x86CallerCountedVisit, string) { + var visits []x86CallerCountedVisit + returned := 0 + for residue := 0; residue <= 3; residue++ { + work := []x86CallerCountedWork{{x86CallerCountedVisit: x86CallerCountedVisit{ + residue: residue, iteration: -1, loop: -1, state: x86CloneCallerState(initial), + numbers: x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{}, source: -1}, + }}} + for len(work) != 0 { + if len(visits) >= 8192 { + return nil, "counted-loop finite path budget exhausted" + } + item := work[0] + work = work[1:] + pc := item.pc + if loopIndex, header := plan.headers[pc]; header { + loop := plan.loops[loopIndex] + bound, bOK := item.numbers.regs[loop.bound] + counter, cOK := item.numbers.regs[loop.counter] + if item.loop != -1 && item.loop != loopIndex { + return nil, "counted-loop entry crosses an active SCC" + } + if item.loop == -1 { + item.loop = loopIndex + item.iteration, item.bound = 0, bound + item.entry = x86CloneCallerState(item.state) + } + if !bOK || !cOK || !bound.known || !counter.known || bound.value == 0 || bound.value > 3 || + bound.source != item.numbers.source || bound.source < 0 || bound.mask == 0 || bound.mask > 3 || + !reflect.DeepEqual(bound, item.bound) || counter.value != uint64(item.iteration) || item.iteration >= 3 { + return nil, "counted-loop entry lacks immutable bounded value and zero-based counter" + } + } + visits = append(visits, item.x86CallerCountedVisit) + state, numbers := x86CloneCallerState(item.state), item.numbers.clone() + ins := &fn.Instrs[instructions[pc]] + ins.x86CallerMemory, ins.x86SourceStackFP = nil, nil + isReturn, reason := transfer(pc, ins, &state) + if reason != "" { + return nil, fmt.Sprintf("normalized instruction %d: %s", instructions[pc], reason) + } + if isReturn { + if item.loop != -1 { + return nil, "counted-loop return bypasses its exact latch" + } + returned++ + continue + } + numbers.step(pc, residue, *ins, item.state, proof) + nexts := plan.edges[pc] + if loopIndex, latch := plan.latches[pc]; latch { + loop := plan.loops[loopIndex] + flags, counter, bound := numbers.flags, numbers.regs[loop.counter], numbers.regs[loop.bound] + if item.loop != loopIndex { + return nil, "counted-loop latch lacks its original SCC entry" + } + if !flags.known || flags.pc != pc-1 || !counter.known || counter.value != uint64(item.iteration+1) || + !reflect.DeepEqual(bound, item.bound) || !x86CallerCountedOrigins(item.entry, state, proof) { + return nil, "counted-loop latch lacks exact counter, bound and comparison versions" + } + for index, reg := range flags.regs { + if (reg != loop.counter && reg != loop.bound) || flags.versions[index] != numbers.regs[reg].version { + return nil, "counted-loop comparison observes stale operand versions" + } + } + } + if (ins.Op == "JE" || ins.Op == "JNE") && numbers.flags.known { + taken := numbers.flags.equal == (ins.Op == "JE") + if taken { + nexts = nexts[1:] + } else { + nexts = nexts[:1] + } + } + for _, next := range nexts { + iteration, loopIndex := item.iteration, item.loop + bound, entry := item.bound, item.entry + if item.loop >= 0 { + loop := plan.loops[item.loop] + if pc == loop.latch && next == loop.header { + iteration++ + } else if !loop.body[next] { + if pc != loop.latch { + return nil, "counted-loop exit bypasses its exact latch" + } + // Clear only this loop's control witness. All source origins, + // initialized bytes, numerical versions and flags survive. + iteration, loopIndex = -1, -1 + bound, entry = x86CallerCountedNumber{}, x86SourceStackState{} + } + } + work = append(work, x86CallerCountedWork{x86CallerCountedVisit: x86CallerCountedVisit{ + pc: next, residue: residue, iteration: iteration, loop: loopIndex, + state: x86CloneCallerState(state), numbers: numbers.clone(), + }, bound: bound, entry: x86CloneCallerState(entry)}) + } + } + } + if returned == 0 { + return nil, "counted-loop replay has no original source return" + } + return visits, "" +} diff --git a/amd64_go_caller_counted_loop_boundary_test.go b/amd64_go_caller_counted_loop_boundary_test.go new file mode 100644 index 000000000..85d40fc9a --- /dev/null +++ b/amd64_go_caller_counted_loop_boundary_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GoCallerCountedLoopRetainsUnprovedPaths(t *testing.T) { + original := x86CallerCountedLoopScalarAssembly(false) + for name, change := range map[string][2]string{ + "missing nonzero guard": {"JE done\n", ""}, + "reversed guard": {"JE done", "JNE done"}, + "stale guard flags": {"JE done", "ADDL $1,R10\nJE done"}, + "different guard bits": {"TESTB $3,R9", "TESTB $4,R9"}, + "byte flags truncate": {"TESTB $3,R9\nJE done", "MOVL $256,R8\nTESTB R8,R8\nJNE done"}, + "unmapped length": {"MOVQ (CX),R9", "MOVQ (SI),R9"}, + "unknown length": {"MOVQ (CX),R9", "MOVQ R12,R9"}, + "bound four": {"ANDL $3,AX", "ANDL $7,AX"}, + "unrelated bound": {"MOVL R9,AX", "MOVL $2,AX"}, + "partial bound": {"MOVL R9,AX", "MOVW R9,AX"}, + "bound overwritten": {"MOVL (DI),BX", "ANDL $3,AX\nMOVL (DI),BX"}, + "bound alias": {"ANDL $3,AX", "ANDL $3,CX"}, + "counter absent": {"XORL CX,CX", ""}, + "counter nonzero": {"XORL CX,CX", "MOVQ $1,CX"}, + "counter high bits": {"XORL CX,CX", "MOVQ $0x100000000,CX"}, + "partial counter": {"XORL CX,CX", "XORW CX,CX"}, + "zero stride": {"ADDQ $1,CX", "ADDQ $0,CX"}, + "negative stride": {"ADDQ $1,CX", "SUBQ $1,CX"}, + "two stride": {"ADDQ $1,CX", "ADDQ $2,CX"}, + "double update": {"ADDQ $1,CX", "ADDQ $1,CX\nADDQ $1,CX"}, + "update bypass": {"ADDQ $1,CX", "TESTL BX,BX\nJE compare\nADDQ $1,CX\ncompare:"}, + "stale latch flags": {"JNE again", "ADDL $1,R10\nJNE again"}, + "different latch operands": {"CMPL CX,AX", "CMPL BX,AX"}, + "side entry": {"JE done", "JE again"}, + "nested loop": {"MOVL (DI),BX", "inner:\nCMPQ R9,$0\nJNE inner\nMOVL (DI),BX"}, + "unknown indirect target": {"JNE again", "JNE again\nJMP R12"}, + "unknown dead effect": {"again:", "JMP again\nRDTSC\nagain:"}, + "different input origin": {"ADDQ $4,DI", "MOVQ SI,DI"}, + "missing input origin": {"ADDQ $4,DI", "MOVQ $0,DI"}, + "readonly write": {"MOVL (DI),BX", "MOVL R10,(DI)\nMOVL (DI),BX"}, + "result displacement": {"ADDQ $4,DI", "ADDQ $1,DX\nADDQ $4,DI"}, + "source stack escape": {"ADDQ $4,DI", "MOVQ DI,SP\nADDQ $4,DI"}, + "input cap": {"ADDQ $4,DI", "ADDQ $1048576,DI"}, + "pointer truncation": {"ADDQ $4,DI", "LEAL 4(DI),DI"}, + "unknown first GP": {"ADDL R11,BX", "ADDL R12,BX"}, + "late first GP": {"MOVL (DI),BX", "ADDL BX,R10\nMOVL (DI),BX"}, + "unknown first vector": {"MOVL (DI),BX", "VEXTRACTF128 $1,Y0,X1\nMOVL (DI),BX"}, + "unknown first mask": {"MOVL (DI),BX", "VMOVUPS (DI),Z0\nVPADDD.Z Z0,Z0,K1,Z1\nMOVL (DI),BX"}, + "mask zero no rights": {"MOVL (DI),BX", "VMOVUPS (DI),Z0\nMOVL $0,BX\nKMOVQ BX,K1\nVPADDD.Z (DX),Z0,K1,Z1\nMOVL (DI),BX"}, + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(original, change[0], change[1], 1) + if assembly == original { + t.Fatal("negative failed to change its original source") + } + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved %s admitted: %v", name, err) + } + }) + } +} + +func TestX86GoCallerCountedLoopSealsAllPartitionsAndSource(t *testing.T) { + for _, change := range []string{"none", "original", "normalized", "missing caller", "certificate width"} { + t.Run(change, func(t *testing.T) { + assembly := x86CallerCountedLoopScalarAssembly(true) + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil || normalized.Funcs[0].x86SourceStack == nil { + t.Fatalf("complete finite source proof: %v", err) + } + proof := normalized.Funcs[0].x86SourceStack + if proof.countedIterations != [4]int{0, 1, 2, 3} { + t.Fatalf("missing or altered feasible residue visits: %v", proof.countedIterations) + } + actualHash, err := x86CallerSourceFingerprint(normalized.Funcs[0]) + if err != nil || actualHash != proof.normalized { + t.Fatal("complete normalized identity was not sealed") + } + if change == "original" { + file.Funcs[0].Instrs[1].Args[0].FPOffset++ + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + failed := normalizeX86RawReturnSourceStack(decoded, options, true, file.Funcs[0]) + if failed.x86SourceStack != nil { + t.Fatal("changed original source borrowed its old caller identity") + } + return + } + for index := range normalized.Funcs[0].Instrs { + ins := &normalized.Funcs[0].Instrs[index] + if ins.Op != "MOVL" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem { + continue + } + if !x86CallerMemoryCertified(*ins, 0) { + t.Fatal("original full-width read lacks a final replay certificate") + } + if change == "certificate width" { + access := ins.x86CallerMemory[0] + access.width++ + ins.x86CallerMemory[0] = access + if x86CallerMemoryCertified(*ins, 0) || x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("mutated private width acquired admission") + } + } else if change == "normalized" { + ins.Args[0].Mem.Base = DX + } + } + if change == "missing caller" { + sig := sigs["caller.kernel"] + sig.x86CallerRegions = nil + sigs[sig.Name] = sig + } + _, err = Translate(normalized, options) + if change == "none" || change == "certificate width" { + if err != nil { + t.Fatal(err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("edited %s retained final proof: %v", change, err) + } + }) + } +} + +func TestX86GoCallerCountedLoopFinitePathBudgetFailsClosed(t *testing.T) { + var body strings.Builder + for index := 0; index < 14; index++ { + fmt.Fprintf(&body, "CMPL BX,$0\nJE branch%d\nMOVL $7,R11\nbranch%d:\n", index, index) + } + original := x86CallerCountedLoopScalarAssembly(false) + assembly := strings.Replace(original, "MOVL (DI),BX", "MOVL (DI),BX\n"+body.String(), 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "budget exhausted") { + t.Fatalf("unproved/exhausted path set acquired a certificate: %v", err) + } +} + +func TestX86GoCallerCountedLoopStatusWidthsUseActualLowBits(t *testing.T) { + for _, item := range []struct { + op Op + value uint64 + }{ + {"TESTB", 1 << 8}, {"TESTW", 1 << 16}, {"TESTL", 1 << 32}, {"TESTQ", 0}, + } { + t.Run(string(item.op), func(t *testing.T) { + assembly := fmt.Sprintf("TEXT ·status(SB),4,$0-0\nMOVQ $%d,R8\n%s R8,R8\nRET\n", item.value, item.op) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + facts := x86CallerCountedNumbers{regs: map[Reg]x86CallerCountedNumber{}, source: -1} + for pc, ins := range file.Funcs[0].Instrs { + facts.step(pc, 0, ins, x86SourceStackState{}, nil) + } + if !facts.flags.known || !facts.flags.equal { + t.Fatalf("actual %s truncation to zero did not prove ZF: %+v", item.op, facts.flags) + } + }) + } +} + +func TestX86GoCallerCountedLoopAbsentNumericalFactStaysUnknown(t *testing.T) { + facts := x86CallerCountedNumbers{ + regs: map[Reg]x86CallerCountedNumber{}, source: 0, + } + facts.step(1, 0, Instr{Op: "TESTQ", Args: []Operand{ + {Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: Reg("R8")}, + }}, x86SourceStackState{}, nil) + if facts.flags.known { + t.Fatal("an absent numerical fact borrowed the first partition source") + } +} + +func TestX86GoCallerCountedLoopAlternateRegistersAndWidths(t *testing.T) { + for _, counterWidth := range []string{"L", "Q"} { + for _, compareWidth := range []string{"L", "Q"} { + t.Run(counterWidth+compareWidth, func(t *testing.T) { + assembly := x86CallerCountedLoopScalarAssembly(false) + assembly = strings.NewReplacer( + "JE done", "JE empty_final", "done:", "empty_final:", + "again:", "generic_body:", "JNE again", "JNE generic_body", + "MOVL R9,AX", "MOVL R9,R13", "ANDL $3,AX", "ANDL $3,R13", + "XORL CX,CX", "XORL R12,R12", "ADDQ $1,CX", "ADD"+counterWidth+" $1,R12", + "CMPL CX,AX", "CMP"+compareWidth+" R12,R13", + ).Replace(assembly) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } +} + +func TestX86GoCallerCountedLoopFiveTargetsKeepAMD64Authority(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + if target.arch == "amd64" { + assembly := x86CallerCountedLoopScalarAssembly(false) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, target.triple); err != nil { + t.Fatal(err) + } + return + } + // These two objects are the existing ordinary 386 scalar contract. + // They are not copied AMD64 input or finite-loop authority. + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerCountedLoopScalarSource()) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "386"); !errors.Is(err, ErrProbeNeedsContext) || proof != nil { + t.Fatal("mode32 borrowed the AMD64 caller input contract") + } + source, scalarSig, _ := x86SourceStackNumericalFixture("386", "@MOV @WORD(SP),AX\nADDL $7,AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3") + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions("386", target.triple, scalarSig)) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mode32-counted-boundary.ll", "mode32-counted-boundary.o", module.String()) + }) + } +} diff --git a/amd64_go_caller_counted_loop_runtime_test.go b/amd64_go_caller_counted_loop_runtime_test.go new file mode 100644 index 000000000..f82e43b25 --- /dev/null +++ b/amd64_go_caller_counted_loop_runtime_test.go @@ -0,0 +1,194 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86GoCallerCountedLoopOriginalInitializedValues(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the separately required Linux/amd64 caller runtime counterpart") + } + var source, assembly, calls strings.Builder + source.WriteString("package caller\nimport \"unsafe\"\n") + for _, raw := range []bool{false, true} { + for _, separateExit := range []bool{false, true} { + name := fmt.Sprintf("counted%t%t", raw, separateExit) + declaration := x86CallerCountedLoopScalarSource() + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1) + source.WriteString(declaration) + machine := x86CallerCountedLoopScalarAssembly(raw) + if separateExit { + machine = x86CallerCountedLoopSeparateExitAssembly(raw) + } + assembly.WriteString(strings.ReplaceAll(machine, "·kernel", "·"+name)) + fmt.Fprintf(&calls, ` + if got:=Call%s(x.data[:length],y.data[:]); got!=want || + snapshot(&x)!=originalX || snapshot(&y)!=originalY { + t.Fatalf("%s length=%%d got=%%#x want=%%#x",length,got,want) + } +`, name, name) + } + } + goTest := `package caller +import "testing" + +type guardedInput struct { + before [4]uint32 + data [16]uint32 + after [4]uint32 +} + +//go:noinline +func snapshot(input *guardedInput) guardedInput { return *input } + +func TestOriginalInitializedCountedValues(t *testing.T) { + for repeat:=0; repeat<4; repeat++ { + for _, value:=range []uint32{0,1,7,0x80000000,0xffffffff,0x12345678} { + for length:=0; length<16; length++ { + x := guardedInput{ + before: [4]uint32{11,17,29,53}, + data: [16]uint32{value,value^0xaabbccdd,19,23,29,31,37,41,43,47,53,59,61,67,71,73}, + after: [4]uint32{0xaabbccdd,0x12345678,0x87654321,0xffffffff}, + } + y := guardedInput{ + before: x.after, + data: [16]uint32{31,37,value,43,47,53,59,61,67,71,73,79,83,89,97,101}, + after: x.before, + } + originalX,originalY:=snapshot(&x),snapshot(&y) + var want uint32 + for index:=0;index +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + var calls strings.Builder + for _, raw := range []bool{false, true} { + for _, separateExit := range []bool{false, true} { + name := fmt.Sprintf("counted%t%t", raw, separateExit) + declaration := x86CallerCountedLoopScalarSource() + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + source.WriteString(strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1)) + machine := x86CallerCountedLoopScalarAssembly(raw) + if separateExit { + machine = x86CallerCountedLoopSeparateExitAssembly(raw) + } + assembly.WriteString(strings.ReplaceAll(machine, "·kernel", "·"+name)) + fmt.Fprintf(&oracle, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + fmt.Fprintf(&calls, ` + memset(output,0xc9,sizeof(output));memcpy(expected,output,sizeof(output)); + memcpy(expected+5,&want,sizeof(want)); + %s(a+1,b+3,output+5,&length); + if (memcmp(output,expected,sizeof(output)) || memcmp(a,original_a,sizeof(a)) || + memcmp(b,original_b,sizeof(b)) || length!=original_length) return 1; + calls++; +`, name) + } + } + oracle.WriteString(`int main(void) { + const uint32_t values[]={0,1,7,0x80000000u,0xffffffffu,0x12345678u}; + unsigned calls=0; + for (unsigned repeat=0;repeat<4;repeat++) { + for (unsigned value=0;value 1 + for _, next := range edges[component[0]] { + cyclic = cyclic || next == component[0] + } + if !cyclic { + continue + } + body := make(map[int]bool, len(component)) + for _, pc := range component { + body[pc] = true + } + header, latch := -1, -1 + entries, exits := 0, 0 + for pc, nexts := range edges { + if !reachable[pc] { + continue + } + for _, next := range nexts { + if !body[pc] && body[next] { + entries++ + header = next + } + if body[pc] && !body[next] { + exits++ + latch = pc + } + } + } + if entries != 1 || exits != 1 { + return nil + } + loop := deriveX86CallerCountedLoop(fn, instructions, edges, reachable, body, header, latch) + if loop == nil || !x86CallerCountedBodyDAG(*loop, edges) { + return nil + } + plan.headers[header], plan.latches[latch] = len(plan.loops), len(plan.loops) + plan.loops = append(plan.loops, *loop) + } + if len(plan.loops) == 0 { + return nil + } + var err error + plan.original, err = x86CallerSourceFingerprint(original) + if err != nil { + return nil + } + plan.normalized, err = x86CallerSourceFingerprint(fn) + if err != nil { + return nil + } + return plan +} + +// Removing exactly the proved latch backedge must eliminate every body cycle. +// A nested/alternate cycle cannot borrow another SCC's finite iteration cap. +func x86CallerCountedBodyDAG(loop x86CallerCountedLoop, edges [][]int) bool { + indegree := map[int]int{} + for pc := range loop.body { + for _, next := range edges[pc] { + if loop.body[next] && !(pc == loop.latch && next == loop.header) { + indegree[next]++ + } + } + } + var work []int + for pc := range loop.body { + if indegree[pc] == 0 { + work = append(work, pc) + } + } + removed := 0 + for len(work) > 0 { + pc := work[0] + work = work[1:] + removed++ + for _, next := range edges[pc] { + if loop.body[next] && !(pc == loop.latch && next == loop.header) { + indegree[next]-- + if indegree[next] == 0 { + work = append(work, next) + } + } + } + } + return removed == len(loop.body) +} diff --git a/amd64_go_caller_counted_scc_boundary_test.go b/amd64_go_caller_counted_scc_boundary_test.go new file mode 100644 index 000000000..1a4689f8f --- /dev/null +++ b/amd64_go_caller_counted_scc_boundary_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerCountedSCCRejectThreeAPIs(t *testing.T, assembly string) { + t.Helper() + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + before := fmt.Sprintf("%#v %#v", file, sigs) + t.Run("Translate", func(t *testing.T) { + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved SCC source acquired text permission: %v", err) + } + }) + t.Run("Module", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved SCC source acquired module permission: %v", err) + } + }) + t.Run("GoModule", func(t *testing.T) { + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved SCC source acquired Go module permission: %v", err) + } + }) + if before != fmt.Sprintf("%#v %#v", file, sigs) { + t.Fatal("failed SCC replay mutated caller/source inputs") + } +} + +func TestX86GoCallerCountedSCCRetainsEveryPathAndOrigin(t *testing.T) { + original := x86CallerCountedSCCAssembly(false, true) + for name, change := range map[string][2]string{ + "second missing guard": {"JE done\n", ""}, + "second reversed guard": {"JE done", "JNE done"}, + "second unknown bound": {"MOVL R9,R13", "MOVL BX,R13"}, + "second unrelated bound": {"MOVL R9,R13", "MOVL $2,R13"}, + "second fresh length source": {"MOVL R9,R13", "MOVQ length+24(FP),R8\nMOVQ (R8),R9\nMOVL R9,R13"}, + "second wider partition": {"ANDL $3,R13", "ANDL $7,R13"}, + "second partial bound": {"MOVL R9,R13", "MOVW R9,R13"}, + "second missing counter": {"XORL R12,R12", ""}, + "second nonzero counter": {"XORL R12,R12", "MOVL $1,R12"}, + "second partial counter": {"XORL R12,R12", "XORW R12,R12"}, + "second stride zero": {"ADDQ $1,R12", "ADDQ $0,R12"}, + "second stride two": {"ADDQ $1,R12", "ADDQ $2,R12"}, + "second decrement": {"ADDQ $1,R12", "SUBQ $1,R12"}, + "second double update": {"ADDQ $1,R12", "ADDQ $1,R12\nADDQ $1,R12"}, + "second update bypass": {"ADDQ $1,R12", "TESTL BX,BX\nJE second_compare\nADDQ $1,R12\nsecond_compare:"}, + "second stale flags": {"JNE second_body", "ADDL $1,R10\nJNE second_body"}, + "second side entry": {"JE done", "JE second_body"}, + "second early exit": {"second_body:", "second_body:\nTESTL BX,BX\nJE done"}, + "second nested cycle": {"second_body:", "second_body:\ninner:\nTESTL BX,BX\nJNE inner"}, + "dead unknown effect": {"second_body:", "JMP second_body\nRDTSC\nsecond_body:"}, + "first unknown GP": {"ADDL R11,BX", "ADDL R8,BX"}, + "diamond missing GP": {"MOVL $5,R11", "MOVL R8,R11"}, + "diamond missing vector": {"MOVL $5,R11", "VEXTRACTF128 $1,Y0,X1\nMOVL $5,R11"}, + "diamond missing mask": {"MOVL $5,R11", "VMOVUPS (DI),Z0\nVPADDD.Z Z0,Z0,K1,Z1\nMOVL $5,R11"}, + "diamond readonly write": {"MOVL $5,R11", "MOVL $5,(DI)\nMOVL $5,R11"}, + "diamond unknown origin": {"MOVL $5,R11", "MOVQ R8,DI\nMOVL $5,R11"}, + "diamond lost result": {"MOVL $5,R11", "MOVQ DI,DX\nMOVL $5,R11"}, + "diamond displaced result": {"MOVL $5,R11", "ADDQ $1,DX\nMOVL $5,R11"}, + "diamond SP escape": {"MOVL $5,R11", "MOVQ DI,SP\nMOVL $5,R11"}, + "diamond BP escape": {"MOVL $5,R11", "MOVQ DI,BP\nMOVL $5,R11"}, + "diamond g clobber": {"MOVL $5,R11", "MOVQ $0,R14\nMOVL $5,R11"}, + } { + t.Run(name, func(t *testing.T) { + if strings.Count(original, change[0]) == 0 { + t.Fatal("negative does not name original source") + } + assembly := strings.Replace(original, change[0], change[1], 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCRejectThreeAPIs(t, assembly) + }) + } +} + +func TestX86GoCallerCountedSCCCannotCloseUnknownCallers(t *testing.T) { + for name, change := range map[string][2]string{ + "indirect exit": {"done:", "done:\nJMP R8"}, + "unknown call": {"first_done:", "first_done:\nCALL ·unregistered(SB)"}, + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerCountedSCCAssembly(false, false), change[0], change[1], 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + if _, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unclosed caller graph supplied a producer: %v", err) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unclosed caller graph translated: %v", err) + } + }) + } +} + +func TestX86GoCallerCountedSCCDoesNotMergeDifferentReadonlyPaths(t *testing.T) { + // These paths are individually legal. Replay must keep their different + // authenticated input origins rather than deriving a synthetic common one. + assembly := strings.Replace(x86CallerCountedSCCAssembly(false, true), "MOVL $5,R11", "MOVQ SI,DI\nMOVL $5,R11", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCObjects(t, x86CallerCountedLoopScalarSource(), assembly, "x86_64-unknown-linux-gnu") +} + +func TestX86GoCallerCountedSCCPreservesResidueAndSourceSeals(t *testing.T) { + assembly := x86CallerCountedSCCAssembly(true, false) + pkg, file := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil || normalized.Funcs[0].x86SourceStack == nil { + t.Fatalf("complete SCC replay proof missing: %v", err) + } + proof := normalized.Funcs[0].x86SourceStack + if proof.countedIterations != [4]int{0, 2, 4, 6} { + t.Fatalf("multiple SCCs changed residue denominator: %v", proof.countedIterations) + } + if fingerprint, err := x86CallerSourceFingerprint(normalized.Funcs[0]); err != nil || fingerprint != proof.normalized { + t.Fatal("normalized identity not sealed") + } + for _, name := range []string{"original source", "normalized source", "missing caller", "access width"} { + t.Run(name, func(t *testing.T) { + _, original := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + copy, err := normalizeX86RawFile(original, "amd64", options) + if err != nil { + t.Fatal(err) + } + switch name { + case "original source": + original.Funcs[0].Instrs[1].Args[0].FPOffset++ + decoded, err := decodeX86RawDirectives(original.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + failed := normalizeX86RawReturnSourceStack(decoded, options, true, original.Funcs[0]) + if failed.x86SourceStack != nil { + t.Fatal("modified original acquired stale SCC permission") + } + return + case "missing caller": + privateOptions := options + privateOptions.Sigs = map[string]FuncSig{} + for symbol, sig := range sigs { + sig.x86CallerRegions = nil + privateOptions.Sigs[symbol] = sig + } + if _, err := Translate(copy, privateOptions); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("lost caller identity acquired SCC permission: %v", err) + } + return + } + for index := range copy.Funcs[0].Instrs { + ins := ©.Funcs[0].Instrs[index] + if ins.Op != "MOVL" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.x86CallerMemory == nil { + continue + } + if !x86CallerMemoryCertified(*ins, 0) { + t.Fatal("complete read certificate missing") + } + if name == "access width" { + access := ins.x86CallerMemory[0] + access.width++ + ins.x86CallerMemory[0] = access + if x86CallerMemoryCertified(*ins, 0) || x86RawReturnMemoryBound(copy.Funcs[0], copy, options, true) { + t.Fatal("tampered read width borrowed SCC permission") + } + return + } + ins.Args[0].Mem.Base = DX + if _, err := Translate(copy, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("modified normalized source retained SCC permission: %v", err) + } + return + } + t.Fatal("no source access selected for negative") + }) + } +} + +func TestX86GoCallerCountedSCCMasksAndWidths(t *testing.T) { + for _, mask := range []int{1, 2, 3} { + for _, width := range []string{"L", "Q"} { + t.Run(fmt.Sprintf("mask%d/%s", mask, width), func(t *testing.T) { + assembly := strings.ReplaceAll(x86CallerCountedSCCAssembly(false, false), "$3,", fmt.Sprintf("$%d,", mask)) + assembly = strings.NewReplacer("ADDQ $1,CX", "ADD"+width+" $1,CX", + "ADDQ $1,R12", "ADD"+width+" $1,R12", "CMPL CX,AX", "CMP"+width+" CX,AX", + "CMPL R12,R13", "CMP"+width+" R12,R13").Replace(assembly) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCObjects(t, x86CallerCountedLoopScalarSource(), assembly, "x86_64-unknown-linux-gnu") + }) + } + } +} + +func TestX86GoCallerCountedSCCIndependentMasksAndBodyDiamonds(t *testing.T) { + for _, masks := range [][2]int{{1, 2}, {2, 3}, {3, 1}} { + t.Run(fmt.Sprintf("%d/%d", masks[0], masks[1]), func(t *testing.T) { + assembly := x86CallerCountedSCCAssembly(false, false) + assembly = strings.NewReplacer("TESTB $3,R9\nJE first_done", fmt.Sprintf("TESTB $%d,R9\nJE first_done", masks[0]), + "ANDL $3,AX", fmt.Sprintf("ANDL $%d,AX", masks[0]), + "TESTB $3,R9\nJE done", fmt.Sprintf("TESTB $%d,R9\nJE done", masks[1]), + "ANDL $3,R13", fmt.Sprintf("ANDL $%d,R13", masks[1])).Replace(assembly) + // Both arms of this internal diamond are known numerical writes. + // The +1 update still dominates every path to the exact latch. + assembly = strings.Replace(assembly, "ADDQ $1,R12", `TESTL BX,BX +JE zero_value +MOVL $3,R11 +JMP body_join +zero_value: +MOVL $5,R11 +body_join: +ADDL R11,R10 +ADDQ $1,R12`, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + x86CallerCountedSCCObjects(t, x86CallerCountedLoopScalarSource(), assembly, "x86_64-unknown-linux-gnu") + }) + } +} + +func TestX86GoCallerCountedSCCKeepsWholeReplayBudget(t *testing.T) { + var diamond strings.Builder + for index := 0; index < 14; index++ { + fmt.Fprintf(&diamond, "TESTL BX,BX\nJE budget%d\nMOVL $7,R11\nbudget%d:\n", index, index) + } + assembly := strings.Replace(x86CallerCountedSCCAssembly(false, false), "ADDQ $1,R12", diamond.String()+"ADDQ $1,R12", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerCountedLoopScalarSource(), assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) || + !strings.Contains(err.Error(), "finite path budget exhausted") { + t.Fatalf("SCC composition expanded or bypassed the finite replay budget: %v", err) + } +} diff --git a/amd64_go_caller_counted_scc_runtime_test.go b/amd64_go_caller_counted_scc_runtime_test.go new file mode 100644 index 000000000..6f868ad97 --- /dev/null +++ b/amd64_go_caller_counted_scc_runtime_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Both existing required counted-loop runtime roots consume these fixtures. +// Their host/runner policy is unchanged; Rosetta remains diagnostic rather +// than evidence of native Linux execution or physical source-stack identity. +func x86CallerCountedSCCRuntimeFixtures() (string, string, []string) { + var source, assembly strings.Builder + source.WriteString("package caller\nimport \"unsafe\"\n") + var names []string + for _, raw := range []bool{false, true} { + for _, diamond := range []bool{false, true} { + name := fmt.Sprintf("scc%t%t", raw, diamond) + declaration := x86CallerCountedLoopScalarSource() + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + source.WriteString(strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1)) + assembly.WriteString(strings.ReplaceAll(x86CallerCountedSCCAssembly(raw, diamond), "·kernel", "·"+name)) + names = append(names, name) + } + } + return source.String(), assembly.String(), names +} + +func runX86CallerCountedSCCOriginalValues(t *testing.T) { + source, assembly, names := x86CallerCountedSCCRuntimeFixtures() + var calls strings.Builder + for index, name := range names { + fmt.Fprintf(&calls, ` + { + want := sum + if %t { + if x.data[bound] == 0 { want += 5 } else { want += 3 } + } + if got := Call%s(x.data[:length],y.data[:]); got != want || + snapshot(&x) != originalX || snapshot(&y) != originalY { + t.Fatalf("%s length=%%d got=%%#x want=%%#x",length,got,want) + } + calls++ + } +`, index%2 == 1, name, name) + } + testSource := `package caller +import "testing" +type guardedInput struct { before [4]uint32; data [16]uint32; after [4]uint32 } +//go:noinline +func snapshot(input *guardedInput) guardedInput { return *input } +func TestOriginalSCCValues(t *testing.T) { + calls := 0 + for repeat := 0; repeat < 4; repeat++ { + for _, value := range []uint32{0,1,7,0x80000000,0xffffffff,0x12345678} { + for length := 0; length < 16; length++ { + x := guardedInput{ + before: [4]uint32{11,17,29,53}, + data: [16]uint32{value,0,value^0xaabbccdd,0,29,31,37,41,43,47,53,59,61,67,71,73}, + after: [4]uint32{0xaabbccdd,0x12345678,0x87654321,0xffffffff}, + } + y := guardedInput{before:x.after, + data:[16]uint32{31,37,value,43,47,53,59,61,67,71,73,79,83,89,97,101},after:x.before} + originalX,originalY := snapshot(&x),snapshot(&y) + bound := length & 3 + var sum uint32 + for index := 0; index < 2*bound; index++ { sum += x.data[index]+y.data[index] } +` + calls.String() + ` + } + } + } + if calls != 1536 { t.Fatalf("calls=%d", calls) } +} +` + dir := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": source, + "value_amd64.s": assembly, "value_test.go": testSource, + } { + bytes := []byte(content) + if strings.HasSuffix(name, ".go") { + var err error + bytes, err = format.Source(bytes) + if err != nil { + t.Fatalf("format generated %s: %v", name, err) + } + } + if err := os.WriteFile(filepath.Join(dir, name), bytes, 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("first-party original Go serial SCC integer/canary oracle: %v\n%s", err, output) + } + t.Log("1536 original Go calls: two serial bounded loops, raw/named controls, both diamond arms, u32 wrap and complete guarded input snapshots") +} + +func runX86CallerCountedSCCLLVMValues(t *testing.T) { + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang required") + } + source, assembly, names := x86CallerCountedSCCRuntimeFixtures() + var oracle, calls strings.Builder + oracle.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for index, name := range names { + fmt.Fprintf(&oracle, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + fmt.Fprintf(&calls, ` + { + uint32_t want = sum; + if (%d) want += x[bound] == 0 ? 5 : 3; + memset(output,0xc9,sizeof(output)); memcpy(expected,output,sizeof(output)); + memcpy(expected+5,&want,sizeof(want)); + %s(a+1,b+3,output+5,&length); + if (memcmp(output,expected,sizeof(output)) || memcmp(a,original_a,sizeof(a)) || + memcmp(b,original_b,sizeof(b)) || length != original_length) return 1; + calls++; + } +`, index%2, name) + } + oracle.WriteString(`int main(void) { + const uint32_t values[]={0,1,7,0x80000000u,0xffffffffu,0x12345678u}; + unsigned calls=0; + for (unsigned repeat=0;repeat<4;repeat++) { + for (unsigned value=0;value= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(form.source.Reg)] & width + if form.memory >= 0 { + // Even dwords are the numerical inputs, but the possible native read + // remains m128/m256/m512 or m64bcst, independent of the mask value. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + for start := 0; start < form.byteWidth; start += 8 { + input := uint64(15) << uint(start) + output := uint64(255) << uint(start) + if first&input == input && second&input == input { + known |= output + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_dword_product_test.go b/amd64_go_caller_dword_product_test.go new file mode 100644 index 000000000..fb34ac7a3 --- /dev/null +++ b/amd64_go_caller_dword_product_test.go @@ -0,0 +1,270 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerDwordProductIndependentSourceEffect(t *testing.T) { + for _, row := range []string{"PMULDQ X0,X1", "PMULULQ X0,X1", "VPMULDQ Y0,Y1,Y2", "VPMULUDQ Y0,Y1,Y2"} { + t.Run(row, func(t *testing.T) { + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerDwordProductCompleteMaskBroadcastAndMemoryForms(t *testing.T) { + rows := []string{"PMULDQ X0,X1", "PMULDQ X1,X1", "PMULDQ (DI),X1", "PMULULQ X0,X1", "PMULULQ X1,X1", "PMULULQ (DI),X1"} + for _, name := range x86DwordProductOracleNames[2:] { + for _, row := range x86OfficialPackedBinaryWitnesses(t)[name] { + rows = append(rows, strings.Replace(row, "(AX)", "(DI)", 1)) + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("complete signed/unsigned family caller failed", row, triple, err) + } + } + }) + } +} + +func TestX86GoCallerDwordProductEveryPotentialByteAndUpperAliases(t *testing.T) { + for _, row := range []string{"PMULDQ X0,X1", "PMULULQ X0,X1", "VPMULDQ X0,X1,X2", "VPMULUDQ Y0,Y1,Y2", "VPMULDQ Z0,Z1,K1,Z2", "VPMULUDQ.Z Z0,Z1,K1,Z2"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86DwordProductProperties(ins.Op, ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, missing := range []Reg{"Z0", "Z1", "Z2"} { + for index := 0; index < 64; index++ { + t.Run(fmt.Sprintf("%s/missing%s/byte%d", row, missing, index), func(t *testing.T) { + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + state.vectors[missing] &^= uint64(1) << uint(index) + old := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] + if !x86CallerDwordProductStep(&ins, form, state, nil) { + t.Fatal("register-only numerical effect failed") + } + var want uint64 + for output := 0; output < 64; output++ { + known := true + if output >= form.byteWidth { + known = !form.legacy || old&(uint64(1)<= start+4 + } + if form.mask >= 0 && !form.zeroing && missing == x86CallerVectorRoot(ins.Args[form.destination].Reg) && index == output { + known = false + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("potential input/merge/upper byte lost: got=%016x want=%016x", got, want) + } + }) + } + } + } +} + +func TestX86GoCallerDwordProductExactMemoryCertificates(t *testing.T) { + for _, row := range []string{"PMULDQ (DI),X0", "PMULULQ (DI),X0", "VPMULDQ (DI),Y0,Y2", "VPMULUDQ.Z (DI),Z0,K1,Z2", "VPMULDQ.BCST.Z (DI),Z0,K1,Z2", "VPMULUDQ.BCST (DI),X0,X2"} { + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X0" + } + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86DwordProductProperties(ins.Op, ins, "amd64") + if !known || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + wantWidth := form.byteWidth + if strings.Contains(row, ".BCST") { + wantWidth = 8 + } + if !exists || access.width != int64(wantWidth) || access.write || !x86CallerMemoryCertified(ins, index) { + t.Fatal("zero mask/selected dwords shrank native envelope", row, access) + } + for _, mutation := range []string{"opcode", "width", "direction", "address", "destination", "mask", "address size"} { + changed := ins + changedAccess := access + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch mutation { + case "opcode": + changed.Op = "VPMULDQ" + if strings.Contains(row, "PMULDQ") { + changed.Op = "VPMULUDQ" + } + case "width": + changedAccess.width-- + changed.x86CallerMemory[index] = changedAccess + case "direction": + changedAccess.write = true + changed.x86CallerMemory[index] = changedAccess + case "address": + changed.Args[index].Mem.Off++ + case "destination": + changed.Args[form.destination].Reg = "X7" + case "mask": + if form.mask >= 0 { + changed.Args[form.mask].Reg = "K7" + } else { + changed.Args = append(changed.Args, Operand{Kind: OpReg, Reg: "K7"}) + } + case "address size": + changed.x86AddressBits = 32 + } + if x86CallerMemoryCertified(changed, index) { + t.Fatal("stale native read certificate survived", row, mutation) + } + } + } + if !found { + t.Fatal("normalized source omitted dword product", row) + } + } +} + +func TestX86GoCallerDwordProductUnprovedInputsAndGuardsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPMULDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPMULUDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "partial even dword": "MOVSS (DI),X3\nMOVSS X3,X0\nVMOVUPS (SI),Z1\nVPMULDQ Z0,Z1,Z2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVPMULDQ.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVPMULUDQ Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPMULDQ.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full tuple": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMULDQ.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "broadcast full qword": "VMOVUPS (DI),Z0\nVPMULUDQ.BCST (DX),Z0,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMULDQ (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private SP": "VMOVUPS (DI),Z0\nVPMULDQ (SP),Z0,Z2\nVMOVSS X2,(DX)", + "SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMULDQ (BX),X0\nVMOVSS X0,(DX)", + "BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPMULULQ (BX),X0\nVMOVSS X0,(DX)", + "g use": "VMOVUPS (DI),Z0\nVPMULDQ (R14),Z0,Z2\nVMOVSS X2,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPMULDQ Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX caller": "PMULULQ M0,M1\nVMOVSS X1,(DX)", + "unknown CFG join": "VMOVUPS (DI),Y0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Y1\njoined:\nVPMULDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved input/native/caller guard acquired permission", err) + } + }) + } +} + +func TestX86GoCallerDwordProductRawTuplesBindNativeReadWidths(t *testing.T) { + for _, name := range []string{"VPMULDQ", "VPMULUDQ"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + op, tuple := name, width + if broadcast { + op, tuple = name+".BCST", 8 + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, offset := range []int{-tuple, 64 - tuple, 64} { + row := fmt.Sprintf("%s %d(DI),%s1,K1,%s2", op, offset, bank, bank) + t.Run(row, func(t *testing.T) { + original := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+row+"\nRET\n") + body := "VMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVQ $1,AX\nKMOVQ AX,K1\n" + x86DwordProductDirectives(original[:len(original)-1]) + "VMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("original raw tuple lost existing readonly slice role", row, err) + } + // The input is a dynamic slice role, not an observed fixed + // 64-byte allocation. Certify only the exact possible native + // instruction read, preserving native faulting addresses. + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86DwordProductProperties(ins.Op, ins, "amd64") + if !known || err != nil { + continue + } + found = true + access, ok := ins.x86CallerMemory[0] + if !ins.x86Encoded || form.memory != 0 || ins.Args[0].Mem.Base != DI || ins.Args[0].Mem.Off != int64(offset) || !ok || access.width != int64(tuple) || access.write || !x86CallerMemoryCertified(ins, 0) { + t.Fatal("raw source tuple/address/native-width certificate drift", row, ins, access) + } + } + if !found { + t.Fatal("original raw source omitted from normalized proof", row) + } + // Conversely the fresh result is a real bounded four-byte + // region: no full-vector or m64bcst read fits this role. + boundedRow := strings.Replace(row, "(DI)", "(DX)", 1) + bounded := x86OrderingHintGoBytes(t, "amd64", "TEXT original(SB),4,$0-0\n"+boundedRow+"\nRET\n") + boundedBody := strings.Replace(body, x86DwordProductDirectives(original[:len(original)-1]), x86DwordProductDirectives(bounded[:len(bounded)-1]), 1) + boundedAssembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", boundedBody, 1) + boundedPkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, boundedAssembly) + if err := x86CallerCoreTranslate(t, boundedPkg, boundedAssembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("actual raw tuple gained bounded result-object permission", boundedRow, err) + } + }) + } + } + } + } +} diff --git a/amd64_go_caller_explicit_imul.go b/amd64_go_caller_explicit_imul.go new file mode 100644 index 000000000..75e05bfda --- /dev/null +++ b/amd64_go_caller_explicit_imul.go @@ -0,0 +1,52 @@ +package plan9asm + +// Only the existing validated explicit IMUL descriptor is consumed here. This +// supplies byte-definedness, not a new pointer, frame or memory capability. +func x86CallerExplicitIMULStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (recognized, valid bool) { + if _, known, err := x86ExplicitIMULProperties(ins.Op, *ins, "amd64"); !known || err != nil { + return known, false + } + effect, known, err := x86TypedGPByteEffect(ins.Op, *ins, "amd64") + if !known || err != nil || effect.dynamicMemory { + return true, false + } + // All ordered source bytes (including the two-operand old destination) + // must be defined before any destination can overwrite their evidence. + for _, operand := range effect.operands { + if !operand.read { + continue + } + if operand.operand.Kind == OpMem { + if proof == nil || !x86CallerOrdinaryMemoryStep(ins, operand.index, state, proof, operand.width, false) { + return true, false + } + } else if !x86CallerNumericRead(operand.operand, operand.width, state) { + return true, false + } + } + for _, operand := range effect.operands { + if operand.write && (operand.operand.Kind != OpReg || !x86CallerNumericWrite(operand.operand.Reg, operand.width, state)) { + return true, false + } + } + return true, true +} + +func x86SourceStackNumericFlagMask(state *x86SourceStackState) uint8 { + if state.numericFlags { + return x86ByteFlagsAll // Preserve the pre-existing complete-flags witness. + } + return state.partialNumericFlags +} + +func x86SourceStackNumericFlagsRead(state *x86SourceStackState, required uint8) bool { + return required != 0 && x86SourceStackNumericFlagMask(state)&required == required +} + +func x86SourceStackSetNumericFlags(state *x86SourceStackState, mask uint8) { + state.numericFlags = mask == x86ByteFlagsAll + state.partialNumericFlags = 0 + if !state.numericFlags { + state.partialNumericFlags = mask + } +} diff --git a/amd64_go_caller_explicit_imul_test.go b/amd64_go_caller_explicit_imul_test.go new file mode 100644 index 000000000..34d8f5884 --- /dev/null +++ b/amd64_go_caller_explicit_imul_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// This directory is independent of the typed effect descriptor. It exercises +// all explicit yimul/yimul3 source classes; one-operand IMUL is not in it. +var x86CallerExplicitIMULRows = []struct { + name, instruction string + three bool +}{ + {"register", "IMUL@ AX,BX", false}, + {"memory", "IMUL@ (DI),BX", false}, + {"immediate8", "IMUL@ $-7,BX", false}, + {"immediate32", "IMUL@ $65537,BX", false}, + {"three-register8", "IMUL3@ $-7,AX,BX", true}, + {"three-register32", "IMUL3@ $65537,AX,BX", true}, + {"three-memory8", "IMUL3@ $-7,(DI),BX", true}, + {"three-memory32", "IMUL3@ $65537,(DI),BX", true}, +} + +func x86CallerExplicitIMULFixture(t *testing.T, body string) (GoPackage, *File, string, Options) { + t.Helper() + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint64") + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", body+"\nMOVQ BX,(DX)", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, declaration, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + // goSigsForAsmFile already binds the complete actual SourceSelection. + if sigs["caller.kernel"].x86CallerRegions == nil { + t.Fatalf("actual complete typed caller precondition failed: %v", err) + } + if !x86CallerRegionSourceMatches(sigs["caller.kernel"], file.Funcs[0], "amd64") { + t.Fatal("actual complete typed caller does not match its original source/signature") + } + return pkg, file, assembly, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} +} + +func TestX86GoCallerExplicitIMULRawReturnRegisteredForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, suffix := range []string{"W", "L", "Q"} { + for _, row := range x86CallerExplicitIMULRows { + t.Run(suffix+"/"+row.name, func(t *testing.T) { + body := "MOVQ $7,AX\n" + // W preserves the old upper bytes. Two-operand IMUL also reads + // the old low result; IMUL3L/Q must not require that old result. + if suffix == "W" || !row.three { + body += "MOVQ $3,BX\n" + } + body += strings.ReplaceAll(row.instruction, "@", suffix) + pkg, file, assembly, options := x86CallerExplicitIMULFixture(t, body) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + // Separate format and API/target axes: two representative + // rows cover all APIs/targets; every row covers text/Linux. + allAPIs := suffix == "Q" && row.name == "register" || suffix == "L" && row.name == "three-memory32" + if !allAPIs && triple != "x86_64-unknown-linux-gnu" { + continue + } + options.TargetTriple = triple + t.Run(triple+"/text", func(t *testing.T) { + ir, err := Translate(file, options) + if err != nil { + t.Fatalf("Go-accepted explicit multiply lacks source/caller effect: %v", err) + } + compileLLVMToObject(t, llc, triple, "imul-caller.ll", "imul-caller.o", ir) + }) + if !allAPIs { + continue + } + t.Run(triple+"/module", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "imul-caller-module.ll", "imul-caller-module.o", module.String()) + }) + t.Run(triple+"/go", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: triple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "imul-caller-go.ll", "imul-caller-go.o", translated.Module.String()) + }) + } + }) + } + } +} + +func TestX86SourceStackExplicitIMULKillsOldComparisonFlags(t *testing.T) { + for _, op := range []string{"IMULW", "IMULL", "IMULQ", "IMUL3W", "IMUL3L", "IMUL3Q"} { + t.Run(op, func(t *testing.T) { + row := op + " AX,BX" + if strings.HasPrefix(op, "IMUL3") { + row = op + " $7,AX,BX" + } + ins := x86YMMReadSource(t, row) + state := x86SourceStackState{numericFlags: true, callers: map[Reg]x86CallerPointerValue{}} + if !x86SourceStackFlagsStep(ins, &state) || state.numericFlags { + t.Fatal("IMUL retained complete old CMP flags instead of only CF/OF") + } + for _, branch := range []string{"JE", "JNE", "JLT", "JGE", "JLE", "JGT", "JMI", "JPL", "JHI", "JLS", "JPS", "JPC", "JCS", "JCC", "JOS", "JOC"} { + accepted := x86SourceStackFlagsStep(Instr{Op: Op(branch)}, &state) + want := branch == "JCS" || branch == "JCC" || branch == "JOS" || branch == "JOC" + if accepted != want { + t.Fatalf("%s after %s: accepted=%t want=%t", branch, op, accepted, want) + } + } + }) + } +} + +func TestX86GoCallerExplicitIMULRetainsContext(t *testing.T) { + for _, test := range []struct{ name, body string }{ + {"undefined-input", "MOVQ $7,BX\nIMULQ AX,BX"}, + {"undefined-old-destination", "MOVQ $7,AX\nIMULQ AX,BX"}, + {"partial-input", "MOVB $7,AX\nIMULQ AX,BX"}, + {"partial-word-result", "MOVQ $7,AX\nIMUL3W $7,AX,BX"}, + {"pointer-input", "MOVQ $7,BX\nIMULQ DI,BX"}, + {"pointer-old-destination", "MOVQ DI,BX\nIMULQ $7,BX"}, + {"pointer-survives-word-result", "MOVQ DI,BX\nMOVQ $7,AX\nIMUL3W $7,AX,BX"}, + {"unknown-memory", "MOVQ $7,BX\nIMULQ (R11),BX"}, + {"wide-length-read", "MOVQ $7,BX\nIMULQ 1(CX),BX"}, + {"native-return-pc", "MOVQ $7,BX\nIMULQ 8(BP),BX"}, + {"FP-pointer-input", "MOVQ $7,BX\nIMULQ a+0(FP),BX"}, + {"g-input", "MOVQ $7,BX\nIMULQ R14,BX"}, + {"g-unrestored", "MOVQ $7,AX\nIMUL3Q $7,AX,R14\nMOVQ R14,BX"}, + {"SP-destination", "MOVQ $7,BX\nIMULQ BX,SP"}, + {"BP-input", "MOVQ $7,BX\nIMULQ BP,BX"}, + {"implicit-IMUL", "MOVQ $7,AX\nMOVQ $7,BX\nIMULQ BX"}, + {"implicit-MUL", "MOVQ $7,AX\nMOVQ $7,BX\nMULQ BX"}, + {"implicit-DIV", "MOVQ $7,AX\nMOVQ $7,BX\nMOVQ $0,DX\nDIVQ BX"}, + {"implicit-IDIV", "MOVQ $7,AX\nMOVQ $7,BX\nMOVQ $0,DX\nIDIVQ BX"}, + {"unknown-CFG-input", "MOVQ $7,BX\nCMPQ BX,$0\nJE joined\nMOVQ $7,AX\njoined:\nIMULQ AX,BX"}, + } { + t.Run(test.name, func(t *testing.T) { + pkg, file, assembly, options := x86CallerExplicitIMULFixture(t, test.body) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved IMUL effect acquired authority: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module accepted unproved IMUL: %v", err) + } + ctx = llvm.NewContext() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: options.TargetTriple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + translated.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go API accepted unproved IMUL: %v", err) + } + }) + } +} + +func TestX86GoCallerExplicitIMULActualInitializedLocalRead(t *testing.T) { + // The retained actual caller declares `var res uint64`: Go initializes + // those complete numerical bytes before the call. This is not an unknown + // native entry region, nor permission to read an arbitrary result pointer. + _, file, _, options := x86CallerExplicitIMULFixture(t, "MOVQ $7,BX\nIMULQ (DX),BX") + if _, err := Translate(file, options); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerExplicitIMULFlagsAndSavedFlags(t *testing.T) { + for _, condition := range []string{"CC", "CS", "EQ", "NE", "GE", "LT", "GT", "LE", "HI", "LS", "MI", "PL", "OC", "OS", "PC", "PS"} { + for _, consumer := range []string{"SET", "CMOVQ", "J"} { + t.Run(consumer+condition, func(t *testing.T) { + body := "MOVQ $7,AX\nMOVQ $3,BX\nCMPQ AX,$0\nIMULQ AX,BX\n" + switch consumer { + case "SET": + body += "SET" + condition + " BL" + case "CMOVQ": + body += "CMOVQ" + condition + " AX,BX" + case "J": + body += "J" + condition + " joined\njoined:" + } + _, file, _, options := x86CallerExplicitIMULFixture(t, body) + _, err := Translate(file, options) + defined := condition == "CC" || condition == "CS" || condition == "OC" || condition == "OS" + if defined && err != nil || !defined && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("IMUL flag consumer defined=%t: %v", defined, err) + } + }) + } + } + for _, test := range []struct { + name, body string + accepted bool + }{ + {"restore-complete", "CMPQ AX,$0\nPUSHFQ\nIMULQ AX,BX\nPOPFQ\nSETEQ BL", true}, + {"restore-partial", "IMULQ AX,BX\nPUSHFQ\nCMPQ AX,$0\nPOPFQ\nSETEQ BL", false}, + {"restore-CF", "IMULQ AX,BX\nPUSHFQ\nCMPQ AX,$0\nPOPFQ\nSETCS BL", true}, + {"carry-after-multiply", "IMULQ AX,BX\nADCQ $1,BX", true}, + {"CMP-redefines-flags", "IMULQ AX,BX\nCMPQ AX,$0\nSETEQ BL", true}, + {"shift-does-not-resurrect-OF", "IMULQ AX,BX\nSHLQ $2,BX\nSETOS BL", false}, + } { + t.Run(test.name, func(t *testing.T) { + _, file, _, options := x86CallerExplicitIMULFixture(t, "MOVQ $7,AX\nMOVQ $3,BX\n"+test.body) + _, err := Translate(file, options) + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("FLAGS dependency accepted=%t: %v", test.accepted, err) + } + }) + } +} + +func TestX86SourceStackExplicitIMULWithoutCallerRemainsContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, suffix := range []string{"W", "L", "Q"} { + if arch == "386" && suffix == "Q" { + continue // Independently Go-rejected Q rows are in the encoder oracle. + } + for _, three := range []bool{false, true} { + body := "@MOV $7,AX\n@MOV $3,BX\nIMUL" + suffix + " AX,BX\n@MOV BX,@RESULT(SP)\nBYTE $0xc3" + if three { + body = strings.Replace(body, "IMUL"+suffix+" AX,BX", "IMUL3"+suffix+" $7,AX,BX", 1) + } + t.Run(fmt.Sprintf("%s/%s/three=%t", arch, suffix, three), func(t *testing.T) { + source, sig, _ := x86SourceStackNumericalFixture(arch, body) + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, x86SourceStackOptions(arch, "", sig)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("declaration-only source acquired complete numerical input proof: %v", err) + } + }) + } + } + } +} diff --git a/amd64_go_caller_float_compare.go b/amd64_go_caller_float_compare.go new file mode 100644 index 000000000..67d5599b2 --- /dev/null +++ b/amd64_go_caller_float_compare.go @@ -0,0 +1,38 @@ +package plan9asm + +// Comparison writes either complete predicate lanes or a fully defined K +// result. A K destination is never an old merge input; its optional source +// mask must still be initialized, even when source and destination alias. +func x86CallerFloatCompareStep(ins *Instr, form x86FloatCompareForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := form.byteWidth + if form.scalar { + width = form.bits / 8 + } + required := x86CallerVectorByteMask(int64(width)) + if form.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + for _, index := range []int{form.first, form.second} { + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + if form.maskResult { + state.masks[ins.Args[form.destination].Reg] = true + return true + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + known := ^uint64(0) + if form.legacy { + known = state.vectors[destination] | required + } else if form.scalar { + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] + upper := x86CallerVectorByteMask(16) &^ required + known = ^x86CallerVectorByteMask(16) | required | second&upper + } + state.vectors[destination] = known + return true +} diff --git a/amd64_go_caller_float_compare_test.go b/amd64_go_caller_float_compare_test.go new file mode 100644 index 000000000..d73712371 --- /dev/null +++ b/amd64_go_caller_float_compare_test.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerFloatingCompareCompleteInitializedForms(t *testing.T) { + for _, original := range x86FloatCompareGoForms("amd64") { + t.Run(original, func(t *testing.T) { + row := strings.NewReplacer("8(AX)", "8(DI)", "(AX)(CX*4)", "(DI)").Replace(original) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatCompareProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("independent source form not recognized: %v", err) + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z20\nVMOVUPS (SI),Z21\nKMOVQ AX,K1\n" + row + destination := ins.Args[form.destination].Reg + if form.maskResult { + body += "\nKMOVD " + string(destination) + ",(DX)" + } else { + index, _ := amd64VectorRegisterIndex(destination, form.byteWidth) + body += fmt.Sprintf("\nVMOVSS X%d,(DX)", index) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerFloatingCompareRegisterKnowledgeAndMasks(t *testing.T) { + rows := x86FloatCompareGoForms("amd64") + rows = append(rows, "VCMPPS $0,Z0,Z1,K1,K1", "VCMPSS $255,X0,X1,K1,K1") + for _, row := range rows { + if strings.Contains(row, "(AX)") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86FloatCompareProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + width := form.byteWidth + if form.scalar { + width = form.bits / 8 + } + required := x86CallerVectorByteMask(int64(width)) + first, second := x86CallerVectorRoot(ins.Args[form.first].Reg), x86CallerVectorRoot(ins.Args[form.second].Reg) + for seed := 0; seed < 64; seed++ { + upper := uint64(seed) * 0x391072d5e64b8a0f + state := x86SourceStackState{ + vectors: map[Reg]uint64{first: required, second: required | upper}, + masks: map[Reg]bool{"K1": true}, + } + if !x86CallerVectorMoveKnown(ins) || !x86CallerVectorMoveStep(&ins, &state, nil) { + t.Fatalf("complete registered compare effect rejected %s", row) + } + if form.maskResult { + if !state.masks[ins.Args[form.destination].Reg] || state.vectors[first] != required || state.vectors[second] != required|upper { + t.Fatalf("K-result changed input vectors or required old mask contents: %s", row) + } + continue + } + want := ^uint64(0) + if form.legacy { + want = required | upper + } else if form.scalar { + want = ^x86CallerVectorByteMask(16) | required | upper&(x86CallerVectorByteMask(16)&^required) + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("%s seed%d defined bytes=%#x want=%#x", row, seed, got, want) + } + } + for _, missing := range []int{form.first, form.second, form.mask} { + if missing < 0 { + continue + } + state := x86SourceStackState{ + vectors: map[Reg]uint64{first: required, second: required}, + masks: map[Reg]bool{"K1": true}, + } + if missing == form.mask { + state.masks[ins.Args[missing].Reg] = false + } else { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= 1 + } + vectors, masks := x86CloneCallerVectorState(state.vectors, state.masks) + if x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state.vectors, vectors) || !reflect.DeepEqual(state.masks, masks) { + t.Fatalf("%s missing role%d invented evidence or updated partial state", row, missing) + } + } + } +} + +func TestX86GoCallerFloatingCompareRejectsUnprovedInputAndEnvelope(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "CMPPS X0,X1,$0\nVMOVSS X1,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVCMPPS $0,Z0,Z1,K1,K2\nKMOVD K2,(DX)", + "unknown old K is read": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVCMPPS $0,Z0,Z1,K1,K1\nKMOVD K1,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ DI,K1\nVCMPPS $0,Z0,Z1,K1,K2\nKMOVD K2,(DX)", + "unknown base": "VMOVUPS (SI),X1\nCMPPS (BX),X1,$0\nVMOVSS X1,(DX)", + "native stack": "VMOVUPS (SI),X1\nCMPPS (SP),X1,$0\nVMOVSS X1,(DX)", + "pointer FP": "VMOVUPS (SI),X1\nCMPPS a+0(FP),X1,$0\nVMOVSS X1,(DX)", + "read scalar length": "VMOVUPS (SI),X1\nCMPPS (CX),X1,$0\nVMOVSS X1,(DX)", + "read output": "VMOVUPS (SI),X1\nCMPPS (DX),X1,$0\nVMOVSS X1,(DX)", + "write source": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nCMPPS X0,X1,$0\nVMOVSS X1,(DI)", + "wide output": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nCMPPS X0,X1,$0\nVMOVUPS X1,(DX)", + // A VEX memory load defines zero upper bytes. Legacy register MOVSS + // preserves X1's unknown upper bytes, which scalar VCMP copies and + // the following packed VCMP genuinely reads. + "unknown upper reused": "VMOVSS (DI),X0\nMOVSS X0,X1\nVCMPSS $0,X0,X1,X2\nVCMPPS $0,X2,X2,X3\nVMOVSS X3,(DX)", + "mixed CFG": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVMOVUPS (SI),X1\nCMPPS X0,X1,$0\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved compare acquired caller continuation: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_fma3.go b/amd64_go_caller_fma3.go new file mode 100644 index 000000000..33564eaeb --- /dev/null +++ b/amd64_go_caller_fma3.go @@ -0,0 +1,37 @@ +package plan9asm + +func x86CallerFMA3Step(ins *Instr, form amd64FMA3Form, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.spec.rawOnly || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := int64(form.byteWidth) + if form.spec.scalar { + width = int64(form.spec.laneBits / 8) + } + required := x86CallerVectorByteMask(width) + if form.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + // All three operands participate in every order, including the old + // destination in zeroing forms. An initialized unknown-valued mask is + // not a proof of inactive lanes; arithmetic cannot invent missing bytes. + for _, index := range []int{0, 1, form.destination} { + if index == form.memory { + continue + } + root := x86CallerVectorRoot(ins.Args[index].Reg) + if state.vectors[root]&required != required { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + known := ^uint64(0) + if form.spec.scalar { + // Scalar FMA preserves the old destination's remaining XMM bytes, + // independently of its low-lane mask, and zeroes higher Y/Z aliases. + upperX := x86CallerVectorByteMask(16) &^ required + known = ^x86CallerVectorByteMask(16) | required | state.vectors[destination]&upperX + } + state.vectors[destination] = known + return true +} diff --git a/amd64_go_caller_fma3_runtime_test.go b/amd64_go_caller_fma3_runtime_test.go new file mode 100644 index 000000000..62d5c185b --- /dev/null +++ b/amd64_go_caller_fma3_runtime_test.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86FMA3GoCallerNumericalFamily(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the separately required Linux/amd64 FMA3 caller runtime oracle") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + formats := x86FMA3NamedFormats("amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + // Caller proofs fingerprint the complete source selection. Keep that + // selection bounded instead of repeatedly hashing a giant fixture for + // every function. Each format still has its own actual typed Go caller, + // translation and numerical oracle; every batch releases its LLVM objects. + const batchSize = 64 + attempted, passed := 0, 0 + for start := 0; start < len(formats); start += batchSize { + end := start + batchSize + if end > len(formats) { + end = len(formats) + } + t.Run(fmt.Sprintf("formats_%04d_%04d", start, end), func(t *testing.T) { + attempted += end - start + runX86FMA3GoCallerNumericalBatch(t, llc, clang, formats[start:end], start, runner) + passed += end - start + }) + } + t.Logf("%d/%d attempted FMA3 formats passed (%d available): numerical low result, directed rounding, unknown-valued initialized mask, read-only inputs and full output canaries", passed, attempted, len(formats)) +} + +func runX86FMA3GoCallerNumericalBatch(t *testing.T, llc, clang string, formats []string, first int, runner []string) { + t.Helper() + var goSource, assembly, declarations, checks strings.Builder + goSource.WriteString("package caller\nimport \"unsafe\"\n") + for index, instruction := range formats { + name := fmt.Sprintf("fmacase%d", first+index) + opcode := strings.Fields(instruction)[0] + base := strings.Split(opcode, ".")[0] + bits := 32 + if strings.HasSuffix(base, "PD") || strings.HasSuffix(base, "SD") { + bits = 64 + } + declaration := x86CallerFixtureSource[strings.Index(x86CallerFixtureSource, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1) + if bits == 64 { + declaration = strings.ReplaceAll(declaration, "float32", "float64") + } + goSource.WriteString(declaration) + + bank := "X" + if strings.Contains(instruction, "Y1") { + bank = "Y" + } else if strings.Contains(instruction, "Z1") { + bank = "Z" + } + body := fmt.Sprintf("VMOVUPS (DI),%s0\nVMOVUPS (SI),%s1\nVMOVUPS 64(SI),%s2\nKMOVQ AX,K1\n", bank, bank, bank) + body += strings.ReplaceAll(instruction, "(AX)", "(DI)") + "\n" + store := "VMOVSS X2,(DX)" + if bits == 64 { + store = "VMOVSD X2,(DX)" + } + body += store + machine := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+name) + machine = strings.Replace(machine, "MOVL $7,AX", "MOVQ (CX),AX", 1) + assembly.WriteString(strings.Replace(machine, "MOVL AX,(DX)", body, 1)) + fmt.Fprintf(&declarations, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + + order, err := strconv.Atoi(base[len(base)-5 : len(base)-2]) + if err != nil { + t.Fatal(err) + } + mode := 0 + switch { + case strings.HasPrefix(base, "VFMADDSUB"): + mode = 4 + case strings.HasPrefix(base, "VFMSUBADD"): + mode = 5 + case strings.HasPrefix(base, "VFNMADD"): + mode = 2 + case strings.HasPrefix(base, "VFNMSUB"): + mode = 3 + case strings.HasPrefix(base, "VFMSUB"): + mode = 1 + } + rounding := 0 + for suffix, value := range map[string]int{".RD_SAE": 1, ".RU_SAE": 2, ".RZ_SAE": 3} { + if strings.Contains(opcode, suffix) { + rounding = value + } + } + fmt.Fprintf(&checks, " if (check(%s,%q,%d,%d,%d,%t,%t,%d)) return 1;\n", + name, instruction, bits, order, mode, strings.Contains(instruction, "K1"), strings.HasSuffix(opcode, ".Z"), rounding) + } + // Actual Go accepts every original instruction and supplies the exact + // typed caller. Portable LLVM/C execution is not native AVX512 execution + // and is not a whole third-party library/ledger pass. + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + pkg := x86CallerRuntimePackage(t, goSource.String(), assembly.String()) + ctx := llvm.NewContext() + defer ctx.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly.String()), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + oracle := x86FMA3CallerCOracle + declarations.String() + "\nint main(void) {\n" + checks.String() + " return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "fma3-caller-numerical", triple, translation.Module.String(), oracle, runner) +} + +const x86FMA3CallerCOracle = `#include +#include +#include +#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif + +#pragma STDC FENV_ACCESS ON +static int check(void (*fn)(void*,void*,void*,void*), const char *instruction, unsigned bits, + unsigned order, unsigned mode, bool masked, bool zero, + unsigned rounding) { + const int modes[] = {FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + for (uint64_t length=64; length<68; length++) { + uint8_t first[512], second[512], before_first[512], before_second[512]; + uint8_t output[48], expected[48]; + for (unsigned i=0; i<512/(bits/8); i++) { + if (bits == 32) { + float a=1.00000011920928955078125f; + float b=-1.0000002384185791015625f; + if (i != 0) { a=(float)((int)(i%7)-3)/4; b=(float)((int)(i%5)-2)/8; } + memcpy(first+i*4,&a,4); memcpy(second+i*4,&b,4); + } else { + double a=1.0000000000000002220446049250313; + double b=-1.0000000000000004440892098500626; + if (i != 0) { a=(double)((int)(i%7)-3)/4; b=(double)((int)(i%5)-2)/8; } + memcpy(first+i*8,&a,8); memcpy(second+i*8,&b,8); + } + } + memcpy(before_first,first,512); memcpy(before_second,second,512); + memset(output,0xa5,48); memcpy(expected,output,48); + const uint64_t original_length=length; + fn(first,second,output+16,&length); + if (length != original_length) return 1; + if (fesetround(modes[rounding])) return 2; + if (bits == 32) { + float first_value, second_value, old, a, b, c, result; + memcpy(&first_value,first,4); memcpy(&second_value,second,4); memcpy(&old,second+64,4); + a=old; b=first_value; c=second_value; + if (order == 213) { a=second_value; b=old; c=first_value; } + if (order == 231) { a=second_value; b=first_value; c=old; } + if (mode == 2 || mode == 3) a=-a; + if (mode == 1 || mode == 3 || mode == 4) c=-c; + result=fmaf(a,b,c); + if (masked && !(length&1)) result=zero ? 0 : old; + memcpy(expected+16,&result,4); + } else { + double first_value, second_value, old, a, b, c, result; + memcpy(&first_value,first,8); memcpy(&second_value,second,8); memcpy(&old,second+64,8); + a=old; b=first_value; c=second_value; + if (order == 213) { a=second_value; b=old; c=first_value; } + if (order == 231) { a=second_value; b=first_value; c=old; } + if (mode == 2 || mode == 3) a=-a; + if (mode == 1 || mode == 3 || mode == 4) c=-c; + result=fma(a,b,c); + if (masked && !(length&1)) result=zero ? 0 : old; + memcpy(expected+16,&result,8); + } + if (fesetround(FE_TONEAREST)) return 3; + if (memcmp(output,expected,48) || memcmp(first,before_first,512) || + memcmp(second,before_second,512)) { + fprintf(stderr,"FMA3 %s bits/order/mode/mask/zero/rounding/length %u/%u/%u/%u/%u/%u/%llu\n", + instruction,bits,order,mode,masked,zero,rounding,(unsigned long long)length); + for (unsigned i=16; i<16+bits/8; i++) { + fprintf(stderr,"%02x/%02x ",output[i],expected[i]); + } + fprintf(stderr,"\n"); + return 4; + } + } + return 0; +} +` diff --git a/amd64_go_caller_fma3_test.go b/amd64_go_caller_fma3_test.go new file mode 100644 index 000000000..707d6bb78 --- /dev/null +++ b/amd64_go_caller_fma3_test.go @@ -0,0 +1,134 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerFMA3InitializedNumericalInputs(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVFMADD231SS (DI),X0,X1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Y0\nVMOVUPS (SI),Y1\nVFMADD132PS (DI),Y0,Y1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVFNMADD213PD.BCST (DI),Z0,Z1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVFMADDSUB231PS.Z Z0,Z0,K1,Z1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVFMSUBADD132PD.RD_SAE Z0,Z0,Z1\nVMOVSS X1,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", ";"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerFMA3CompleteFamilyRegisterEffects(t *testing.T) { + for _, instruction := range x86FMA3NamedFormats("amd64") { + if strings.Contains(instruction, "(AX)") { + continue // Actual typed caller-memory/objects are checked separately. + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if !x86CallerVectorMoveKnown(ins) { + t.Fatalf("complete named FMA3 format has no shared effect: %s", instruction) + } + form, _, err := amd64FMA3Properties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, missing := range []int{-1, 0, 1, 2} { + state := x86SourceStackState{vectors: map[Reg]uint64{ + "Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0), + }, masks: map[Reg]bool{"K1": true}} + if missing >= 0 { + state.vectors[Reg(fmt.Sprintf("Z%d", missing))] &^= 1 + } + before, masks := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerFMA3Step(&ins, form, &state, nil) + if accepted != (missing == -1) || !reflect.DeepEqual(state.masks, masks) || + !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing input %d: invented numerical data or changed masks: %t/%v", instruction, missing, accepted, state.vectors) + } + } + } + for opcode, spec := range amd64FMA3Specs { + if !spec.rawOnly { + continue + } + ins := Instr{Op: opcode, x86Encoded: true, Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X2"}, + }} + if x86CallerVectorMoveKnown(ins) { + t.Fatalf("raw-only FP16 acquired a Go-named caller contract: %s", opcode) + } + } +} + +func TestX86GoCallerFMA3ScalarPreservesUnknownUpperXBytes(t *testing.T) { + for _, instruction := range []string{ + "VFMADD132SS X0,X1,X2", "VFMSUB213SD.RD_SAE X0,X1,X2", + "VFNMADD231SS.Z X0,X1,K1,X2", "VFNMSUB132SD.RU_SAE.Z X0,X1,K1,X2", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := amd64FMA3Properties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + low := x86CallerVectorByteMask(int64(form.spec.laneBits / 8)) + for _, oldUpper := range []uint64{0, 0xff00, 0xff000, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{ + "Z0": low, "Z1": low, "Z2": low | oldUpper, + }, masks: map[Reg]bool{"K1": true}} + if !x86CallerFMA3Step(&ins, form, &state, nil) { + t.Fatalf("initialized low inputs incorrectly require unknown upper X data: %s", instruction) + } + var expected uint64 + for index := 0; index < 64; index++ { + bit := uint64(1) << uint(index) + if index >= 16 || index < form.spec.laneBits/8 || oldUpper&bit != 0 { + expected |= bit + } + } + if state.vectors["Z2"] != expected { + t.Fatalf("%s changed byte %x initialization: got %x want %x", instruction, oldUpper, state.vectors["Z2"], expected) + } + } + } +} + +func TestX86GoCallerFMA3CannotInventNumericalInputs(t *testing.T) { + for _, body := range []string{ + "VFMADD231SS (DI),X0,X1\nVMOVSS X1,(DX)", + "VMOVSS (DI),X0\nVFMADD231SS X0,X0,X1\nVMOVSS X1,(DX)", + "VMOVSS (DI),X0\nVFMADD231SS.Z X0,X0,K1,X0\nVMOVSS X0,(DX)", + "VMOVSS (DI),X0\nKMOVQ DI,K1\nVFMADD231SS.Z X0,X0,K1,X0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (BX),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS a+0(FP),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (CX),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (R14),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS 0(SP),Y0,Y0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVFMADD231PS (DI),Y0,Y0\nVMOVUPS Y0,(DX)", + "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),Y0\njoined:\nVFMADD231PS (DI),Y0,Y0\nVMOVSS X0,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", ";"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved FMA input acquired caller continuation: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_gp_simd_transfer.go b/amd64_go_caller_gp_simd_transfer.go new file mode 100644 index 000000000..f1e6a5e38 --- /dev/null +++ b/amd64_go_caller_gp_simd_transfer.go @@ -0,0 +1,42 @@ +package plan9asm + +func x86CallerGPSIMDTransferStep(ins *Instr, f x86GPSIMDTransferForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.mmx || f.spec.rawOnly { + return false + } + width := int64(f.bits / 8) + lowMask := x86CallerVectorByteMask(width) + var known uint64 + if f.sourceX { + known = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] & lowMask + } else if f.memory < 0 { + root, valid := x86CallerGP(ins.Args[0].Reg) + _, pointer := state.callers[root] + if ins.Args[0].Kind != OpReg || !valid || root == SP || root == BP || pointer { + return false + } + known = uint64(state.known[root]) & lowMask + } else { + known = lowMask + } + if !f.destinationX && known != lowMask { + return false + } + if f.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, width, f.memoryWrite) { + return false + } + if f.destinationX { + root := x86CallerVectorRoot(ins.Args[1].Reg) + if f.spec.legacy { + state.vectors[root] = state.vectors[root]&^x86CallerVectorByteMask(16) | + (x86CallerVectorByteMask(16) &^ lowMask) | known + } else { + state.vectors[root] = ^lowMask | known + } + return true + } + if f.memoryWrite { + return true + } + return x86CallerNumericWrite(ins.Args[1].Reg, width, state) +} diff --git a/amd64_go_caller_horizontal_float.go b/amd64_go_caller_horizontal_float.go new file mode 100644 index 000000000..36c23d6b4 --- /dev/null +++ b/amd64_go_caller_horizontal_float.go @@ -0,0 +1,24 @@ +package plan9asm + +func x86CallerHorizontalFloatStep(ins *Instr, form x86HorizontalFloatForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.memory >= 0 { + // SSE3's legacy m128 requires alignment. A slice/nonalias witness + // does not establish that; VEX forms allow unaligned memory. + if !form.spec.vector || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.byteWidth), false) { + return false + } + } + required := x86CallerVectorByteMask(int64(form.byteWidth)) + for _, index := range []int{form.first, form.second} { + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + known := ^uint64(0) + if !form.spec.vector { + known = state.vectors[destination] | required + } + state.vectors[destination] = known + return true +} diff --git a/amd64_go_caller_horizontal_float_test.go b/amd64_go_caller_horizontal_float_test.go new file mode 100644 index 000000000..bd7b4ca70 --- /dev/null +++ b/amd64_go_caller_horizontal_float_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerHorizontalFloatOriginalSourceReachesObjects(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Y0\nLONG $0xc07cffc5\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Y0\nVMOVUPS (SI),Y1\nVHADDPS Y0,Y1,Y2\nVMOVSS X2,(DX)", + "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nHSUBPD X0,X1\nVMOVSS X1,(DX)", + } { + t.Run(strings.Split(body, "\n")[1], func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerHorizontalFloatWholeFamilyByteEffects(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + if strings.Contains(row, "(BX)") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86HorizontalFloatProperties(ins, "amd64") + if !recognized || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("missing shared effect %s: %v", row, err) + } + required := x86CallerVectorByteMask(int64(form.byteWidth)) + for _, missing := range []int{-1, form.first, form.second, form.destination} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": required, "Z1": required, "Z2": 0}} + if missing >= 0 { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= uint64(1) << uint(form.byteWidth-1) + } + before, _ := x86CloneCallerVectorState(state.vectors, nil) + accepted := x86CallerHorizontalFloatStep(&ins, form, &state, nil) + want := missing == -1 || missing == form.destination && form.spec.vector + if accepted != want || !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing %d: %t want %t, invented high source bytes", row, missing, accepted, want) + } + } + for _, upper := range []uint64{0, 0xff00000000, ^required} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": required, "Z1": required | upper, "Z2": upper}} + if !x86CallerHorizontalFloatStep(&ins, form, &state, nil) { + t.Fatal("fully initialized sources rejected") + } + want := ^uint64(0) + if !form.spec.vector { + want = required | upper + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("%s definitions %x want %x", row, got, want) + } + } + } +} + +func TestX86GoCallerHorizontalFloatAllOpcodesAndMemoryBounds(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + memory := strings.Contains(row, "(BX)") + legacy := !strings.HasPrefix(row, "V") + row = strings.Replace(row, "(BX)", "(DI)", 1) + destination := "X1" + if !legacy { + destination = "X2" + } + body := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n%s\nVMOVSS %s,(DX)", row, destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + err := x86CallerCoreTranslate(t, pkg, assembly, triple) + if legacy && memory { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved legacy alignment: %s: %v", row, err) + } + } else if err != nil { + t.Fatalf("%s/%s: %v", row, triple, err) + } + } + } +} + +func TestX86GoCallerHorizontalFloatDoesNotInventPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VHADDPS X0,X1,X2\nVMOVSS X2,(DX)", + "not a zero idiom": "VHSUBPS X0,X0,X2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVUPS (DI),X0\nHSUBPD X0,X1\nVMOVSS X1,(DX)", + "partially defined source": "VMOVSS X0,X1,X2\nVMOVUPS (DI),X0\nVHADDPS X0,X2,X1\nVMOVSS X1,(DX)", + "unknown input": "VMOVUPS (DI),X0\nVHADDPD (BX),X0,X1\nVMOVSS X1,(DX)", + "physical SP": "VMOVUPS (DI),X0\nVHADDPS 0(SP),X0,X1\nVMOVSS X1,(DX)", + "FP pointer bits": "VMOVUPS (DI),X0\nVHSUBPS a+0(FP),X0,X1\nVMOVSS X1,(DX)", + "length as vector": "VMOVUPS (DI),X0\nVHADDPD (CX),X0,X1\nVMOVSS X1,(DX)", + "g as input": "VMOVUPS (DI),X0\nVHADDPS (R14),X0,X1\nVMOVSS X1,(DX)", + "result wider than output": "VMOVUPS (DI),Y0\nVHADDPS Y0,Y0,Y1\nVMOVUPS Y1,(DX)", + "write original input": "VMOVUPS (DI),X0\nVHADDPS X0,X0,X1\nVMOVSS X1,(DI)", + "CFG missing incoming value": "CMPQ AX,$0\nJE join\nVMOVUPS (DI),X0\njoin:\nVHADDPS X0,X0,X1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved horizontal state passed: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_horizontal_integer_test.go b/amd64_go_caller_horizontal_integer_test.go new file mode 100644 index 000000000..0a87f3f68 --- /dev/null +++ b/amd64_go_caller_horizontal_integer_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// This first-party caller closes its own source/ABI boundary. The horizontal +// operation has no exemption from the original raw-return stack certificate. +func TestX86GoCallerHorizontalIntegerOriginalSourceThreeAPIs(t *testing.T) { + for _, body := range []string{ + "VMOVDQU (DI),Y0\nVMOVDQU (SI),Y1\nVPHADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "VMOVDQU (SI),Y1\nVPHADDD (DI),Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(strings.Split(body, "\n")[1], func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve} + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + if ir, err := Translate(file, options); err != nil { + t.Error("text", err) + } else { + compileLLVMToObject(t, llc, options.TargetTriple, "horizontal-caller-text.ll", "horizontal-caller-text.o", ir) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Error("module", err) + } else { + compileLLVMToObject(t, llc, options.TargetTriple, "horizontal-caller-module.ll", "horizontal-caller-module.o", module.String()) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); err != nil { + t.Error("selected Go caller", err) + } + }) + } +} + +func TestX86GoCallerHorizontalIntegerCompleteNamedAndRawThreeAPIs(t *testing.T) { + for _, raw := range []bool{false, true} { + var body strings.Builder + for _, op := range x86HorizontalIntegerOpcodes { + rows := []string{op + " X0,X1"} + for _, bank := range []string{"X", "Y"} { + rows = append(rows, fmt.Sprintf("V%s %s0,%s1,%s2", op, bank, bank, bank), fmt.Sprintf("V%s (DI),%s1,%s2", op, bank, bank)) + } + for _, row := range rows { + body.WriteString("VMOVDQU (DI),Y0\nVMOVDQU (SI),Y1\n") + if raw { + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) < 2 || code[len(code)-1] != 0xc3 { + t.Fatal("original Go horizontal bytes have no exact near RET", row, code) + } + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&body, "BYTE $0x%02x\n", value) + } + } else { + body.WriteString(row + "\n") + } + result := "X2" + if !strings.HasPrefix(row, "V") { + result = "X1" + } + fmt.Fprintf(&body, "VMOVSS %s,(DX)\n", result) + } + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body.String(), 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(fmt.Sprint(raw, "/", triple), func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-caller-text.ll", "horizontal-caller-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-caller-module.ll", "horizontal-caller-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerHorizontalIntegerCompleteMemoryCertificates(t *testing.T) { + for _, op := range x86HorizontalIntegerOpcodes { + for _, bank := range []string{"X", "Y"} { + row := fmt.Sprintf("V%s (DI),%s0,%s1", op, bank, bank) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVDQU (SI),Y0\n"+row+"\nVMOVSS X1,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86HorizontalIntegerProperties(ins, "amd64") + if !known || err != nil || form.memory < 0 { + continue + } + found = true + access, exists := ins.x86CallerMemory[form.memory] + if !exists || access.width != int64(form.byteWidth) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatal("full horizontal source envelope lost its instruction identity", row) + } + for _, change := range []string{"opcode", "source", "destination", "width", "address", "direction", "address size"} { + access := access + local := ins + local.Args = append([]Operand(nil), ins.Args...) + local.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch change { + case "opcode": + local.Op = "VPHADDSW" + if local.Op == ins.Op { + local.Op = "VPHSUBSW" + } + case "source": + local.Args[form.second].Reg = Reg(bank + "7") + case "destination": + local.Args[form.destination].Reg = Reg(bank + "7") + case "width": + access.width-- + local.x86CallerMemory[form.memory] = access + case "address": + local.Args[form.memory].Mem.Off++ + case "direction": + access.write = true + local.x86CallerMemory[form.memory] = access + case "address size": + local.x86AddressBits = 32 + } + if x86CallerMemoryCertified(local, form.memory) { + t.Fatal("edited horizontal instruction borrowed stale memory identity", row, change) + } + } + } + if !found { + t.Fatal("fixture did not consume horizontal memory", row) + } + } + } +} + +func TestX86GoCallerHorizontalIntegerRetainsContextBoundaries(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVDQU (DI),Y1\nVPHADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVDQU (DI),Y0\nVPHSUBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVDQU (DI),X0\nPHADDSW X0,X1\nVMOVSS X1,(DX)", + "unknown same subtract": "VPHSUBD Y0,Y0,Y2\nVMOVSS X2,(DX)", + "partial upper input": "MOVUPS (DI),X0\nVMOVDQU (SI),Y1\nVPHADDW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown pointer": "VMOVDQU (SI),Y0\nVPHADDD (BX),Y0,Y1\nVMOVSS X1,(DX)", + "full width length envelope": "VMOVDQU (SI),Y0\nVPHADDW (CX),Y0,Y1\nVMOVSS X1,(DX)", + "full width scalar result envelope": "VMOVDQU (SI),Y0\nVPHSUBSW (DX),Y0,Y1\nVMOVSS X1,(DX)", + "physical SP": "VMOVDQU (SI),Y0\nVPHADDD (SP),Y0,Y1\nVMOVSS X1,(DX)", + "physical BP": "VMOVDQU (SI),Y0\nVPHSUBD (BP),Y0,Y1\nVMOVSS X1,(DX)", + "private stack alias": "MOVQ SP,BX\nVMOVDQU (SI),Y0\nVPHADDW (BX),Y0,Y1\nVMOVSS X1,(DX)", + "g memory": "VMOVDQU (SI),Y0\nVPHADDSW (R14),Y0,Y1\nVMOVSS X1,(DX)", + "FP pointer data": "VMOVDQU (SI),X0\nVPHADDD a+0(FP),X0,X1\nVMOVSS X1,(DX)", + "legacy native alignment": "VMOVDQU (SI),X0\nPHADDW (DI),X0\nVMOVSS X0,(DX)", + "MMX state": "PHADDD M0,M1\nMOVL AX,(DX)", + "pointer input write": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nVMOVSS X1,(DI)", + "wide result write": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nVMOVDQU Y1,(DX)", + "unrestored SP": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nVMOVSS X1,(DX)", + "CFG missing source": "VMOVDQU (DI),Y0\nCMPQ AX,$0\nJE join\nVMOVDQU (SI),Y1\njoin:\nVPHADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown effect": "VMOVDQU (DI),Y0\nVPHADDD Y0,Y0,Y1\nXCHGL AX,(DI)\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + if name == "unrestored SP" { + assembly = strings.Replace(assembly, "BYTE $0xc3", "SUBQ $8,SP\nBYTE $0xc3", 1) + } + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved %s acquired source/caller privilege: %v", name, err) + } + }) + } + ins := x86YMMReadSource(t, "PHADDD M0,M1") + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0)}} + state = x86CloneCallerState(state) + before := x86CloneCallerState(state) + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(before, state) { + t.Fatal("MMX grammar inherited a physical caller state proof") + } +} diff --git a/amd64_go_caller_immediate_shuffle.go b/amd64_go_caller_immediate_shuffle.go new file mode 100644 index 000000000..f52ff0c46 --- /dev/null +++ b/amd64_go_caller_immediate_shuffle.go @@ -0,0 +1,36 @@ +package plan9asm + +// Byte availability follows the exact immediate permutation. No unused byte, +// initialized mask, or shadow zero value establishes additional input data. +func x86CallerImmediateShuffleStep(ins *Instr, form x86ImmediateShuffleForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.spec.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + var source uint64 + if form.memory >= 0 { + // Caller-region identity does not promise legacy m128 alignment. + // E4NF memory keeps its full native tuple, including at mask zero. + if form.spec.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + source = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + source = state.vectors[x86CallerVectorRoot(form.source.Reg)] + } + known := uint64(0) + for byteIndex := 0; byteIndex < form.byteWidth; byteIndex++ { + known |= (source >> uint(form.sourceByte(byteIndex)) & 1) << uint(byteIndex) + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old // Unknown initialized K may retain any old lane. + } + if form.spec.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_insert_extract.go b/amd64_go_caller_insert_extract.go new file mode 100644 index 000000000..fb4e57e32 --- /dev/null +++ b/amd64_go_caller_insert_extract.go @@ -0,0 +1,62 @@ +package plan9asm + +func x86CallerInsertExtractStep(ins *Instr, f x86InsertExtractForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.mask >= 0 && !state.masks[ins.Args[f.mask].Reg] || f.mask >= 0 && f.memoryWrite { + return false + } + if f.memory >= 0 && !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.spec.chunk), f.memoryWrite) { + return false + } + source := state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] + low := x86CallerVectorByteMask(int64(f.spec.chunk)) + if !f.spec.insert { + if source>>uint(f.offset)&low != low { + return false + } + if f.memoryWrite { + return true + } + if f.spec.scalar { + return x86CallerNumericWrite(ins.Args[f.destination].Reg, 8, state) + } + known := low + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + if f.mask >= 0 && !f.zeroing { + known &= state.vectors[destination] + } + state.vectors[destination] = ^low | known + return true + } + base := state.vectors[x86CallerVectorRoot(ins.Args[f.base].Reg)] + outputMask := x86CallerVectorByteMask(int64(f.output)) + if f.spec.scalar { + insertLane := int(f.immediate>>4) & 3 + insertMask := uint64(15) << uint(insertLane*4) + knownSource := source >> uint((f.immediate>>6)*4) & 15 + if f.memory >= 0 { + knownSource = 15 + } + known := base&^insertMask | knownSource<= 0 && !f.zeroing { + known &= state.vectors[x86CallerVectorRoot(ins.Args[f.destination].Reg)] + } + state.vectors[x86CallerVectorRoot(ins.Args[f.destination].Reg)] = ^outputMask | known&outputMask + return true +} diff --git a/amd64_go_caller_insert_extract_test.go b/amd64_go_caller_insert_extract_test.go new file mode 100644 index 000000000..268e58c29 --- /dev/null +++ b/amd64_go_caller_insert_extract_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerInsertExtractOriginalSourceReachesObjects(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Y0\nLONG $0x197de3c4\nWORD $0x01c1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Z0\nVEXTRACTF64X4 $1,Z0,Y1\nVMOVSS X1,(DX)", + "VMOVUPS (DI),Y0\nVMOVUPS (SI),X1\nVINSERTF128 $1,X1,Y0,Y2\nVMOVSS X2,(DX)", + "VMOVUPS (DI),X0\nVEXTRACTPS $3,X0,AX\nMOVL AX,(DX)", + "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVINSERTPS $64,X1,X0,X2\nVMOVSS X2,(DX)", + } { + t.Run(strings.Split(body, "\n")[1], func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerInsertExtractWholeFamilyByteKnowledge(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + f, recognized, err := x86InsertExtractProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatal(err) + } + if f.memory >= 0 { + continue + } + if f.spec.scalar && !f.spec.insert { + continue // full typed scalar-state coverage is exercised via public API below + } + for _, missing := range []int{-1, 0, 7, 15, 16, 31, 32, 63} { + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + if missing >= 0 { + state.vectors[x86CallerVectorRoot(ins.Args[f.source].Reg)] &^= uint64(1) << uint(missing) + } + accepted := x86CallerInsertExtractStep(&ins, f, &state, nil) + if !f.spec.insert { + want := missing < f.offset || missing >= f.offset+f.output + if accepted != want { + t.Fatalf("%s missing source byte %d: %t want %t", row, missing, accepted, want) + } + } else if !f.spec.scalar && accepted != (missing < 0 || missing >= f.spec.chunk) { + t.Fatalf("%s missing insertion byte %d: %t", row, missing, accepted) + } + } + } + for _, row := range []string{"VEXTRACTF128 $1,Y1,X0", "VINSERTI128 $1,X0,Y1,Y2", "VINSERTPS $0,X0,X1,X2"} { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + f, _, _ := x86InsertExtractProperties(ins, "amd64") + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": ^uint64(0), "Z2": 0}, masks: map[Reg]bool{}} + if !x86CallerInsertExtractStep(&ins, f, &state, nil) || state.vectors[x86CallerVectorRoot(ins.Args[f.destination].Reg)] != ^uint64(0) { + t.Fatalf("complete inputs did not establish physical upper-zero: %s", row) + } + } +} + +func TestX86GoCallerInsertExtractAllOpcodesAndUnknownPermissions(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + row = strings.Replace(row, "(BX)", "(DI)", 1) + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + f, _, _ := x86InsertExtractProperties(ins, "amd64") + if f.memoryWrite { + continue // caller's scalar result is only four bytes, not vector output + } + store := fmt.Sprintf("VMOVSS X%d,(DX)", ins.Args[f.destination].Reg[len(ins.Args[f.destination].Reg)-1]-'0') + if f.spec.scalar && !f.spec.insert { + store = "MOVL AX,(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nKMOVQ AX,K1\n"+row+"\n"+store, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s: %v", row, err) + } + } + for name, body := range map[string]string{ + "unknown selected extract": "VEXTRACTF128 $1,Y0,X1\nVMOVSS X1,(DX)", + "unknown inserted bytes": "VMOVUPS (DI),Y1\nVINSERTF128 $1,X0,Y1,Y2\nVEXTRACTF128 $1,Y2,X2\nVMOVSS X2,(DX)", + "uninitialized retained base": "VMOVUPS (DI),X0\nVINSERTI128 $1,X0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVEXTRACTI32X4.Z $0,Z0,K1,X1\nVMOVSS X1,(DX)", + "unknown masked merge destination": "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVEXTRACTF32X4 $0,Z0,K1,X1\nVMOVSS X1,(DX)", + "pointer FP not numerical": "VMOVUPS (DI),Y1\nVINSERTF128 $0,a+0(FP),Y1,Y2\nVMOVSS X2,(DX)", + "unknown memory even zero mask": "VMOVUPS (DI),Z1\nMOVQ $0,AX\nKMOVQ AX,K1\nVINSERTF32X4.Z $0,(BX),Z1,K1,Z2\nVMOVSS X2,(DX)", + "read source SP": "VMOVUPS (DI),Y1\nVINSERTI128 $0,(SP),Y1,Y2\nVMOVSS X2,(DX)", + "source length too narrow": "VMOVUPS (DI),Y1\nVINSERTF128 $0,(CX),Y1,Y2\nVMOVSS X2,(DX)", + "write input": "VMOVUPS (DI),X0\nVEXTRACTPS $0,X0,(DI)", + "write g": "VMOVUPS (DI),X0\nVEXTRACTPS $0,X0,R14", + "CFG incomplete": "CMPQ AX,$0\nJE join\nVMOVUPS (DI),Y0\njoin:\nVEXTRACTF128 $1,Y0,X1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown source permission invented: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_lea_runtime_test.go b/amd64_go_caller_lea_runtime_test.go new file mode 100644 index 000000000..a1fc2e2ac --- /dev/null +++ b/amd64_go_caller_lea_runtime_test.go @@ -0,0 +1,216 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86CallerLEARuntimeForm struct { + name, suffix string + bits, scale int +} + +func x86CallerLEARuntimeForms() []x86CallerLEARuntimeForm { + var forms []x86CallerLEARuntimeForm + for _, width := range []struct { + suffix string + bits int + }{{"W", 16}, {"L", 32}, {"Q", 64}} { + for _, scale := range []int{1, 2, 4, 8} { + forms = append(forms, x86CallerLEARuntimeForm{ + name: fmt.Sprintf("kernel%s%d", width.suffix, scale), suffix: width.suffix, + bits: width.bits, scale: scale, + }) + } + } + return forms +} + +func x86CallerLEARuntimeSources() (goSource, assembly, goTest, cOracle string) { + var declarations, machine, tests, c strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + tests.WriteString(`package caller +import "testing" +func TestOriginalLEAMathematics(t *testing.T) { + values := []uint64{0, 1, 7, 19, 65535, 1<<16, 1<<31, 1<<32-1, 1<<32, 1<<63, ^uint64(0)} +`) + c.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86CallerLEARuntimeForms() { + fmt.Fprintf(&declarations, ` +//go:noescape +func %[1]s(a, b, res, length unsafe.Pointer) +func Call%[1]s(x, y []uint64) uint64 { + var res uint64 + length := len(x) + %[1]s(unsafe.Pointer(unsafe.SliceData(x)), unsafe.Pointer(unsafe.SliceData(y)), + unsafe.Pointer(&res), unsafe.Pointer(&length)) + return res +} +`, form.name) + fmt.Fprintf(&machine, `TEXT ·%s(SB),4,$0-32 +MOVQ a+0(FP),DI +MOVQ b+8(FP),SI +MOVQ res+16(FP),DX +MOVQ length+24(FP),CX +BYTE $0x55 +WORD $0x8948 +BYTE $0xe5 +LONG $0xf8e48348 +MOVQ (DI),R9 +MOVQ (SI),BX +MOVQ $0x1122334455667788,AX +LEA%s -7(R9)(BX*%d),AX +MOVQ AX,(DX) +WORD $0x8948 +BYTE $0xec +BYTE $0x5d +BYTE $0xc3 +`, form.name, form.suffix, form.scale) + fmt.Fprintf(&tests, ` + for _, x := range values { + for _, y := range values { + a, b := []uint64{x, 23}, []uint64{y, 37} + canary := [4]uint64{11, 41, 53, 67} + want := x + y*%[1]d - 7 +`, form.scale) + switch form.bits { + case 16: + tests.WriteString(" want = 0x1122334455660000 | uint64(uint16(want))\n") + case 32: + tests.WriteString(" want = uint64(uint32(want))\n") + } + fmt.Fprintf(&tests, ` + if got := Call%[1]s(a, b); got != want || a[0] != x || a[1] != 23 || b[0] != y || b[1] != 37 || + canary != [4]uint64{11, 41, 53, 67} { + t.Fatalf("%[1]s x=%%#x y=%%#x got=%%#x want=%%#x", x, y, got, want) + } + } + } +`, form.name) + fmt.Fprintf(&c, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + } + tests.WriteString("}\n") + c.WriteString(`int main(void) { + const uint64_t values[] = {0,1,7,19,65535,UINT64_C(1)<<16,UINT64_C(1)<<31,UINT32_MAX, + UINT64_C(1)<<32,UINT64_C(1)<<63,UINT64_MAX}; +`) + for _, form := range x86CallerLEARuntimeForms() { + fmt.Fprintf(&c, ` + for (unsigned i=0; i= 0 { + // Preserve the existing packed-move boundary: ordinary numerical + // SliceData is not proof of the aligned opcode's native address. + if form.aligned { + return false + } + if form.memoryWrite && state.vectors[x86CallerVectorRoot(form.source.Reg)]&mask != mask { + return false + } + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, width, form.memoryWrite) { + return false + } + if form.memoryWrite { + return true + } + known = mask + } else { + // Register copies propagate unknown source bytes without making + // them numeric. A later observable consumer still needs its bytes. + known = state.vectors[x86CallerVectorRoot(form.source.Reg)] & mask + } + destination := x86CallerVectorRoot(form.destination.Reg) + state.vectors[destination] = state.vectors[destination]&^mask | known + return true // Legacy writes preserve every upper Y/Z byte and all FLAGS. +} diff --git a/amd64_go_caller_legacy_moves_test.go b/amd64_go_caller_legacy_moves_test.go new file mode 100644 index 000000000..e3dc579a9 --- /dev/null +++ b/amd64_go_caller_legacy_moves_test.go @@ -0,0 +1,215 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +// The independent Go directory includes all twelve yxmov opcodes and MOVOA's +// frontend alias. Sharing yxmov does not share lane, source or memory effects. +var x86CallerLegacyMoveDirectory = []struct { + name, prefix, bytes string + width int64 + half, aligned bool +}{ + {"MOVAPD", "Pe", "0x28, 0x29", 16, false, true}, + {"MOVAPS", "Pm", "0x28, 0x29", 16, false, true}, + {"MOVO", "Pe", "0x6f, 0x7f", 16, false, true}, + {"MOVOU", "Pf3", "0x6f, 0x7f", 16, false, false}, + {"MOVHPD", "Pe", "0x16, 0x17", 8, true, false}, + {"MOVHPS", "Pm", "0x16, 0x17", 8, true, false}, + {"MOVLPD", "Pe", "0x12, 0x13", 8, true, false}, + {"MOVLPS", "Pm", "0x12, 0x13", 8, true, false}, + {"MOVSD", "Pf2", "0x10, 0x11", 8, false, false}, + {"MOVSS", "Pf3", "0x10, 0x11", 4, false, false}, + {"MOVUPD", "Pe", "0x10, 0x11", 16, false, false}, + {"MOVUPS", "Pm", "0x10, 0x11", 16, false, false}, + {"MOVOA", "Pe", "0x6f, 0x7f", 16, false, true}, +} + +func TestX86GoCallerLegacyMoveCompleteGoRows(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A(MOV\w+), yxmov, (\w+), opBytes\{([^}]+)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 12 { + t.Fatalf("actual Go yxmov opcode/attribute inventory changed: %d", len(rows)) + } + for index, row := range rows { + want := x86CallerLegacyMoveDirectory[index] + if row[1] != want.name || row[2] != want.prefix || row[3] != want.bytes { + t.Fatalf("actual Go opcode attributes differ: %v want %+v", row, want) + } + } + table := regexp.MustCompile(`(?s)var yxmov = \[\]ytab\{\n(.*?)\n\}`).FindSubmatch(data) + if len(table) != 2 || strings.Join(strings.Fields(string(table[1])), " ") != + "{Zm_r_xm, 1, argList{Yxm, Yxr}}, {Zr_m_xm, 1, argList{Yxr, Yxm}}," { + t.Fatal("actual Go operand classes, zcases or row-byte advances changed") + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || !strings.Contains(string(aliases), `instructions["MOVOA"] = x86.AMOVO`) { + t.Fatal("actual Go MOVOA frontend identity changed", err) + } +} + +func x86CallerLegacyMoveInstruction(t *testing.T, text string) Instr { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+text+"\nRET\n") + if err != nil { + t.Fatal(err) + } + return file.Funcs[0].Instrs[1] +} + +func TestX86GoCallerLegacyMoveRegisteredEffects(t *testing.T) { + // A transfer-domain model, not a package/caller authority producer. Complete + // actual source/signature binding is tested separately below. Input slices + // retain the existing numerical-origin/nonalias contract, not allocation bounds. + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{ + {kind: x86CallerSliceInput, width: 4}, + {kind: x86CallerFreshResult, width: 8}, + }} + for _, row := range x86CallerLegacyMoveDirectory { + for _, operands := range []string{"X0,X1", "(DI),X1", "X0,(DX)"} { + t.Run(row.name+"/"+operands, func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, row.name+" "+operands) + original := fmt.Sprintf("%#v", ins) + if known := x86CallerVectorMoveKnown(ins); known == row.half { + t.Errorf("caller known axis: got %t, half-unregistered=%t", known, row.half) + } + if x86SourceStackInstructionKnown(ins, "MOVQ", "Q", "amd64") { + t.Error("a move shape alone acquired non-caller source-stack admission") + } + if original != fmt.Sprintf("%#v", ins) { + t.Fatal("known/grammar observation changed original instruction") + } + state := x86SourceStackState{ + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}, DX: {argument: 1}}, + vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": 1 << 40}, + known: map[Reg]uint8{}, masks: map[Reg]bool{}, partialNumericFlags: x86ByteFlagCF | x86ByteFlagOF, + } + wantValid := !row.half && (!row.aligned || operands == "X0,X1") && + (operands != "X0,(DX)" || row.width <= 8) + if valid := x86CallerVectorMoveStep(&ins, &state, proof); valid != wantValid { + t.Errorf("caller step axis: got %t want %t", valid, wantValid) + } + if !x86SourceStackFlagsStep(ins, &state) || x86SourceStackNumericFlagMask(&state) != x86ByteFlagCF|x86ByteFlagOF { + t.Error("legacy move changed or fabricated numerical flags") + } + memory := -1 + if operands == "(DI),X1" { + memory = 0 + } else if operands == "X0,(DX)" { + memory = 1 + } + if memory >= 0 { + width, write, known := x86InstructionMemoryShape(ins, memory, "amd64") + if known == row.half || known && (width != row.width || write != (memory == 1)) { + t.Errorf("memory-shape axis differs: %d/%t/%t", width, write, known) + } + if wantValid && (!x86CallerMemoryCertified(ins, memory) || ins.x86CallerMemory[memory].width != row.width) { + t.Error("accepted read/write lacks its exact instruction-local native width") + } + } else if !row.half { + want := uint64(1)<<40 | 0xffff + if row.width < 16 { + want = uint64(1)<<40 | x86CallerVectorByteMask(row.width) + } + if state.vectors["Z1"] != want { + t.Errorf("register transfer fabricated/cleared old X/Y/Z bytes: %x want %x", state.vectors["Z1"], want) + } + } + }) + } + } +} + +func TestX86GoCallerLegacyMoveActualSourceProducer(t *testing.T) { + for _, row := range x86CallerLegacyMoveDirectory { + t.Run(row.name, func(t *testing.T) { + body := row.name + " (DI),X1" + if row.aligned { + // The current caller contract has no native alignment witness. + // Register recognition must not authorize aligned memory. + body = "VMOVUPS (DI),X1\n" + row.name + " X1,X2" + } + assembly := "TEXT ·kernel(SB),4,$0-32\nMOVQ a+0(FP),DI\n" + body + "\nBYTE $0xc3\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + sig := sigs["caller.kernel"] + if sig.Ret != Void || len(sig.Frame.Results) != 0 || sig.x86CallerRegions == nil || + !x86CallerRegionSourceMatches(sig, file.Funcs[0], "amd64") { + t.Fatal("actual original source and pointer-only Void signature prerequisite missing") + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", ResolveSym: resolve, Sigs: sigs} + proved := normalizeX86RawReturnSourceStack(decoded, options, true, file.Funcs[0]) + if row.half { + if proved.x86SourceStack != nil { + t.Fatal("half-vector move inherited an unproved caller byte effect") + } + return + } + if proved.x86SourceStack == nil || !x86CallerNormalizedSourceMatches(proved, sig) { + t.Fatalf("existing typed move lacks whole-source caller producer hook: %s", proved.x86SourceStackFailure) + } + for _, ins := range proved.Instrs { + if !row.aligned && ins.Op == Op(row.name) && (!x86CallerMemoryCertified(ins, 0) || ins.x86CallerMemory[0].width != row.width) { + t.Fatal("actual instruction did not retain exact numerical-source read width") + } + } + }) + } +} + +func TestX86GoCallerLegacyMoveDoesNotBorrowUnknownBytesOrRegions(t *testing.T) { + for _, row := range x86CallerLegacyMoveDirectory { + if row.half { + continue + } + for _, text := range []string{row.name + " X0,X1", row.name + " (BX),X1", row.name + " X0,(DI)", row.name + " X0,1(DX)"} { + t.Run(text, func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, text) + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{ + {kind: x86CallerSliceInput, width: 4}, {kind: x86CallerFreshResult, width: row.width}, + }} + state := x86SourceStackState{ + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}, DX: {argument: 1}}, + vectors: map[Reg]uint64{"Z0": x86CallerVectorByteMask(row.width) &^ 1}, + known: map[Reg]uint8{}, masks: map[Reg]bool{}, + } + valid := x86CallerVectorMoveStep(&ins, &state, proof) + if text == row.name+" X0,X1" { + // A register copy may propagate unknown bytes. It may never + // turn those bytes into a numerically initialized output. + if valid && state.vectors["Z1"]&x86CallerVectorByteMask(row.width) == x86CallerVectorByteMask(row.width) { + t.Fatal("register transfer fabricated a missing source byte") + } + ins = x86CallerLegacyMoveInstruction(t, row.name+" X1,(DX)") + valid = x86CallerVectorMoveStep(&ins, &state, proof) + } + if valid { + t.Fatal("move acquired unknown numerical byte/address or exceeded the fresh result envelope") + } + if !reflect.DeepEqual(state.callers, map[Reg]x86CallerPointerValue{DI: {argument: 0}, DX: {argument: 1}}) { + t.Fatal("failed numerical transfer changed pointer provenance") + } + }) + } + } +} diff --git a/amd64_go_caller_logical.go b/amd64_go_caller_logical.go new file mode 100644 index 000000000..1ea3a8ccc --- /dev/null +++ b/amd64_go_caller_logical.go @@ -0,0 +1,31 @@ +package plan9asm + +// Packed bitwise operations do not write flags or GP/stack state. Their byte +// availability is independent of the shared operand grammar. MMX has no +// reviewed private caller move bank and must not be aliased to virtual Z0. +func x86CallerPackedLogicalStep(ins *Instr, form x86PackedLogicalForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := uint64(0) + if form.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } else { + first = state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86PackedLogicalDefinedBytes(*ins, form, first, second, old) + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + // Every VEX/EVEX width clears all higher physical register bits. + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_logical_runtime_test.go b/amd64_go_caller_logical_runtime_test.go new file mode 100644 index 000000000..16eefe33c --- /dev/null +++ b/amd64_go_caller_logical_runtime_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86GoLogicalAndNotOperandOrder(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires Linux/amd64 runtime counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools missing") + } + var declarations, assembly, tests, oracle strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + tests.WriteString(`package caller +import "testing" +func TestOriginalLogicalMathematics(t *testing.T) { + values:=[]uint64{0,1,7,0x0ff00ff0a55aa55a,0x33cc55aa5aa5f00f,1<<63,^uint64(0)} +`) + oracle.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, op := range []string{"ANDNPS", "ANDNPD", "PANDN"} { + name := strings.ToLower(op) + fmt.Fprintf(&declarations, ` +//go:noescape +func %[1]s(a,b,res,length unsafe.Pointer) +func Call%[1]s(x,y []uint64) uint64 { + var res uint64 + length:=len(x) + %[1]s(unsafe.Pointer(unsafe.SliceData(x)),unsafe.Pointer(unsafe.SliceData(y)),unsafe.Pointer(&res),unsafe.Pointer(&length)) + return res +} +`, name) + body := "MOVSD (DI),X0\nMOVSD (SI),X1\n" + op + " X0,X1\nMOVSD X1,(DX)" + machine := strings.Replace(x86CallerFixtureAssembly, "·kernel", "·"+name, 1) + machine = strings.Replace(machine, "MOVL AX,(DX)", body, 1) + assembly.WriteString(machine) + fmt.Fprintf(&tests, ` + for _,a:=range values { for _,b:=range values { + x,y:=[]uint64{a,17},[]uint64{b,23} + got:=Call%[1]s(x[:1],y[:1]); want:=a &^ b + if got!=want || x[0]!=a || x[1]!=17 || y[0]!=b || y[1]!=23 { t.Fatalf("%[1]s(%%x,%%x)=%%x want %%x",a,b,got,want) } + } } +`, name) + fmt.Fprintf(&oracle, "extern void %s(const void*,const void*,void*,const void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + } + tests.WriteString("}\n") + oracle.WriteString(`int main(void) { + const uint64_t values[]={0,1,7,UINT64_C(0x0ff00ff0a55aa55a),UINT64_C(0x33cc55aa5aa5f00f),UINT64_C(1)<<63,UINT64_MAX}; +`) + for _, name := range []string{"andnps", "andnpd", "pandn"} { + fmt.Fprintf(&oracle, ` + for(int i=0;i<7;++i) for(int j=0;j<7;++j) { + uint8_t a[18],b[18],out[18],before[18]; uint64_t length=1,got; + memset(a,0xa7,18); memset(b,0xb9,18); memset(out,0xc3,18); memcpy(before,out,18); + memcpy(a+1,values+i,8); memcpy(b+3,values+j,8); + %s(a+1,b+3,out+5,&length); + uint64_t want=values[i]&~values[j]; memcpy(&got,out+5,8); memcpy(before+5,&want,8); + if(got!=want || memcmp(before,out,18) || length!=1 || a[0]!=0xa7 || a[9]!=0xa7 || b[2]!=0xb9 || b[11]!=0xb9) return 1; + } +`, name) + } + oracle.WriteString("return 0;\n}\n") + dir := t.TempDir() + for name, src := range map[string]string{"go.mod": "module caller\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(src), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original Go ANDNOT operand order: %v\n%s", err, output) + } + pkg := x86CallerRuntimePackage(t, declarations.String(), assembly.String()) + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(llvmContext, pkg, []byte(assembly.String()), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "logical_andnot_order", triple, translation.Module.String(), oracle.String(), prefix) +} diff --git a/amd64_go_caller_logical_test.go b/amd64_go_caller_logical_test.go new file mode 100644 index 000000000..5d2e0f76a --- /dev/null +++ b/amd64_go_caller_logical_test.go @@ -0,0 +1,217 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerLogicalOriginalZeroLiterals(t *testing.T) { + for _, literal := range []struct{ name, word string }{ + {"VXORPS", "0xc057f8c5"}, {"VPXOR", "0xc0eff9c5"}, + } { + t.Run(literal.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "LONG $"+literal.word+"\nVMOVSS X0,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerLogicalCompleteFamilyInitializedForms(t *testing.T) { + for _, op := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[op] + for _, row := range x86LogicalAcceptedForms(op, "amd64") { + if strings.Contains(row, "M7") { + continue + } // No private MMX bank. + t.Run(row, func(t *testing.T) { + row = strings.NewReplacer("(AX)", "(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(row) + instruction := strings.Split(row, " ")[1] + operands := strings.Split(instruction, ",") + destination := operands[len(operands)-1] + index := destination[1:] + init := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z%s\nKMOVQ AX,K1\nKMOVQ AX,K7\n", index) + // Legacy destructive destinations must be initialized just as + // masked merge destinations; register spelling grants nothing. + if spec.encoding <= x86PackedLogicalLegacyInteger && destination == "X2" { + init += "VMOVUPS (SI),Z2\n" + } + body := init + row + "\nVMOVSS X" + index + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerLogicalRetainsUnknownSourceAndEnvelopeFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VANDPS X0,X1,X2\nVMOVSS X2,(DX)", + "unknown destructive destination": "VMOVUPS (DI),X0\nANDPS X0,X1\nVMOVSS X1,(DX)", + "masked unknown merge": "KMOVQ AX,K1\nVXORPS X0,X0,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VXORPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "mask from pointer": "KMOVQ DI,K1\nVXORPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "private stack": "VXORPS 0(SP),X0,X0\nVMOVSS X0,(DX)", + "unknown address": "VXORPS (BX),X0,X0\nVMOVSS X0,(DX)", + "FP pointer bits": "VXORPS a+0(FP),X0,X0\nVMOVSS X0,(DX)", + "length full vector": "VMOVUPS (DI),X0\nVPAND (CX),X0,X0\nVMOVSS X0,(DX)", + "oversized broadcast": "VMOVUPS (DI),X0\nVANDPD.BCST (DX),X0,X0\nVMOVSS X0,(DX)", + "source data write": "VXORPS X0,X0,X0\nVMOVSS X0,(DI)", + "wide result": "VXORPS Y0,Y0,Y0\nVMOVUPS Y0,(DX)", + "CFG undefined join": "CMPQ AX,$0\nJE joined\nVXORPS X0,X0,X0\njoined:\nVORPS X0,X0,X1\nVMOVSS X1,(DX)", + "legacy undefined other input": "XORPS X0,X0\nVORPS Y0,Y1,Y2\nVMOVSS X2,(DX)", + "MMX zero cannot borrow Z0": "PXOR M0,M0\nVMOVSS X0,(DX)", + "MMX memory bank": "PAND (DI),M0\nMOVQ M0,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved logical value/region passed: %v", err) + } + }) + } +} + +func TestX86GoCallerLogicalByteAvailabilityMatchesPhysicalEncoding(t *testing.T) { + for _, op := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[op] + for _, width := range []int{16, 32, 64} { + if spec.encoding <= x86PackedLogicalLegacyInteger && width != 16 || spec.encoding == x86PackedLogicalVEX && width == 64 { + continue + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, maskMode := range []string{"none", "merge", "zero"} { + if maskMode != "none" && spec.broadcastBytes == 0 { + continue + } + for _, zeroIdiom := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%d/%s/zero=%v", op, width, maskMode, zeroIdiom), func(t *testing.T) { + row := string(op) + " " + bank + "0," + bank + "1" + if zeroIdiom { + row = string(op) + " " + bank + "0," + bank + "0" + } + if spec.encoding > x86PackedLogicalLegacyInteger { + if maskMode != "none" { + row += ",K1" + } + row += "," + bank + "2" + if maskMode == "zero" { + row = strings.Replace(row, string(op), string(op)+".Z", 1) + } + } + file, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedLogicalProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0x0f, "Z1": 0x07, "Z2": 0x05}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedLogicalStep(&ins, form, &state, nil) { + t.Fatal("register effect rejected") + } + known := uint64(0x07) + if zeroIdiom { + known = 0x0f + if strings.Contains(string(op), "XOR") || strings.Contains(string(op), "ANDN") { + known = x86CallerVectorByteMask(int64(width)) + } + } + dest := "Z2" + if form.legacy { + dest = "Z1" + if zeroIdiom { + dest = "Z0" + } + } else { + if maskMode == "merge" { + known &= 0x05 + } + known |= ^x86CallerVectorByteMask(int64(width)) + } + if state.vectors[Reg(dest)] != known { + t.Fatalf("defined bytes %#x want %#x", state.vectors[Reg(dest)], known) + } + }) + } + } + } + } +} + +func TestX86GoCallerLogicalReadCertificatesRetainExactSource(t *testing.T) { + for _, row := range []string{"VPXOR (DI),X0,X1", "VANDPD.BCST (DI),Z0,K1,Z1"} { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (SI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\n"+row+"\nVMOVSS X1,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, instruction := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedLogicalProperties(instruction, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + index := form.memory + access, exists := instruction.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(instruction, index) { + t.Fatal("missing exact read certificate") + } + found = true + for _, change := range []string{"opcode", "suffix", "width", "mask", "address", "source", "direction", "address-size"} { + mutated := instruction + mutated.Args = append([]Operand(nil), instruction.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VANDPS" + case "suffix": + mutated.Op = Op(string(mutated.Op) + ".Z") + case "width": + access.width++ + mutated.x86CallerMemory[index] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + case "address": + mutated.Args[index].Mem.Off++ + case "source": + mutated.Args[1].Reg = "X4" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "address-size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s retained stale read certificate: %s", row, change) + } + } + } + if !found { + t.Fatal("logical source never acquired its independent actual read certificate") + } + } +} diff --git a/amd64_go_caller_low_multiply_test.go b/amd64_go_caller_low_multiply_test.go new file mode 100644 index 000000000..3324f1303 --- /dev/null +++ b/amd64_go_caller_low_multiply_test.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerLowMultiplyUnknownInputsRemainContext(t *testing.T) { + for _, name := range []string{"VPMULLD", "VPMULLQ"} { + for reason, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Z1\n%s Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Z0\n%s Z0,Z1,Z2\nVMOVSS X2,(DX)", + "partial source": "MOVSS (DI),X0\nVMOVUPS (SI),Z1\n%s Z0,Z1,Z2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\n%s.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nMOVL $1,AX\nKMOVQ AX,K1\n%s Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\n%s.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask lacks region": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n%s.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\n%s (BX),Z0,Z2\nVMOVSS X2,(DX)", + "input write": "VMOVUPS (DI),Z0\n%s Z0,Z0,Z2\nVMOVSS X2,(DI)", + "join missing second": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\n%s Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name+"/"+reason, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", fmt.Sprintf(body, name), 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("low product acquired unproved input/caller memory", err) + } + }) + } + } +} + +// Keep this independent of the external package which exposed the missing +// source-stack effect. The original Go caller owns every input/result region. +func TestX86GoCallerLowMultiplyCompleteForms(t *testing.T) { + for _, name := range []string{"PMULLW", "VPMULLW", "PMULLD", "VPMULLD", "VPMULLQ"} { + banks := []string{"X"} + vector := strings.HasPrefix(name, "V") + if vector { + banks = append(banks, "Y", "Z") + } + for _, bank := range banks { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && (!memory || !vector || strings.HasSuffix(name, "W")) { + continue + } + for mode := 0; mode < 3; mode++ { + if !vector && mode != 0 { + continue + } + first, opcode := bank+"0", name + if memory { + first = "(DI)" + } + if broadcast { + opcode += ".BCST" + } + args, result := first+","+bank+"1", "X1" + if vector { + if mode != 0 { + args += ",K1" + } + if mode == 2 { + opcode += ".Z" + } + args += "," + bank + "2" + result = "X2" + } + row := opcode + " " + args + t.Run(row, func(t *testing.T) { + body := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n%s\nVMOVSS %s,(DX)", row, result) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } + } + } + } +} diff --git a/amd64_go_caller_maddword.go b/amd64_go_caller_maddword.go new file mode 100644 index 000000000..05cbbb55b --- /dev/null +++ b/amd64_go_caller_maddword.go @@ -0,0 +1,38 @@ +package plan9asm + +func x86CallerWordMultiplyAddStep(ins *Instr, form x86PackedMultiplyAddForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // E4NF.nb retains the complete source read even for a zero mask. The + // native legacy bridge consumes this same actual pointer and opcode; + // the caller witness grants a read envelope, not alignment/no-fault. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + groupBytes := form.spec.outputBits / 8 + for byteIndex := 0; byteIndex < form.byteWidth; byteIndex += groupBytes { + pair := (uint64(1)<= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_maddword_runtime_test.go b/amd64_go_caller_maddword_runtime_test.go new file mode 100644 index 000000000..05e7908aa --- /dev/null +++ b/amd64_go_caller_maddword_runtime_test.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddActualTypedCaller(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + checks.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for index, bank := range []string{"legacy", "X", "Y", "Z"} { + name := fmt.Sprintf("madd%d", index) + goSource := strings.ReplaceAll(x86CallerFixtureSource, "kernel", name) + goSource = strings.TrimPrefix(goSource, "package caller\nimport \"unsafe\"\n") + goSource = strings.Replace(goSource, "func Call(", "func Call"+name+"(", 1) + goSource = strings.Replace(goSource, ") float32 {", ") uint32 {", 1) + goSource = strings.Replace(goSource, "var res float32", "var res uint32", 1) + declarations.WriteString(goSource) + body := "MOVSS (DI),X0\nMOVSS (SI),X1\nPMADDWL X0,X1\nMOVSS X1,(DX)" + if bank != "legacy" { + body = "VMOVUPS (DI)," + bank + "0\nVMOVUPS (SI)," + bank + "1\nVPMADDWD " + bank + "0," + bank + "1," + bank + "2\nVMOVSS X2,(DX)" + } + machine := strings.Replace(x86CallerFixtureAssembly, "·kernel", "·"+name, 1) + machine = strings.Replace(machine, "MOVL AX,(DX)", body, 1) + assembly.WriteString(machine) + fmt.Fprintf(&checks, "extern void %s(const void*,const void*,void*,const void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + } + goTests := `package caller +import("testing";"unsafe") +func TestOriginalSSEPair(t *testing.T) { + values:=[]uint32{0,1,0x80008000,0x7fff7fff,0xffff0001,0x1234abcd,0xfffe8000} + signed:=func(v uint32) int64 {x:=int64(v&65535);if x>=32768{x-=65536};return x} + for _,a:=range values {for _,b:=range values { + var x,y [16]float32 + *(*uint32)(unsafe.Pointer(&x[0]))=a;*(*uint32)(unsafe.Pointer(&y[0]))=b + got:=Callmadd0(x[:],y[:]);want:=uint32(signed(a)*signed(b)+signed(a>>16)*signed(b>>16)) + if got!=want || *(*uint32)(unsafe.Pointer(&x[0]))!=a || *(*uint32)(unsafe.Pointer(&y[0]))!=b {t.Fatalf("pair %x/%x got%x want%x",a,b,got,want)} + }} +} +` + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module caller\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": goTests} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original actual typed SSE caller: %v\n%s", err, output) + } + checks.WriteString(` +static int64_t signedWord(uint32_t bits) {uint32_t v=bits&65535;return v<32768 ? (int64_t)v : (int64_t)v-65536;} +int main(void) { + const uint32_t values[]={0,1,0x80008000,0x7fff7fff,0xffff0001,0x1234abcd,0xfffe8000}; +`) + for index := 0; index < 4; index++ { + fmt.Fprintf(&checks, ` + for(int i=0;i<7;++i) for(int j=0;j<7;++j) { + uint8_t a[70],b[70],out[12],before[12];uint64_t length=16;uint32_t got; + memset(a,0xa7,sizeof(a));memset(b,0xb9,sizeof(b));memset(out,0xc3,sizeof(out));memcpy(before,out,sizeof(out)); + memcpy(a+1,values+i,4);memcpy(b+3,values+j,4); + madd%d(a+1,b+3,out+5,&length); + uint32_t want=(uint32_t)(signedWord(values[i])*signedWord(values[j])+signedWord(values[i]>>16)*signedWord(values[j]>>16)); + memcpy(&got,out+5,4);memcpy(before+5,&want,4); + if(got!=want || memcmp(before,out,sizeof(out)) || length!=16 || a[0]!=0xa7 || a[65]!=0xa7 || b[2]!=0xb9 || b[67]!=0xb9) return 1; + } +`, index) + } + checks.WriteString("return 0;\n}\n") + pkg := x86CallerRuntimePackage(t, declarations.String(), assembly.String()) + llvmContext := llvm.NewContext() + defer llvmContext.Dispose() + translation, err := translateGoModuleInContext(llvmContext, pkg, []byte(assembly.String()), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + if translation != nil && !translation.Module.IsNil() { + translation.Module.Dispose() + } + t.Fatal(err) + } + defer translation.Module.Dispose() + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := translation.Module.String() + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd-actual-caller", triple, text, checks.String(), prefix) + }) + } +} diff --git a/amd64_go_caller_maddword_test.go b/amd64_go_caller_maddword_test.go new file mode 100644 index 000000000..29ae6ed7c --- /dev/null +++ b/amd64_go_caller_maddword_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86GoCallerWordMultiplyAddOriginalRawEffect(t *testing.T) { + // This is the unchanged raw instruction from Weaviate's dot_byte_256. + // The independent fixture supplies actual typed, complete selected callers; + // it does not borrow the external function's unclosed pointer loop. + body := "VMOVDQU (DI),Y2\nVMOVDQU (SI),Y4\nLONG $0xd4f5edc5\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerWordMultiplyAddOriginalFollowingAddEffect(t *testing.T) { + // Consecutive unchanged LONGs from dot_byte_avx256.s. Each arithmetic + // family now has its separate complete typed caller effect. + body := "VMOVDQU (DI),Y2\nVMOVDQU (SI),Y4\nVPXOR Y0,Y0,Y0\nLONG $0xd4f5edc5\nLONG $0xc0feedc5\nVMOVSS X0,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerWordMultiplyAddCompleteActualSourceConsumers(t *testing.T) { + rows := []string{"PMADDWL X0,X1", "PMADDWL X1,X1", "PMADDWL (DI),X1"} + for _, bank := range []string{"X", "Y", "Z"} { + for _, first := range []string{bank + "0", "(DI)"} { + rows = append(rows, + "VPMADDWD "+first+","+bank+"1,"+bank+"2", + "VPMADDWD "+first+","+bank+"1,K1,"+bank+"2", + "VPMADDWD.Z "+first+","+bank+"1,K1,"+bank+"2") + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + result := "X2" + if strings.HasPrefix(row, "PMADDWL") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatalf("actual typed caller %s: %v", triple, err) + } + } + }) + } +} + +func TestX86GoCallerWordMultiplyAddRetainsUnknownSourceFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPMADDWD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPMADDWD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown upper pairs": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPMADDWD Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPMADDWD.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVPMADDWD Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPMADDWD.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full read": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMADDWD.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMADDWD (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private stack": "VMOVUPS (DI),Z0\nVPMADDWD (SP),Z0,Z2\nVMOVSS X2,(DX)", + "legacy SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMADDWL (BX),X0\nVMOVSS X0,(DX)", + "legacy BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPMADDWL (BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPMADDWD Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX separate bank": "PMADDWL M0,M1\nVMOVSS X1,(DX)", + "undefined branch": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPMADDWD Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved packed multiply-add became caller permission: %v", err) + } + }) + } +} + +func TestX86GoCallerWordMultiplyAddMemoryCertificatesRetainExactFullRead(t *testing.T) { + for _, row := range []string{"PMADDWL (DI),X0", "VPMADDWD.Z (DI),Z0,K1,Z2", "PMADDUBSW (DI),X0", "VPMADDUBSW.Z (DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X2,(DX)" + if strings.HasPrefix(row, "PMADD") { + body = strings.Replace(body, "VMOVSS X2", "VMOVSS X0", 1) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedMultiplyAddProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, index) { + t.Fatal("original legacy or zero-mask E4NF lost exact native full-read witness") + } + for _, change := range []string{"opcode", "width", "address", "destination", "direction", "mask", "address size"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VPMADDUBSW" + if strings.Contains(row, "UBSW") { + mutated.Op = "VPMADDWD" + } + case "width": + access.width-- + mutated.x86CallerMemory[index] = access + case "address": + mutated.Args[index].Mem.Off++ + case "destination": + mutated.Args[form.destination].Reg = "X7" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K7" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K1"}) + } + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s stale %s native read permission survived", row, change) + } + } + } + if !found { + t.Fatal("normalized source omitted actual multiply-add") + } + } +} diff --git a/amd64_go_caller_mask_vector_test.go b/amd64_go_caller_mask_vector_test.go new file mode 100644 index 000000000..fa52d3c44 --- /dev/null +++ b/amd64_go_caller_mask_vector_test.go @@ -0,0 +1,392 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// The denominator comes from every original Go encoder row, not the caller +// registry or the library which first exposed its missing effect registration. +func x86OfficialMaskVectorRows(t *testing.T) []string { + t.Helper() + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + family := map[string]bool{ + "VPMOVM2B": true, "VPMOVM2W": true, "VPMOVM2D": true, "VPMOVM2Q": true, + "VPMOVB2M": true, "VPMOVW2M": true, "VPMOVD2M": true, "VPMOVQ2M": true, + } + var rows []string + counts := map[string]int{} + for _, rule := range rules { + if !family[rule.Opcode] { + continue + } + toVector := strings.HasPrefix(rule.Opcode, "VPMOVM2") + vectorIndex, maskIndex := 0, 1 + if toVector { + vectorIndex, maskIndex = 1, 0 + } + if len(rule.Classes) != 2 || rule.Classes[maskIndex] != "Yk" || + rule.Prefix != "Pavx" || rule.ZCase != "Zevex_rm_v_r" || + rule.ZOffset != 3 || rule.EncodingOffset != rule.RowIndex*3 || len(rule.OpBytes) != 9 { + t.Fatalf("unmodeled official mask/vector row: %+v", rule) + } + bank := map[string]string{"YxrEvex": "X", "YyrEvex": "Y", "Yzr": "Z"}[rule.Classes[vectorIndex]] + bits := map[string]string{"X": "128", "Y": "256", "Z": "512"}[bank] + w := "evexW0" + if strings.HasSuffix(rule.Opcode, "W") || strings.HasSuffix(rule.Opcode, "Q") || + strings.Contains(rule.Opcode, "W2M") || strings.Contains(rule.Opcode, "Q2M") { + w = "evexW1" + } + terms := strings.Split(rule.OpBytes[rule.EncodingOffset], " | ") + opcode := "0x28" + if strings.Contains(rule.Opcode, "D") || strings.Contains(rule.Opcode, "Q") { + opcode = "0x38" + } + if !toVector { + opcode = map[string]string{"0x28": "0x29", "0x38": "0x39"}[opcode] + } + if bank == "" || !reflect.DeepEqual(terms, []string{"avxEscape", "evex" + bits, "evexF3", "evex0F38", w}) || + rule.OpBytes[rule.EncodingOffset+1] != "0" || !strings.EqualFold(rule.OpBytes[rule.EncodingOffset+2], opcode) { + t.Fatalf("unmodeled encoding generation/attributes: %+v", rule) + } + operands := bank + "1,K1" + if toVector { + operands = "K1," + bank + "1" + } + rows = append(rows, rule.Opcode+" "+operands) + counts[rule.Opcode]++ + } + for opcode := range family { + if counts[opcode] != 3 { + t.Fatalf("%s complete X/Y/Z denominator changed: %d", opcode, counts[opcode]) + } + } + if len(rows) != 24 { + t.Fatalf("complete mask/vector Go-row denominator changed: %d", len(rows)) + } + return rows +} + +func TestX86GoCallerMaskVectorCompleteOfficialEffectRegistration(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !x86CallerVectorMoveKnown(file.Funcs[0].Instrs[1]) { + t.Fatal("Go-accepted complete mask/vector row lacks its caller effect") + } + }) + } +} + +func TestX86GoCallerMaskVectorActualTypedSourceAndObjects(t *testing.T) { + for index, row := range x86OfficialMaskVectorRows(t) { + t.Run(fmt.Sprintf("%02d/%s", index, row), func(t *testing.T) { + body := "KMOVQ AX,K1\nVMOVUPS (DI),Z1\n" + row + "\n" + if strings.HasPrefix(row, "VPMOVM2") { + body += "VMOVSS X1,(DX)" + } else { + body += "KMOVQ K1,AX\nMOVL AX,(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "mask-vector-text.ll", "mask-vector-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "mask-vector-module.ll", "mask-vector-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("actual selected typed caller/LLVM object failed", err) + } + } + }) + } +} + +func TestX86MaskVectorGrammarKeepsModeAndDispatchBoundaries(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + for _, bank := range []string{"X", "Y", "Z"} { + if !strings.Contains(row, bank+"1") { + continue + } + for _, index := range []int{0, 7, 8, 31} { + spelling := strings.Replace(row, bank+"1", fmt.Sprint(bank, index), 1) + t.Run(spelling, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+spelling+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + for _, arch := range []string{"amd64", "386"} { + _, known, err := x86MaskVectorProperties(ins, arch) + if !known { + t.Fatal("recognized family fell through") + } + if arch == "386" && index >= 8 { + if bank == "Z" { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+spelling+"\nRET\n", "", false) + if err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go-invalid mode32 Z bank became a physical binding") + } + } else { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+spelling+"\nRET\n", "", true) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved physical mode32 X/Y binding was accepted", err) + } + } + } else if err != nil { + t.Fatal("valid physical bank rejected", err) + } + } + }) + } + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nVPMOVM2B K1,X1\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ins.Op = "vpmovm2b" + var output strings.Builder + ctx := amd64Ctx{b: &output, goarch: "amd64"} + if known, _, err := ctx.lowerMaskVectorMove("VPMOVM2B", ins); !known || err != nil { + t.Fatal("equivalent case-normalized opcode lost its existing dispatch", err) + } + output.Reset() + if known, _, err := ctx.lowerMaskVectorMove("VPMOVM2W", ins); !known || err == nil || output.Len() != 0 { + t.Fatal("contradictory dispatch emitted a different opcode", err) + } +} + +func TestX86GoCallerMaskVectorInitializationAndUnaffectedState(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + t.Run(row, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86MaskVectorProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatal("shared grammar lost official row", err) + } + initial := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, known: map[Reg]uint8{AX: 0xff}, + vectors: map[Reg]uint64{"Z1": ^uint64(0), "Z31": 0x55}, + masks: map[Reg]bool{"K1": true, "K0": false, "K7": true}, + } + state, want := x86CloneCallerState(initial), x86CloneCallerState(initial) + if form.spec.maskToVector { + want.vectors["Z1"] = ^uint64(0) + } else { + want.masks["K1"] = true + } + if !x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, want) { + t.Fatalf("mask/vector effect changed unrelated state: got=%+v want=%+v", state, want) + } + for missing := 0; missing < form.byteWidth; missing++ { + state = x86CloneCallerState(initial) + if form.spec.maskToVector { + state.masks["K1"] = false + } else { + state.vectors["Z1"] &^= uint64(1) << uint(missing) + } + before := x86CloneCallerState(state) + if x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(before, state) { + t.Fatalf("missing input %d acquired effect or mutated state", missing) + } + } + if form.spec.maskToVector { + state = x86CloneCallerState(initial) + state.vectors["Z1"] = 0 + if !x86CallerVectorMoveStep(&ins, &state, nil) || state.vectors["Z1"] != ^uint64(0) { + t.Fatal("EVEX overwrite did not define low data and clear wider aliases") + } + } else { + state = x86CloneCallerState(initial) + state.vectors["Z1"] = x86CallerVectorByteMask(int64(form.byteWidth)) + state.masks["K1"] = false + if !x86CallerVectorMoveStep(&ins, &state, nil) || !state.masks["K1"] { + t.Fatal("complete selected vector requires no unrelated upper aliases or old destination mask") + } + } + }) + } +} + +func TestX86GoCallerMaskVectorRejectsUninitializedActualSources(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + t.Run(row, func(t *testing.T) { + var body string + if strings.HasPrefix(row, "VPMOVM2") { + body = row + "\nVMOVSS X1,(DX)" + } else { + body = row + "\nKMOVQ K1,AX\nMOVL AX,(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown entry mask/vector acquired raw-return permission: %v", err) + } + }) + } +} + +func TestX86GoCallerMaskVectorK0IsInitializedDataNotWritemasking(t *testing.T) { + for _, row := range x86OfficialMaskVectorRows(t) { + if !strings.HasPrefix(row, "VPMOVM2") { + continue + } + row = strings.ReplaceAll(row, "K1", "K0") + for _, initialized := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/initialized=%t", row, initialized), func(t *testing.T) { + body := row + "\nVMOVSS X1,(DX)" + if initialized { + body = "KMOVQ AX,K0\n" + body + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if initialized && err != nil || !initialized && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("K0 acquired implicit all-one/zero value or lost defined data: %v", err) + } + }) + } + } +} + +func TestX86MaskVectorGrammarRejectsEverySiblingInvalidForm(t *testing.T) { + for _, opcode := range []string{ + "VPMOVM2B", "VPMOVM2W", "VPMOVM2D", "VPMOVM2Q", + "VPMOVB2M", "VPMOVW2M", "VPMOVD2M", "VPMOVQ2M", + } { + toVector := strings.HasPrefix(opcode, "VPMOVM2") + args := []string{"K1,Z1", "K1,K2", "K1,0(AX)", "X1,Z1", "K8,Z1", "K1,Z32", "K1,K2,Z1", "K1"} + if !toVector { + args = []string{"Z1,K1", "K1,K2", "0(AX),K1", "Z1,X1", "Z1,K8", "Z32,K1", "Z1,K2,K1", "Z1"} + } + for index, operands := range args { + spelling := opcode + " " + operands + if index == 0 { + spelling = opcode + ".Z " + operands + } + t.Run(spelling, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT f(SB),4,$0-0\n" + spelling + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue // Source parser rejection grants no supported form. + } + ins := file.Funcs[0].Instrs[1] + if _, known, err := x86MaskVectorProperties(ins, arch); !known || err == nil { + t.Fatal("invalid sibling form escaped the shared grammar", err) + } + if x86CallerVectorMoveKnown(ins) { + t.Fatal("Go-rejected sibling acquired a caller effect") + } + } + }) + } + } +} + +func TestX86MaskVectorOfficialRowsThreeAPIsFiveTargets(t *testing.T) { + rows := x86OfficialMaskVectorRows(t) + var assembly, declarations strings.Builder + declarations.WriteString("package maskvector\n") + for index, row := range rows { + fmt.Fprintf(&assembly, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", index, row) + fmt.Fprintf(&declarations, "func F%d()\n", index) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + goAssemblerOracleOutput(t, target.arch, assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "example.invalid/maskvector", declarations.String()) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mask-vector-family-text.ll", "mask-vector-family-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mask-vector-family-module.ll", "mask-vector-family-module.o", module.String()) + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly.String()), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "mask-vector-family-go.ll", "mask-vector-family-go.o", translated.Module.String()) + t.Log("24 independently derived Go rows; three APIs and LLVM22 objects, not native entry/runtime permission") + }) + } +} diff --git a/amd64_go_caller_mul_div_test.go b/amd64_go_caller_mul_div_test.go new file mode 100644 index 000000000..d55b9fcf0 --- /dev/null +++ b/amd64_go_caller_mul_div_test.go @@ -0,0 +1,69 @@ +package plan9asm + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func TestX86GoCallerFloatMulDivWholeFamilyByteEffects(t *testing.T) { + for _, row := range x86FloatMulDivFormats("amd64") { + if strings.Contains(row, "(AX)") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86FloatMulDivProperties(ins, "amd64") + if !recognized || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("missing registered family effect %s: %v", row, err) + } + for _, missing := range []int{-1, form.first, form.second, form.destination, 4} { + state := x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}, + masks: map[Reg]bool{"K1": true}, + } + if missing >= 0 && missing < len(ins.Args) && ins.Args[missing].Kind == OpReg { + state.vectors[x86CallerVectorRoot(ins.Args[missing].Reg)] &^= 1 + } + if missing == 4 { + state.masks["K1"] = false + } + before, masksBefore := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerVectorMoveStep(&ins, &state, nil) + want := missing == -1 || missing == form.destination && !form.legacy && (form.mask < 0 || form.zeroing) || missing == 4 && form.mask < 0 + if accepted != want || !reflect.DeepEqual(state.masks, masksBefore) || !accepted && !reflect.DeepEqual(state.vectors, before) { + t.Fatalf("%s missing %d: %t want %t, invented numerical/mask evidence", row, missing, accepted, want) + } + } + } +} + +func TestX86GoCallerFloatMulDivCannotInventValuesOrPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VMULPS X0,X1,X2\nVMOVSS X2,(DX)", + "division no identity": "VDIVPS X0,X0,X2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ AX,K1\nVMULPS X0,X1,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVDIVPS.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),X0\nKMOVQ DI,K1\nVMULPS.Z X0,X0,K1,X2\nVMOVSS X2,(DX)", + "physical stack": "VMOVUPS (DI),X0\nVMULPS (SP),X0,X2\nVMOVSS X2,(DX)", + "FP pointer": "VMOVUPS (DI),X0\nVDIVPS a+0(FP),X0,X2\nVMOVSS X2,(DX)", + "unknown memory": "VMOVUPS (DI),X0\nVMULPS (R11),X0,X2\nVMOVSS X2,(DX)", + "g read": "VMOVUPS (DI),X0\nVDIVPS (R14),X0,X2\nVMOVSS X2,(DX)", + "wide result": "VMOVUPS (DI),X0\nVMULPS X0,X0,X2\nVMOVUPS X2,(DX)", + "input write": "VMOVUPS (DI),X0\nVDIVPS X0,X0,X2\nVMOVSS X2,(DI)", + "unknown CFG join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVMULPS X0,X0,X2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved MUL/DIV source passed: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_packed_add.go b/amd64_go_caller_packed_add.go new file mode 100644 index 000000000..c2d68c8cd --- /dev/null +++ b/amd64_go_caller_packed_add.go @@ -0,0 +1,44 @@ +package plan9asm + +func x86CallerPackedWrappingAddStep(ins *Instr, form x86PackedAddForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + return x86CallerPackedLaneArithmeticStep(ins, form.x86PackedBinaryForm, form.spec.laneBits, state, proof) +} + +// Each complete input lane determines one output lane for wrapping or +// saturating integer add/subtract or low-product multiplication. Form sharing +// grants no address permission. +func x86CallerPackedLaneArithmeticStep(ins *Instr, form x86PackedBinaryForm, bits int, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // Source permission still requires the full original possible access + // envelope; an initialized K mask does not prove a smaller region. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + step := bits / 8 + for index := 0; index < form.byteWidth; index += step { + lane := x86CallerVectorByteMask(int64(step)) << uint(index) + if first&lane == lane && second&lane == lane { + known |= lane + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_packed_add_runtime_test.go b/amd64_go_caller_packed_add_runtime_test.go new file mode 100644 index 000000000..4f8b41678 --- /dev/null +++ b/amd64_go_caller_packed_add_runtime_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddActualTypedCaller(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var goSource, assembly, declarations, checks strings.Builder + goSource.WriteString("package caller\nimport \"unsafe\"\n") + declarations.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + count := 0 + for opcodeIndex, opcode := range x86WrappingAddOpcodes { + banks := []string{"X"} + if strings.HasPrefix(opcode, "V") { + banks = []string{"X", "Y", "Z"} + } + for _, bank := range banks { + name := fmt.Sprintf("calleradd%d", count) + count++ + declaration := x86CallerFixtureSource[strings.Index(x86CallerFixtureSource, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+name+"(", 1) + declaration = strings.Replace(declaration, ") float32 {", ") uint32 {", 1) + declaration = strings.Replace(declaration, "var res float32", "var res uint32", 1) + goSource.WriteString(declaration) + body := "MOVSS (DI),X0\nMOVSS (SI),X1\n" + opcode + " X0,X1\nMOVSS X1,(DX)" + if strings.HasPrefix(opcode, "V") { + body = "VMOVUPS (DI)," + bank + "0\nVMOVUPS (SI)," + bank + "1\n" + opcode + " " + bank + "0," + bank + "1," + bank + "2\nVMOVSS X2,(DX)" + } + machine := strings.Replace(x86CallerFixtureAssembly, "·kernel", "·"+name, 1) + assembly.WriteString(strings.Replace(machine, "MOVL AX,(DX)", body, 1)) + fmt.Fprintf(&declarations, "extern void %s(const void*,const void*,void*,const void*) __asm__(LABEL(\"caller.%s\"));\n", name, name) + fmt.Fprintf(&checks, ` + for(int round=0;round<32;++round) { + uint8_t a[70],b[70],out[12],before[12];uint64_t length=16; + memset(a,0xa7,70);memset(b,0xb9,70);memset(out,0xc3,12);memcpy(before,out,12); + for(int i=0;i<64;++i){a[i+1]=(uint8_t)(i*17+round*31);b[i+3]=(uint8_t)(i*23+round*41);} + %[1]s(a+1,b+3,out+5,&length); + for(int i=0;i<4;i+=%[2]d) { + uint64_t sum=0,av=0,bv=0;int bytes=%[2]d; + if(bytes>4)bytes=4; + for(int j=0;j>(8*j)); + } + if(memcmp(out,before,12)||length!=16||a[0]!=0xa7||a[65]!=0xa7||b[2]!=0xb9||b[67]!=0xb9)return 1; + } +`, name, 1<= 0 { + root := x86CallerVectorRoot(instruction.Args[operand].Reg) + vectors[root] &^= uint64(1) << uint(missing) + } + state := &x86SourceStackState{vectors: vectors, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedWrappingAddStep(&instruction, form, state, nil) { + t.Fatal("register grammar rejected") + } + want := ^uint64(0) + if missing >= 0 { + want &^= x86CallerVectorByteMask(int64(step)) << uint(missing/step*step) + } + got := state.vectors[x86CallerVectorRoot(instruction.Args[form.destination].Reg)] + // Legacy preserves its old upper view, while each vector form + // defines it to zero. Both start fully known in this axis. + if got != want || got&^width != want&^width { + t.Fatalf("%s/%s missing%d input%d known=%x want=%x", opcode, mode, missing, operand, got, want) + } + } + } + } + } +} + +func TestX86GoCallerPackedAddUnknownBoundariesRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPADDD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPADDQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown high byte": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPADDQ Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPADDB.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVPADDW Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPADDQ.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask no permission": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPADDD.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPADDB (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private stack": "VMOVUPS (DI),Z0\nVPADDW (SP),Z0,Z2\nVMOVSS X2,(DX)", + "legacy SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPADDL (BX),X0\nVMOVSS X0,(DX)", + "input store": "VMOVUPS (DI),Z0\nVPADDQ Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX": "PADDB M0,M1\nVMOVSS X1,(DX)", + "saturating separate": "VMOVUPS (DI),Z0\nVPADDSW Z0,Z0,Z2\nVMOVSS X2,(DX)", + "unknown joined bank": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPADDW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "different pointer offset": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nVPADDD (DI),Z0,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s escaped actual source boundary: %v", name, err) + } + }) + } + // Independent caller effect shape does not allow another opcode or a + // narrowed masked source to reuse a private instruction certificate. + for _, row := range []string{"PADDL (DI),X0", "VPADDQ.BCST.Z (DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X0,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86PackedWrappingAddProperties(ins, "amd64") + if !known || err != nil { + continue + } + access := ins.x86CallerMemory[form.memory] + if access.width != int64(form.memoryBytes) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatalf("missing full possible source extent for %s", row) + } + for _, mutation := range []string{"opcode", "width", "direction", "address"} { + changedAccess := access + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch mutation { + case "opcode": + changed.Op = "VPSUBQ" + case "width": + changedAccess.width-- + changed.x86CallerMemory[form.memory] = changedAccess + case "direction": + changedAccess.write = true + changed.x86CallerMemory[form.memory] = changedAccess + case "address": + changed.Args[form.memory].Mem.Off++ + } + if x86CallerMemoryCertified(changed, form.memory) { + t.Fatal(fmt.Sprintf("%s stale %s witness", row, mutation)) + } + } + } + } +} + +func TestX86PackedAddLegacyVirtualMemoryCannotForgeNativeAlignment(t *testing.T) { + for _, row := range []string{"PADDB source+0(FP),X0", "PADDW (SP),X0", "PADDL (BP),X0", "PADDQ source+0(FP),X0"} { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sig := x86WordMaddFPType(t) + _, err = Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": sig}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual storage acquired physical aligned m128 permission: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_packed_moves_test.go b/amd64_go_caller_packed_moves_test.go new file mode 100644 index 000000000..7345feefe --- /dev/null +++ b/amd64_go_caller_packed_moves_test.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "sort" + "strings" + "testing" +) + +func TestX86GoCallerPackedMovesPreserveNumericalInputBytes(t *testing.T) { + for _, form := range []struct{ name, body string }{ + {"VEX input", "VMOVDQU (DI),Y0\nVMOVSS X0,(DX)"}, + {"EVEX input", "VMOVDQU8 (DI),Z31\nVMOVSS X31,(DX)"}, + {"EVEX zero mask", "KMOVQ AX,K1\nVMOVUPS.Z (DI),K1,Z0\nVMOVSS X0,(DX)"}, + } { + t.Run(form.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", form.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerPackedMovesCompleteFamilyRegisterCopies(t *testing.T) { + var opcodes []string + for opcode := range amd64PackedVectorMoveSpecs { + opcodes = append(opcodes, string(opcode)) + } + sort.Strings(opcodes) + for _, opcode := range opcodes { + spec := amd64PackedVectorMoveSpecs[Op(opcode)] + for _, bank := range []string{"X", "Y", "Z"} { + if bank == "Z" && spec.encoding == amd64PackedMoveVEX { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if mode != "unmasked" && spec.encoding == amd64PackedMoveVEX { + continue + } + body := fmt.Sprintf("VMOVDQU8 (DI),%s0\n", bank) + name, mask := opcode, "" + if mode != "unmasked" { + body += fmt.Sprintf("VMOVDQU8 (SI),%s1\nKMOVQ AX,K1\n", bank) + mask = "K1," + if mode == "zero" { + name += ".Z" + } + } + body += fmt.Sprintf("%s %s0,%s%s1\nVMOVSS X1,(DX)", name, bank, mask, bank) + t.Run(opcode+"/"+bank+"/"+mode, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } + } +} + +func TestX86GoCallerPackedMovesRetainUnknownAndBoundedMemoryFailures(t *testing.T) { + for _, example := range []struct{ name, body string }{ + {"unknown source", "VMOVUPS X0,X1\nVMOVSS X1,(DX)"}, + {"unknown mask", "VMOVUPS.Z (DI),K1,Z0\nVMOVSS X0,(DX)"}, + {"pointer mask", "KMOVQ DI,K1\nVMOVUPS.Z (DI),K1,Z0\nVMOVSS X0,(DX)"}, + {"unknown merge destination", "KMOVQ AX,K1\nVMOVUPS (DI),K1,Z0\nVMOVSS X0,(DX)"}, + {"unknown source in zero mask", "KMOVQ AX,K1\nVMOVUPS.Z Z0,K1,Z1\nVMOVSS X1,(DX)"}, + {"unknown base", "VMOVUPS (BX),Y0\nVMOVSS X0,(DX)"}, + {"pointer argument bits", "VMOVUPS a+0(FP),X0\nVMOVSS X0,(DX)"}, + {"saved BP bytes", "VMOVUPS 0(SP),X0\nVMOVSS X0,(DX)"}, + {"unknown input alignment", "VMOVAPS (DI),X0\nVMOVSS X0,(DX)"}, + {"unknown integer alignment", "VMOVDQA32 (DI),Y0\nVMOVSS X0,(DX)"}, + {"input write", "VMOVUPS (DI),Y0\nVMOVUPS Y0,(SI)\nVMOVSS X0,(DX)"}, + {"length read too wide", "VMOVUPS (CX),X0\nVMOVSS X0,(DX)"}, + {"length write", "VMOVUPS (DI),X0\nVMOVUPS X0,(CX)\nVMOVSS X0,(DX)"}, + {"wide fresh result", "VMOVUPS (DI),X0\nVMOVUPS X0,(DX)"}, + {"masked wide fresh result", "KMOVQ AX,K1\nVMOVUPS (DI),X0\nVMOVUPS X0,K1,(DX)"}, + {"invalid zero memory store", "KMOVQ AX,K1\nVMOVUPS (DI),X0\nVMOVUPS.Z X0,K1,(DX)"}, + {"indexed fresh result", "VMOVUPS (DI),X0\nVMOVUPS X0,(DX)(AX*1)"}, + {"SP escape", "MOVQ SP,BX\nVMOVUPS (BX),X0\nVMOVSS X0,(DX)"}, + {"g carrier", "VMOVUPS (R14),X0\nVMOVSS X0,(DX)"}, + {"different initialized bytes at join", "CMPQ AX,$0\nJE join\nVMOVUPS (DI),Z0\njoin:\nVMOVSS X0,(DX)"}, + } { + t.Run(example.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", example.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved packed move acquired continuation: %v", err) + } + }) + } +} + +func TestX86PackedMoveInvalidStoreCannotAcquireCallerMemoryCertificate(t *testing.T) { + for _, opcode := range x86PackedZeroStoreOpcodes { + for _, bank := range []string{"X", "Y", "Z"} { + instruction := fmt.Sprintf("%s.Z %s0,K1,(DX)", opcode, bank) + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := parsed.Funcs[0].Instrs[1] + form, recognized, err := amd64PackedVectorMoveProperties(ins, "amd64") + if !recognized || err != nil || !form.undefinedOpcode() { + t.Fatalf("missing syntax/exception distinction: %s: %v", instruction, err) + } + if x86CallerVectorMoveKnown(ins) { + t.Fatalf("#UD instruction acquired an ordinary move effect: %s", instruction) + } + if _, _, valid := x86CallerMoveMemoryShape(ins, 2); valid { + t.Fatalf("#UD instruction acquired a memory certificate shape: %s", instruction) + } + state := x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": ^uint64(0)}, masks: map[Reg]bool{"K1": true}, + } + if x86CallerPackedVectorMoveStep(&ins, form, &state, nil) { + t.Fatalf("#UD instruction acquired caller continuation: %s", instruction) + } + } + } +} + +func TestX86GoCallerPackedMoveByteKnowledgeIncludesMaskAndZeroUpperViews(t *testing.T) { + for _, encoding := range []struct { + op string + width int + }{ + {"VMOVUPS", 16}, {"VMOVUPD", 32}, {"VMOVDQU8", 64}, + {"VMOVDQU16", 16}, {"VMOVDQU32", 32}, {"VMOVDQU64", 64}, + } { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[encoding.width] + for _, sourceKnown := range []uint64{0, 0xf, 0xff, ^uint64(0)} { + for _, oldKnown := range []uint64{0, 0xff, ^uint64(0)} { + for _, mode := range []string{"", "merge", "zero"} { + name, mask := encoding.op, "" + if mode != "" { + mask = "K1," + if mode == "zero" { + name += ".Z" + } + } + instruction := fmt.Sprintf("%s %s0,%s%s1", name, bank, mask, bank) + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := parsed.Funcs[0].Instrs[1] + form, recognized, err := amd64PackedVectorMoveProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("known transfer form: %v", err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": sourceKnown, "Z1": oldKnown}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedVectorMoveStep(&ins, form, &state, nil) { + t.Fatal("register-only effect unexpectedly requires a caller region") + } + var expected uint64 + for index := 0; index < 64; index++ { + bit := uint64(1) << uint(index) + if index >= encoding.width || sourceKnown&bit != 0 && (mode != "merge" || oldKnown&bit != 0) { + expected |= bit + } + } + if state.vectors["Z1"] != expected || state.vectors["Z0"] != sourceKnown || !reflect.DeepEqual(state.masks, map[Reg]bool{"K1": true}) { + t.Fatalf("%s changed initialized-byte/mask state: %+v; want %x", instruction, state, expected) + } + } + } + } + } +} diff --git a/amd64_go_caller_packed_narrow.go b/amd64_go_caller_packed_narrow.go new file mode 100644 index 000000000..e136e21e4 --- /dev/null +++ b/amd64_go_caller_packed_narrow.go @@ -0,0 +1,46 @@ +package plan9asm + +func x86CallerPackedNarrowStep(ins *Instr, form x86PackedNarrowForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // E4NF/.nb requires the complete tuple, even when every output mask + // bit is zero. The form supplies a width, never a pointer permission. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + inputBytes, outputBytes := form.spec.inputBits/8, form.spec.outputBits/8 + var known uint64 + for output := 0; output < form.byteWidth; output += outputBytes { + // PACK concatenates separately within each 128-bit chunk. Go's + // second operand supplies its low half; first supplies its high half. + chunk, local := output/16*16, output%16 + input, source := chunk+(local%8)/outputBytes*inputBytes, second + if local >= 8 { + source = first + } + needed := x86CallerVectorByteMask(int64(inputBytes)) << uint(input) + if source&needed == needed { + known |= x86CallerVectorByteMask(int64(outputBytes)) << uint(output) + } + } + if form.mask >= 0 && !form.zeroing { + // An initialized mask is not a known mask value: a byte must be + // initialized both when computed and when the old destination wins. + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_packed_narrow_alias_test.go b/amd64_go_caller_packed_narrow_alias_test.go new file mode 100644 index 000000000..180e03057 --- /dev/null +++ b/amd64_go_caller_packed_narrow_alias_test.go @@ -0,0 +1,84 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerPackedNarrowDestinationAliasesKeepOriginalInputs(t *testing.T) { + // Whole-source initialization isolates concatenation and alias ordering + // from the finer per-input-byte dependency test. Expected half masks are + // independent constants, repeated once for each 128-bit chunk. + for _, name := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for bankIndex, bank := range []string{"X", "Y", "Z"} { + for _, mask := range []string{"", "merge", "zero"} { + for destination := 0; destination < 3; destination++ { + opcode, operands := name, bank+"0,"+bank+"1" + if mask != "" { + operands += ",K1" + } + if mask == "zero" { + opcode += ".Z" + } + operands += fmt.Sprintf(",%s%d", bank, destination) + row := opcode + " " + operands + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + width := x86CallerVectorByteMask(int64(16 << bankIndex)) + for _, sample := range []struct{ first, second, halves uint64 }{ + {0, ^uint64(0), 0x00ff00ff00ff00ff}, + {^uint64(0), 0, 0xff00ff00ff00ff00}, + {^uint64(0), ^uint64(0), ^uint64(0)}, + {0, 0, 0}, + } { + ins := file.Funcs[0].Instrs[1] + state := &x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": sample.first, "Z1": sample.second, "Z2": 0xaaaaaaaaaaaaaaaa}, + masks: map[Reg]bool{"K1": true}, + } + root := Reg(fmt.Sprintf("Z%d", destination)) + old := state.vectors[root] + want := sample.halves & width + if mask == "merge" { + want &= old + } + want |= ^width + if !x86CallerVectorMoveStep(&ins, state, nil) || state.vectors[root] != want { + t.Fatalf("source/destination alias grant %s first=%x second=%x got=%x want=%x", + row, sample.first, sample.second, state.vectors[root], want) + } + } + }) + } + } + } + } +} + +func TestX86GoCallerPackedNarrowLegacyUpperBytesDoNotBecomeInitialized(t *testing.T) { + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + row := name + " X0,X1" + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": 0xffff}} + ins := file.Funcs[0].Instrs[1] + if !x86CallerVectorMoveStep(&ins, state, nil) || state.vectors["Z1"] != 0xffff { + t.Fatal("legacy PACK borrowed VEX upper-zero initialization", row, state.vectors["Z1"]) + } + // A following YMM use cannot inherit initialized upper bytes from + // the instruction's legal XMM-only grammar. + next := Instr{Op: "VPACKSSDW", Args: []Operand{ + {Kind: OpReg, Reg: "Y1"}, {Kind: OpReg, Reg: "Y1"}, {Kind: OpReg, Reg: "Y2"}, + }} + if !x86CallerVectorMoveStep(&next, state, nil) || state.vectors["Z2"]&0xffff0000 != 0 { + t.Fatal(strings.Join([]string{"legacy PACK initialized an unrelated YMM upper chunk", row}, ": ")) + } + } +} diff --git a/amd64_go_caller_packed_narrow_permissions_test.go b/amd64_go_caller_packed_narrow_permissions_test.go new file mode 100644 index 000000000..b665df192 --- /dev/null +++ b/amd64_go_caller_packed_narrow_permissions_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerPackedNarrowUnknownValuesAndRegionsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown second": "VMOVUPS (DI),Z0\nVPACKSSDW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown first high half": "VMOVUPS (SI),Z1\nVPACKSSDW X0,X1,X2\nVPSRLDQ $8,X2,X3\nVMOVSS X3,(DX)", + "partial second": "MOVSS (DI),X3\nMOVSS X3,X1\nVMOVUPS (SI),Z0\nVPACKSSDW X0,X1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPACKUSWB.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nMOVL $1,AX\nKMOVQ AX,K1\nVPACKSSWB Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPACKUSDW.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full tuple": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPACKSSDW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask broadcast span": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPACKUSDW.BCST.Z 1(DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPACKUSWB (BX),Z0,Z2\nVMOVSS X2,(DX)", + "stack alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPACKSSWB (BX),X0\nVMOVSS X0,(DX)", + "BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPACKUSWB (BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPACKSSDW Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX separate bank": "PACKSSLW M0,M1\nVMOVSS X1,(DX)", + "join missing second": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPACKSSDW Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved PACK acquired caller permission", err) + } + }) + } + // The first input contributes only the high half of each 128-bit chunk. + // Unknown unused bytes must not forbid this initialized low-half write. + body := "VMOVUPS (SI),Z1\nVPACKSSDW Z0,Z1,Z2\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("independent initialized low half was lost", err) + } +} + +func TestX86GoCallerPackedNarrowFullReadCertificateIdentity(t *testing.T) { + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB", + "VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + banks := []string{"X"} + if strings.HasPrefix(name, "V") { + banks = append(banks, "Y", "Z") + } + for index, bank := range banks { + broadcasts := []bool{false} + if name == "VPACKSSDW" || name == "VPACKUSDW" { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + row, result, width := name+" (DI),X0", "X0", int64(16<= 0 { + mutated.Args[form.mask].Reg = "K7" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K7"}) + } + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, form.memory) { + t.Fatal(fmt.Sprintf("stale complete instruction certificate survived %s", change)) + } + } + } + if !found { + t.Fatal("actual PACK memory proof absent") + } + }) + } + } + } +} diff --git a/amd64_go_caller_packed_narrow_test.go b/amd64_go_caller_packed_narrow_test.go new file mode 100644 index 000000000..e739ee395 --- /dev/null +++ b/amd64_go_caller_packed_narrow_test.go @@ -0,0 +1,156 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86NarrowCallerRows(t *testing.T) []string { + t.Helper() + rows := x86OfficialPackedNarrowRows(t, "amd64") + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + rows = append(rows, name+" X0,X1", name+" (DI),X1") + } + if len(rows) != 98 { + t.Fatalf("independent current Go narrow denominator changed: %d", len(rows)) + } + return rows +} + +func TestX86GoCallerPackedNarrowRecognizesEveryOfficialVectorForm(t *testing.T) { + for _, row := range x86NarrowCallerRows(t) { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !x86CallerVectorMoveKnown(file.Funcs[0].Instrs[1]) { + t.Fatal("accepted complete narrow grammar has no caller consumer") + } + }) + } +} + +func TestX86GoCallerPackedNarrowActualTypedCallerAndObjects(t *testing.T) { + for index, original := range x86NarrowCallerRows(t) { + t.Run(fmt.Sprintf("%d/%s", index, original), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+original+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + var args []string + for position, arg := range ins.Args { + if arg.Kind == OpReg && amd64VectorByteWidth(arg.Reg) != 0 { + // Rebind only the physical index; bank, memory, mask and + // opcode attributes remain the independent Go row shape. + arg.Reg = Reg(fmt.Sprintf("%c%d", arg.Reg[0], position)) + } + if arg.Kind == OpMem { + arg.Mem.Base, arg.Mem.Index, arg.Mem.Sym = DI, "", "" + arg.Mem.Off, arg.Mem.Scale = 0, 0 + } + args = append(args, arg.String()) + } + row := string(ins.Op) + " " + strings.Join(args, ",") + result := "X" + fmt.Sprint(len(ins.Args)-1) + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nVMOVUPS (SI),Z3\n" + + "MOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("actual typed caller/LLVM object consumer failed", err) + } + } + }) + } +} + +func TestX86GoCallerPackedNarrowIndependentPerOutputByteDependencies(t *testing.T) { + for _, name := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB", + "VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + legacy := !strings.HasPrefix(name, "V") + banks := []string{"X", "Y", "Z"} + if legacy { + banks = banks[:1] + } + inputBytes := 2 + if strings.HasSuffix(name, "DW") || name == "PACKSSLW" { + inputBytes = 4 + } + for bankIndex, bank := range banks { + for _, mask := range []string{"", "merge", "zero"} { + if legacy && mask != "" { + continue + } + opcode, args := name, bank+"0,"+bank+"1" + if !legacy { + if mask != "" { + args += ",K1" + } + if mask == "zero" { + opcode += ".Z" + } + args += "," + bank + "2" + } + row := opcode + " " + args + t.Run(row, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + width := 16 << bankIndex + widthMask := x86CallerVectorByteMask(int64(width)) + for _, first := range []uint64{0, 1, 3, 15, 0xffff, 0xff00ff00ff00ff00, ^uint64(0)} { + for _, second := range []uint64{0, 7, 0xffff, 0x5555555555555555, ^uint64(0)} { + old := uint64(0xaaaaaaaaaaaaaaaa) + destination := Reg("Z2") + if legacy { + destination, old = "Z1", second + } + state := &x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": first, "Z1": second, "Z2": old}, + masks: map[Reg]bool{"K1": true}, + } + // Each 128-bit chunk contains second's narrowed + // low half, then first's narrowed high half. One + // missing input byte revokes its whole output lane. + var want uint64 + for output := 0; output < width; output += inputBytes / 2 { + chunk, local := output/16*16, output%16 + input, source := chunk+(local%8)/(inputBytes/2)*inputBytes, second + if local >= 8 { + source = first + } + needed := uint64(1< +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +` + declarations.String() + ` +static int check(void (*fn)(void*,void*,void*,void*), unsigned lane, bool unmasked, bool zero) { + for (uint64_t length = 64; length < 68; length++) { + uint8_t input[512], old[512], output[48], before[48]; + for (unsigned i = 0; i < 512; i++) { + input[i] = (uint8_t)(i*7 + length*13); + old[i] = (uint8_t)(i*11 + length*17); + } + memset(output, 0xa5, sizeof(output)); + memcpy(before, output, sizeof(output)); + const uint64_t original_length = length; + fn(input, old, output+16, &length); + if (length != original_length) return 1; + for (unsigned i = 0; i < sizeof(output); i++) { + uint8_t want = before[i]; + if (i >= 16 && i < 20) { + const unsigned byte = i-16; + const bool active = unmasked || ((length >> (byte/lane)) & 1); + want = active ? input[byte] : zero ? 0 : old[byte]; + } + if (output[i] != want) return 2; + } + for (unsigned i = 0; i < 512; i++) { + if (input[i] != (uint8_t)(i*7 + length*13) || + old[i] != (uint8_t)(i*11 + length*17)) return 3; + } + } + return 0; +} + +int main(void) { +` + checks.String() + " return 0;\n}\n" + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-caller-numerical", triple, translation.Module.String(), oracle, runner) +} diff --git a/amd64_go_caller_packed_shift.go b/amd64_go_caller_packed_shift.go new file mode 100644 index 000000000..86bb33928 --- /dev/null +++ b/amd64_go_caller_packed_shift.go @@ -0,0 +1,48 @@ +package plan9asm + +// Availability is separate from encoding. No count, mask, or zero-valued +// shadow register is evidence that an incoming vector contains numeric data. +func x86CallerPackedShiftStep(ins *Instr, form x86PackedShiftForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + if form.memory >= 0 { + // Legacy XMM uniform count loads still need an independently bound + // native-address/fault contract. A slice identity alone does not + // establish it; the original opcode, not MOVDQA, governs faults. + if form.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + data := width + if form.data != form.memory { + data = state.vectors[x86CallerVectorRoot(ins.Args[form.data].Reg)] & width + } + if data != width { + return false + } + counts := ^uint64(0) + if ins.Args[form.count].Kind != OpImm && form.count != form.memory { + countWidth := int64(form.byteWidth) + if form.countMode == x86PackedShiftUniform { + countWidth = 8 // The native register's upper count qword is ignored. + } + needed := x86CallerVectorByteMask(countWidth) + counts = state.vectors[x86CallerVectorRoot(ins.Args[form.count].Reg)] + if counts&needed != needed { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + // The caller-entry contract deliberately retains its complete input + // requirements above. Sharing numerical dependencies does not relax them. + known := x86PackedShiftDefinedBytes(form, data, counts, old) + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_packed_shift_test.go b/amd64_go_caller_packed_shift_test.go new file mode 100644 index 000000000..73284a23e --- /dev/null +++ b/amd64_go_caller_packed_shift_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GoCallerPackedShiftWeaviateOriginalBytes(t *testing.T) { + // Original dot_byte_avx256.s uses c5 ed 71 d1 08 at its first + // VPSRLW. Keep the complete encoded instruction, not a named surrogate. + body := "VMOVDQU (DI),Y1\nLONG $0xd171edc5\nBYTE $0x08\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, bound := range []bool{true, false} { + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if !bound { + pkg.SourceSelection = nil + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + ir, err := Translate(file, options) + if bound { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "original.ll", "original.o", ir) + } else if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("uncontracted Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if bound && err != nil || !bound && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bound=%t module: %v", bound, err) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); bound && err != nil || !bound && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bound=%t GoModule: %v", bound, err) + } + } + } +} + +func TestX86GoCallerPackedShiftWholeFamilyEffects(t *testing.T) { + for _, op := range x86PackedShiftNames() { + for _, row := range x86PackedShiftRows(op, "amd64") { + if strings.Contains(row, "(AX)") || strings.Contains(row, "M0") || strings.Contains(row, "M1") { + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("missing effect %s: %v", row, err) + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + for _, missing := range []string{"none", "data", "count", "mask"} { + state := x86SourceStackState{ + vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": 0}, + masks: map[Reg]bool{"K1": true, "K7": true}, + } + applicable := true + switch missing { + case "data": + state.vectors[x86CallerVectorRoot(ins.Args[form.data].Reg)] &^= uint64(1) << uint(form.byteWidth-1) + case "count": + applicable = ins.Args[form.count].Kind != OpImm + if applicable { + state.vectors[x86CallerVectorRoot(ins.Args[form.count].Reg)] &^= 1 + } + case "mask": + applicable = form.mask >= 0 + if applicable { + delete(state.masks, ins.Args[form.mask].Reg) + } + } + if !applicable { + continue + } + before, _ := x86CloneCallerVectorState(state.vectors, state.masks) + accepted := x86CallerPackedShiftStep(&ins, form, &state, nil) + if accepted != (missing == "none") || !accepted && !reflect.DeepEqual(before, state.vectors) { + t.Fatalf("%s missing %s accepted=%t", row, missing, accepted) + } + if accepted { + want := ^uint64(0) + if form.mask >= 0 && !form.zeroing { + want = ^width + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatalf("%s known=%x want %x", row, got, want) + } + } + } + } + } +} + +func TestX86GoCallerPackedShiftMemoryShapesAndTargets(t *testing.T) { + for _, op := range x86PackedShiftNames() { + for _, row := range x86PackedShiftRows(op, "amd64") { + if strings.Contains(row, "M0") || strings.Contains(row, "M1") { + continue + } + row = strings.ReplaceAll(row, "(AX)", "(DI)") + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if err != nil { + t.Fatal(err) + } + if form.memory >= 0 { + width, write, valid := x86CallerMoveMemoryShape(ins, form.memory) + if !valid || write || int(width) != form.memoryBytes { + t.Fatalf("incorrect read envelope %s: %d/%t/%t", row, width, write, valid) + } + } + body := fmt.Sprintf("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVQ $-1,AX\nKMOVQ AX,K1\nKMOVQ AX,K7\n%s\nVMOVSS X2,(DX)", row) + if form.legacy { + body = strings.Replace(body, "VMOVSS X2,(DX)", "VMOVSS X1,(DX)", 1) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + // Compile one bounded corpus per target below; individual forms here + // also exercise the actual caller transport, not a public Ptr noalias. + err = x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if form.legacy && form.memory >= 0 { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved legacy native-memory contract passed %s: %v", row, err) + } + } else if err != nil { + t.Fatalf("%s: %v", row, err) + } + } + } +} + +func TestX86GoCallerPackedShiftUnprovedInputsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown data": "VPSRLW $8,Y1,Y2\nVMOVSS X2,(DX)", + "zero count unknown data": "VPSRLW $0,Y1,Y2\nVMOVSS X2,(DX)", + "saturated unknown data": "VPSRLW $255,Y1,Y2\nVMOVSS X2,(DX)", + "unknown count": "VMOVUPS (DI),Y1\nVPSRLW X0,Y1,Y2\nVMOVSS X2,(DX)", + "partial count": "MOVL $1,AX\nMOVD AX,X0\nVMOVUPS (DI),Y1\nVPSRLVW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),Y1\nVPSRLW.Z $8,Y1,K1,Y2\nVMOVSS X2,(DX)", + "merge unknown destination": "VMOVUPS (DI),Y1\nMOVL $1,AX\nKMOVW AX,K1\nVPSRLW $8,Y1,K1,Y2\nVMOVSS X2,(DX)", + "physical SP": "VMOVUPS (DI),Y1\nVPSRLW 0(SP),Y1,Y2\nVMOVSS X2,(DX)", + "FP pointer bytes": "VMOVUPS (DI),Y1\nVPSRLW a+0(FP),Y1,Y2\nVMOVSS X2,(DX)", + "g": "VMOVUPS (DI),Y1\nVPSRLW (R14),Y1,Y2\nVMOVSS X2,(DX)", + "unknown base": "VMOVUPS (DI),Y1\nVPSRLW (BX),Y1,Y2\nVMOVSS X2,(DX)", + "length m128": "VMOVUPS (DI),Y1\nVPSRLW (CX),Y1,Y2\nVMOVSS X2,(DX)", + "partial output": "VMOVUPS (DI),Y1\nVPSRLW $8,Y1,Y2\nVMOVUPS Y2,(DX)", + "input write": "VMOVUPS (DI),Y1\nVPSRLW $8,Y1,Y2\nVMOVSS X2,(DI)", + "branch join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),Y1\njoined:\nVPSRLW $8,Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source accepted: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_packed_subtract_test.go b/amd64_go_caller_packed_subtract_test.go new file mode 100644 index 000000000..52d4e43f4 --- /dev/null +++ b/amd64_go_caller_packed_subtract_test.go @@ -0,0 +1,261 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// These rows come from cmd/internal/obj/x86's yxm and _yvandnpd tables. +// Saturation changes arithmetic, not the operand availability obligation. +func x86PackedSubtractCallerRows() []string { + var rows []string + for _, opcode := range []string{ + "PSUBB", "PSUBW", "PSUBL", "PSUBQ", + "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", + } { + rows = append(rows, opcode+" X0,X15", opcode+" (DI),X15") + } + for _, opcode := range []string{ + "VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", + "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW", + } { + for _, bank := range []string{"X", "Y", "Z"} { + rows = append(rows, + fmt.Sprintf("%s %s0,%s1,%s2", opcode, bank, bank, bank), + fmt.Sprintf("%s (DI),%s1,%s2", opcode, bank, bank), + fmt.Sprintf("%s %s20,%s21,%s22", opcode, bank, bank, bank), + ) + for _, suffix := range []string{"", ".Z"} { + rows = append(rows, + fmt.Sprintf("%s%s %s20,%s21,K1,%s22", opcode, suffix, bank, bank, bank), + fmt.Sprintf("%s%s (DI),%s21,K1,%s22", opcode, suffix, bank, bank), + ) + } + if opcode == "VPSUBD" || opcode == "VPSUBQ" { + rows = append(rows, + fmt.Sprintf("%s.BCST (DI),%s21,%s22", opcode, bank, bank), + fmt.Sprintf("%s.BCST (DI),%s21,K1,%s22", opcode, bank, bank), + fmt.Sprintf("%s.BCST.Z (DI),%s21,K1,%s22", opcode, bank, bank), + ) + } + } + } + return rows +} + +func TestX86GoCallerPackedSubtractCompleteOriginalTypedConsumers(t *testing.T) { + rows := x86PackedSubtractCallerRows() + if len(rows) != 202 { + t.Fatalf("complete Go subtract surface changed: %d rows", len(rows)) + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + operands := strings.Split(strings.Fields(row)[1], ",") + destination := operands[len(operands)-1] + result := "X" + destination[1:] + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\n" + + "VMOVUPS (SI),Z15\nVMOVUPS (DI),Z20\nVMOVUPS (SI),Z21\n" + + "VMOVUPS (DI),Z22\nMOVL $1,AX\nKMOVQ AX,K1\n" + + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestX86PackedSubtractMaskedLoadsRetainActiveLaneEffects(t *testing.T) { + for _, opcode := range []string{ + "VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", + "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW", + } { + for _, bank := range []string{"X", "Y", "Z"} { + for _, suffix := range []string{".Z", ".BCST.Z"} { + if suffix == ".BCST.Z" && opcode != "VPSUBD" && opcode != "VPSUBQ" { + continue + } + row := fmt.Sprintf("%s%s (AX),%s1,K1,%s2", opcode, suffix, bank, bank) + t.Run(row, func(t *testing.T) { + assembly := "TEXT maskedsubtract(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"maskedsubtract": {Name: "maskedsubtract", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "load volatile") || !strings.Contains(ir, "br i1") { + t.Fatal("masked source needs conditional volatile active-lane reads, not an unconditional vector load") + } + }) + } + } + } +} + +func TestX86GoCallerPackedSubtractEveryLaneByteKnowledge(t *testing.T) { + for _, row := range x86PackedSubtractCallerRows() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSubtractProperties(instruction, "amd64") + if err != nil { + t.Fatal(err) + } + if form.memory >= 0 { + continue // Actual source extents are covered by the typed consumers. + } + for operand := 0; operand < 2; operand++ { + for missing := -1; missing < form.byteWidth; missing++ { + vectors := make(map[Reg]uint64) + for _, arg := range instruction.Args { + if amd64VectorByteWidth(arg.Reg) != 0 { + vectors[x86CallerVectorRoot(arg.Reg)] = ^uint64(0) + } + } + if missing >= 0 { + vectors[x86CallerVectorRoot(instruction.Args[operand].Reg)] &^= uint64(1) << uint(missing) + } + state := &x86SourceStackState{vectors: vectors, masks: map[Reg]bool{"K1": true}} + if !x86CallerVectorMoveStep(&instruction, state, nil) { + t.Fatalf("registered subtract shape rejected: %s", row) + } + want := ^uint64(0) + if missing >= 0 { + bytes := form.laneBits / 8 + want &^= x86CallerVectorByteMask(int64(bytes)) << uint(missing/bytes*bytes) + } + got := state.vectors[x86CallerVectorRoot(instruction.Args[form.destination].Reg)] + if got != want { + t.Fatalf("%s missing byte%d input%d: known=%x want=%x", row, missing, operand, got, want) + } + } + } + } +} + +func TestX86GoCallerPackedSubtractUnknownBoundariesRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPSUBD Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPSUBQ Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown high byte": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPSUBQ Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVPSUBSB.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nMOVL $1,AX\nKMOVQ AX,K1\nVPSUBSW Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPSUBUSB.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask no permission": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPSUBUSW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPSUBB (BX),Z0,Z2\nVMOVSS X2,(DX)", + "private stack": "VMOVUPS (DI),Z0\nVPSUBW (SP),Z0,Z2\nVMOVSS X2,(DX)", + "legacy SP alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPSUBL (BX),X0\nVMOVSS X0,(DX)", + "input store": "VMOVUPS (DI),Z0\nVPSUBQ Z0,Z0,Z2\nVMOVSS X2,(DI)", + "unknown joined bank": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPSUBW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "different pointer offset": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nADDQ $4,DI\njoined:\nVPSUBD (DI),Z0,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source boundary accepted: %v", err) + } + }) + } +} + +func TestX86PackedSubtractLegacyVirtualMemoryCannotForgeNativeAlignment(t *testing.T) { + for _, opcode := range []string{ + "PSUBB", "PSUBW", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW", + } { + for _, source := range []string{"source+0(FP)", "(SP)", "(BP)"} { + row := opcode + " " + source + ",X0" + t.Run(row, func(t *testing.T) { + assembly := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"f": x86WordMaddFPType(t)}, + }) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual storage acquired native alignment permission: %v", err) + } + }) + } + } +} + +func TestX86GoCallerPackedSubtractMemoryWitnessBindsExactInstruction(t *testing.T) { + for _, row := range []string{"PSUBL (DI),X0", "VPSUBQ.BCST.Z (DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X0,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs, + }) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86PackedSubtractProperties(ins, "amd64") + if !known || err != nil { + continue + } + found = true + access := ins.x86CallerMemory[form.memory] + if access.width != int64(form.memoryBytes) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatalf("missing complete possible source extent for %s", row) + } + for _, mutation := range []string{"opcode", "width", "direction", "address"} { + changedAccess := access + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + changed.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch mutation { + case "opcode": + changed.Op = "PSUBQ" + if form.broadcast { + changed.Op = "VPSUBQ.BCST" + } + case "width": + changedAccess.width-- + changed.x86CallerMemory[form.memory] = changedAccess + case "direction": + changedAccess.write = true + changed.x86CallerMemory[form.memory] = changedAccess + case "address": + changed.Args[form.memory].Mem.Off++ + } + if x86CallerMemoryCertified(changed, form.memory) { + t.Fatalf("%s reused a stale %s witness", row, mutation) + } + } + } + if !found { + t.Fatalf("source subtract disappeared from normalization: %s", row) + } + } +} diff --git a/amd64_go_caller_region_core.go b/amd64_go_caller_region_core.go new file mode 100644 index 000000000..3f83644f7 --- /dev/null +++ b/amd64_go_caller_region_core.go @@ -0,0 +1,400 @@ +package plan9asm + +import "reflect" + +// A pointer is never recovered from arbitrary integer data. The region and +// affine offset survive only exact full-register copies/address arithmetic. +type x86CallerPointerValue struct { + argument int + offset int64 + span int64 +} + +// Instruction-local access, not a function-wide memory exemption. The final +// guard compares the complete operands, effect direction and native width. +type x86CallerMemoryAccess struct { + op Op + args []Operand + width int64 + write bool +} + +func x86CallerSignatureMatches(sig FuncSig, proof *x86GoCallerRegionProof) bool { + if proof == nil || proof.calls == 0 || proof.symbol != sig.Name || len(proof.arguments) != len(sig.Args) { + return false + } + sig.x86CallerRegions = nil + return reflect.DeepEqual(sig, proof.signature) +} + +func x86CallerRegionSourceMatches(sig FuncSig, source Func, arch string) bool { + if arch != "amd64" || !x86CallerSignatureMatches(sig, sig.x86CallerRegions) { + return false + } + fingerprint, err := x86CallerSourceFingerprint(source) + if err != nil { + return false + } + if fingerprint == sig.x86CallerRegions.source { + return true + } + // Module/text lowering normalize more than once. Only the exact result of + // an earlier complete proof may be re-proved; edited public operands cannot + // inherit its private metadata or the original source fingerprint. + previous := source.x86SourceStack + return previous != nil && previous.caller == sig.x86CallerRegions && fingerprint == previous.normalized +} + +func x86CallerNormalizedSourceMatches(fn Func, sig FuncSig) bool { + proof := fn.x86SourceStack + if proof == nil || proof.caller == nil || proof.caller != sig.x86CallerRegions || !x86CallerSignatureMatches(sig, proof.caller) { + return false + } + fingerprint, err := x86CallerSourceFingerprint(fn) + return err == nil && fingerprint == proof.normalized +} + +func x86CallerGP(reg Reg) (Reg, bool) { + // This class includes R14, not numerical knowledge of incoming g. The + // complete source-stack proof separately requires its opaque identity at + // every return; only actual source definitions add numerical/pointer bits. + root, valid := amd64FullRegBase(reg) + if !valid { + return "", false + } + switch root { + case AX, BX, CX, DX, DI, SI, SP, BP, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return root, true + default: + return "", false + } +} + +func x86CallerScalarOperands(ins Instr) bool { + for _, operand := range ins.Args { + if operand.Kind == OpImm && (operand.ImmIsFloat || operand.ImmRaw != "") { + return false + } + if operand.Kind == OpReg { + if _, valid := x86CallerGP(operand.Reg); !valid { + return false + } + } + if operand.Kind == OpMem { + for _, reg := range []Reg{operand.Mem.Base, operand.Mem.Index} { + if reg != "" && reg != PC { + if _, valid := x86CallerGP(reg); !valid { + return false + } + } + } + } + } + return true +} + +func x86CloneCallerPointers(source map[Reg]x86CallerPointerValue) map[Reg]x86CallerPointerValue { + copy := make(map[Reg]x86CallerPointerValue, len(source)) + for reg, value := range source { + copy[reg] = value + } + return copy +} + +func x86CloneCallerKnown(source map[Reg]uint8) map[Reg]uint8 { + copy := make(map[Reg]uint8, len(source)) + for reg, value := range source { + copy[reg] = value + } + return copy +} + +func x86CallerByteMask(reg Reg, width int64) uint8 { + if width < 1 || width > 8 { + return 0 + } + shift := uint(0) + if _, bits, alias := amd64ByteAlias(reg); alias { + if width != 1 { + return 0 + } + shift = bits / 8 + } + return uint8((uint16(1)<= int64(len(proof.arguments)) || + ins.Args[1].Kind != OpReg { + return false + } + root, valid := x86CallerGP(ins.Args[1].Reg) + if !valid || root != ins.Args[1].Reg || root == SP || root == BP { + return false + } + x86CallerRecordMemory(ins, index, width, false) + case OpMem: + // The existing stack proof separately maps complete physical SP + // argument words to actual FP slots, excluding the native return PC. + if offset, mapped := ins.x86SourceStackFP[index]; mapped { + if index != 0 || len(ins.Args) != 2 || width != 8 || offset < 0 || offset%8 != 0 || + offset/8 >= int64(len(proof.arguments)) || ins.Args[1].Kind != OpReg { + return false + } + root, valid := x86CallerGP(ins.Args[1].Reg) + if !valid || root != ins.Args[1].Reg || root == SP || root == BP { + return false + } + x86CallerRecordMemory(ins, index, width, false) + continue + } + if len(ins.Args) != 2 || !x86CallerOrdinaryMemoryStep(ins, index, state, proof, width, index == 1) { + return false + } + } + } + return true +} + +func x86CallerOrdinaryMemoryStep(ins *Instr, index int, state *x86SourceStackState, proof *x86GoCallerRegionProof, width int64, write bool) bool { + if index < 0 || index >= len(ins.Args) || ins.Args[index].Kind != OpMem || width <= 0 { + return false + } + if _, parameter := ins.x86SourceStackFP[index]; parameter { + return false // Only the scalar GP import above can consume pointer slots. + } + memory := ins.Args[index].Mem + if memory.Sym != "" || memory.Segment != "" || ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit { + return false + } + base, valid := x86CallerGP(memory.Base) + pointer, known := state.callers[base] + if !valid || base != memory.Base || !known || pointer.argument < 0 || pointer.argument >= len(proof.arguments) { + return false + } + region := proof.arguments[pointer.argument] + if _, valid := x86CallerPointerEnd(pointer); !valid || pointer.span != 0 && region.kind != x86CallerSliceInput { + return false + } + if memory.Index != "" { + if write || region.kind != x86CallerSliceInput || + memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 || + !x86CallerNumericRead(Operand{Kind: OpReg, Reg: memory.Index}, 8, state) { + return false + } + } + offset := pointer.offset + memory.Off + if write { + if region.kind != x86CallerFreshResult || offset < 0 || offset > region.width || width > region.width-offset { + return false + } + } else if region.kind != x86CallerSliceInput { + if offset < 0 || offset > region.width || width > region.width-offset { + return false + } + } + x86CallerRecordMemory(ins, index, width, write) + return true +} + +// The existing complete source stack handles all SP/BP transformations and +// returns. This additional transfer tracks only actual caller pointer regions +// and numerical byte initialization; it grants no unknown instruction effects. +func x86CallerScalarStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (pointerFlags, valid bool) { + if accepted, recognized := x86CallerNumericalReadStep(*ins, state); recognized { + return false, accepted + } + if _, known, err := x86OrderingHintProperties(ins.Op, *ins, "amd64"); known { + return false, err == nil + } + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpTEXT || op == OpLABEL || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) { + return false, !isAMD64CounterZeroBranch(op) || x86CallerNumericRead(Operand{Kind: OpReg, Reg: CX}, 8, state) + } + if _, stack := amd64StackWidthSpecs[op]; stack { + return false, true // Exact saved values/flags are tracked by the stack step. + } + if recognized, valid := x86CallerExplicitIMULStep(ins, state, proof); recognized { + return false, valid + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + return false, true // Only the preceding exact address step admits this. + } + } + if recognized, valid := x86CallerExtensionStep(ins, state, proof); recognized { + return false, valid + } + if recognized, valid := x86CallerConditionalSetStep(ins, state, proof); recognized { + return false, valid + } + if recognized, valid := x86CallerConditionalMoveStep(ins, state, proof); recognized { + return false, valid + } + width := x86RawReturnScalarMoveBytes(op) + if width != 0 { + if len(ins.Args) != 2 || !x86CallerMemoryStep(ins, state, proof, width) { + return false, false + } + source, destination := ins.Args[0], ins.Args[1] + if isAMD64MemoryOperand(source) && isAMD64MemoryOperand(destination) { + return false, false + } + var pointer x86CallerPointerValue + hasPointer := false + if source.Kind == OpFP { + pointer = x86CallerPointerValue{argument: int(source.FPOffset / 8)} + hasPointer = true + } else if source.Kind == OpMem { + if offset, mapped := ins.x86SourceStackFP[0]; mapped { + pointer = x86CallerPointerValue{argument: int(offset / 8)} + hasPointer = true + } + } else if source.Kind == OpReg { + root, _ := x86CallerGP(source.Reg) + pointer, hasPointer = state.callers[root] + } + if hasPointer { + root, full := x86CallerGP(destination.Reg) + sourceRoot, _ := x86CallerGP(source.Reg) + if width != 8 || source.Kind == OpReg && source.Reg != sourceRoot || + destination.Kind != OpReg || !full || root != destination.Reg { + return false, false // Pointer-as-data stores and partial copies stay unknown. + } + state.callers[root] = pointer + delete(state.known, root) + return false, true + } + if source.Kind != OpMem && !x86CallerNumericRead(source, width, state) { + return false, false + } + if destination.Kind == OpReg { + return false, x86CallerNumericWrite(destination.Reg, width, state) + } + return false, destination.Kind == OpMem + } + if x86LEAWidth(ins.Op, "amd64") != 0 { + return false, x86CallerLEAStep(ins, state) + } + name := string(op) + if len(name) < 2 { + return false, false + } + base, suffix := name[:len(name)-1], name[len(name)-1:] + switch suffix { + case "B": + width = 1 + case "W": + width = 2 + case "L": + width = 4 + case "Q": + width = 8 + default: + return false, false + } + if len(ins.Args) == 2 && (op == "ADDQ" || op == "SUBQ") && ins.Args[0].Kind == OpImm && + ins.Args[0].ImmRaw == "" && !ins.Args[0].ImmIsFloat && ins.Args[1].Kind == OpReg { + root, _ := x86CallerGP(ins.Args[1].Reg) + if pointer, present := state.callers[root]; present { + if root != ins.Args[1].Reg { + return false, false + } + delta := int64(ins.Args[0].Imm) + if delta < -x86SourceStackLimit || delta > x86SourceStackLimit { + return false, false + } + if op == "SUBQ" { + delta = -delta + } + var valid bool + pointer, valid = x86CallerPointerShift(pointer, delta) + if !valid { + return false, false + } + state.callers[root] = pointer + return true, true // Private local addresses cannot define numerical flags. + } + } + zero := base == "XOR" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg && reflect.DeepEqual(ins.Args[0], ins.Args[1]) + for _, argument := range ins.Args { + if !zero && !x86CallerNumericRead(argument, width, state) { + return false, false + } + } + if base == "CMP" || base == "TEST" { + return false, true + } + if len(ins.Args) == 0 || ins.Args[len(ins.Args)-1].Kind != OpReg { + return false, false + } + return false, x86CallerNumericWrite(ins.Args[len(ins.Args)-1].Reg, width, state) +} + +func x86CallerMemoryCertified(ins Instr, index int) bool { + access, found := ins.x86CallerMemory[index] + width, write, valid := x86CallerMoveMemoryShape(ins, index) + // Producer memory proofs admit only native/default addressing. A later + // address-size override cannot reuse even otherwise identical operands. + return found && valid && ins.x86AddressBits == 0 && !ins.x86UnboundAddressOverride && access.op == ins.Op && reflect.DeepEqual(access.args, ins.Args) && + access.width == width && width > 0 && access.write == write +} diff --git a/amd64_go_caller_region_core_test.go b/amd64_go_caller_region_core_test.go new file mode 100644 index 000000000..05db35bc7 --- /dev/null +++ b/amd64_go_caller_region_core_test.go @@ -0,0 +1,286 @@ +package plan9asm + +import ( + "errors" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerCoreFixture(t *testing.T, source, assembly string) (GoPackage, *File) { + t.Helper() + pkg, selected, _, _ := x86CallerFixture(t, source) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + selected.Assembly["kernel_amd64.s"] = file + pkg.SourceSelection = &selected + return pkg, file +} + +func x86CallerCoreTranslate(t *testing.T, pkg GoPackage, assembly, triple string) error { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + return err + } + defer translated.Module.Dispose() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "caller-regions.ll", "caller-regions.o", translated.Module.String()) + return nil +} + +func TestX86GoCallerRegionCoreConsumesFreshScalarWrites(t *testing.T) { + for _, form := range []struct{ typ, move string }{ + {"uint8", "MOVB"}, {"uint16", "MOVW"}, {"uint32", "MOVL"}, {"uint64", "MOVQ"}, + } { + source := strings.Replace(x86CallerFixtureSource, "var res float32", "var res "+form.typ, 1) + source = strings.Replace(source, ") float32 {", ") "+form.typ+" {", 1) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", form.move+" $7,AX\n"+form.move+" AX,(DX)", 1) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(form.move+"/"+triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerRegionCoreDoesNotCertifyBadStackOrWrites(t *testing.T) { + for name, replacement := range map[string]string{ + "input write": "MOVL AX,(DI)", + "length write": "MOVL AX,(CX)", + "wide result": "MOVQ AX,(DX)", + "one byte overlap": "MOVL AX,1(DX)", + "negative offset": "MOVL AX,-1(DX)", + "unknown base": "MOVL AX,(BX)", + "indexed result": "MOVL AX,(DX)(BX*1)", + "pointer as data": "MOVL DI,(DX)", + "partial pointer": "MOVL DX,BX\nMOVL AX,(BX)", + "fresh pointer join": "CMPQ AX,$0\nJE joined\nMOVQ DI,DX\njoined:\nMOVL AX,(DX)", + "return PC": "MOVQ AX,8(BP)", + "saved BP": "MOVB AX,0(BP)", + "SP escape": "MOVQ SP,AX\nMOVL AX,(DX)", + "input pointer to SP": "MOVQ DI,SP\nMOVL AX,(DX)", + "input pointer to BP": "MOVQ DI,BP\nMOVL AX,(DX)", + "input address to SP": "LEAQ 0(DI),SP\nMOVL AX,(DX)", + "input address to BP": "LEAQ 0(DI),BP\nMOVL AX,(DX)", + "g use": "MOVQ R14,AX\nMOVL AX,(DX)", + "g overwrite": "MOVQ $0,R14\nMOVL AX,(DX)", + "pointer FP store": "MOVQ AX,res+16(FP)\nMOVL AX,(DX)", + "unknown numeric input": "MOVL BX,(DX)", + "partial address clobber": "MOVW $0,DX\nMOVL AX,(DX)", + "pointer flags": "ADDQ $0,DX\nJE joined\njoined:\nMOVL AX,(DX)", + "different pointer offsets": "CMPQ AX,$0\nJE joined\nADDQ $1,DX\njoined:\nMOVB AX,(DX)", + "unrestored SP": "MOVL AX,(DX)", + "unknown store effect": "XCHGL AX,(DX)", + "vector effect": "VMOVSS X0,(DX)", + "floating immediate": "MOVQ $1.25,AX\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", replacement, 1) + if name == "unrestored SP" { + assembly = strings.Replace(assembly, "BYTE $0xc3", "SUBQ $8,SP\nBYTE $0xc3", 1) + } + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bad source acquired caller-region continuation: %v", err) + } + }) + } + // An actual producer snapshot is mandatory; a pointer signature remains + // the original failure rather than acquiring source memory privileges. + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, x86CallerFixtureAssembly) + pkg.SourceSelection = nil + if err := x86CallerCoreTranslate(t, pkg, x86CallerFixtureAssembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("missing caller selection acquired pointer-store proof: %v", err) + } +} + +func TestX86GoCallerRegionCoreDoesNotStoreInputPointerAsFullWidthData(t *testing.T) { + source := strings.NewReplacer( + "var res float32", "var res uint64", ") float32 {", ") uint64 {", + ).Replace(x86CallerFixtureSource) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVQ DI,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("input pointer escaped through a correctly sized fresh result: %v", err) + } +} + +func TestX86GoCallerRegionCoreTracksScalarCopiesOffsetsAndInputs(t *testing.T) { + for name, replacement := range map[string]string{ + "input load": "MOVL (DI),AX\nADDL $7,AX\nMOVL AX,(DX)", + "length load": "MOVQ (CX),AX\nMOVL AX,(DX)", + "pointer copy": "MOVQ DX,BX\nMOVL AX,(BX)", + "pointer offset": "ADDQ $1,DX\nMOVB AX,(DX)", + "LEA offset": "LEAQ 3(DX),BX\nMOVB AX,(BX)", + "safe adjacent": "MOVB AX,3(DX)", + "byte alias": "MOVB $7,AH\nMOVB AH,(DX)", + "indexed input": "MOVQ $0,BX\nMOVL (DI)(BX*4),AX\nMOVL AX,(DX)", + "copy input offset": "MOVQ DI,BX\nADDQ $4,BX\nMOVL (BX),AX\nMOVL AX,(DX)", + "saved result pointer": "PUSHQ DX\nMOVQ DI,DX\nPOPQ DX\nMOVL AX,(DX)", + "identical join": "MOVQ DX,BX\nCMPQ AX,$0\nJE joined\nMOVQ DX,BX\njoined:\nMOVL AX,(BX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", replacement, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + physical := strings.NewReplacer( + "a+0(FP)", "8(SP)", "b+8(FP)", "16(SP)", + "res+16(FP)", "24(SP)", "length+24(FP)", "32(SP)", + ).Replace(x86CallerFixtureAssembly) + leaf := strings.NewReplacer( + "BYTE $0x55\nWORD $0x8948\nBYTE $0xe5\nLONG $0xf8e48348\n", "", + "WORD $0x8948\nBYTE $0xec\nBYTE $0x5d\n", "", + ).Replace(x86CallerFixtureAssembly) + for name, assembly := range map[string]string{"actual physical FP": physical, "stack-unobserving leaf": leaf} { + t.Run(name, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerRegionCoreRejectsDifferentCompilerDataLayout(t *testing.T) { + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, x86CallerFixtureAssembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(x86CallerFixtureAssembly), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sizes: types.SizesFor("gc", "386"), FrameSizes: types.SizesFor("gc", "amd64"), + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if translation != nil || !errors.Is(err, ErrProbeNeedsContext) { + if translation != nil { + translation.Module.Dispose() + } + t.Fatalf("typed Args/FP slots concealed a different local int/pointer layout: %v", err) + } +} + +func TestX86GoCallerRegionCoreRejectsStaleSourceSignatureAndAccess(t *testing.T) { + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + for _, change := range []string{"original source", "normalized source", "signature", "slot", "memory access", "missing caller"} { + t.Run(change, func(t *testing.T) { + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, x86CallerFixtureAssembly) + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + if change != "original source" { + file = normalized + } + sig := sigs["caller.kernel"] + switch change { + case "original source", "normalized source": + for index := range file.Funcs[0].Instrs { + instruction := &file.Funcs[0].Instrs[index] + if instruction.Op == "MOVL" && len(instruction.Args) == 2 && instruction.Args[1].Kind == OpMem { + instruction.Args[1].Mem.Off++ + } + } + case "signature": + sig.Args[0] = I64 + case "slot": + sig.Frame.Params[0].Offset = 8 + case "missing caller": + sig.x86CallerRegions = nil + case "memory access": + for index := range file.Funcs[0].Instrs { + instruction := &file.Funcs[0].Instrs[index] + for operand, access := range instruction.x86CallerMemory { + access.width++ + instruction.x86CallerMemory[operand] = access + } + } + if x86RawReturnMemoryBound(file.Funcs[0], file, options, true) { + t.Fatal("edited access inherited a whole-function memory permission") + } + // Final normalization deliberately recomputes a valid certificate + // from unchanged original operands, never trusts the edited one. + return + } + sigs["caller.kernel"] = sig + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed %s borrowed a caller continuation: %v", change, err) + } + }) + } +} + +func TestX86GoCallerRegionCoreCannotBorrowPhysicalFPWithoutAccessProof(t *testing.T) { + assembly := strings.NewReplacer( + "a+0(FP)", "8(SP)", "b+8(FP)", "16(SP)", + "res+16(FP)", "24(SP)", "length+24(FP)", "32(SP)", + ).Replace(x86CallerFixtureAssembly) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + removed := false + for index := range normalized.Funcs[0].Instrs { + instruction := &normalized.Funcs[0].Instrs[index] + for operand := range instruction.x86SourceStackFP { + delete(instruction.x86CallerMemory, operand) + removed = true + } + } + if !removed { + t.Fatal("fixture did not exercise a physical-SP imported parameter") + } + if x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("missing caller access borrowed the old numerical FP continuation") + } +} + +func TestX86GoCallerRegionCoreDoesNotExportScalarOrFloatingStackPermissions(t *testing.T) { + for name, assembly := range map[string]string{ + "external scalar write": strings.Replace(x86CallerFixtureAssembly, + "MOVL AX,(DX)", "MOVL AX,global(SB)\nMOVL AX,(DX)", 1) + "GLOBL global(SB),8,$4\n", + "floating stack adjustment": strings.Replace(x86CallerFixtureAssembly, + "BYTE $0xc3", "ADDQ $0.5,SP\nBYTE $0xc3", 1), + } { + t.Run(name, func(t *testing.T) { + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved permission acquired a caller continuation: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_region_diagnostic_test.go b/amd64_go_caller_region_diagnostic_test.go new file mode 100644 index 000000000..adc8b5dbc --- /dev/null +++ b/amd64_go_caller_region_diagnostic_test.go @@ -0,0 +1,37 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86GoCallerRegionRejectedEffectsRetainInstructionDiagnostic(t *testing.T) { + for _, example := range []struct { + name, replacement, diagnostic string + }{ + { + name: "uninitialized vector arithmetic", replacement: "VFMADD231SS (DI),X0,X1\nVMOVSS X1,(DX)", + diagnostic: "unproved caller-region effect: VFMADD231SS", + }, + { + name: "pointer as scalar data", replacement: "MOVL DI,(DX)", + diagnostic: "unproved caller-region effect: MOVL", + }, + { + name: "different input origins", + replacement: "CMPQ AX,$0\nJE joined\nMOVQ SI,DI\nCMPQ AX,$0\njoined:\nMOVL AX,(DX)", + diagnostic: "different source-stack or caller-region states at join", + }, + } { + t.Run(example.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", example.replacement, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), example.diagnostic) { + t.Fatalf("source rejection lost its actual instruction/state cause: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_region_runtime_test.go b/amd64_go_caller_region_runtime_test.go new file mode 100644 index 000000000..7f6bb287b --- /dev/null +++ b/amd64_go_caller_region_runtime_test.go @@ -0,0 +1,239 @@ +package plan9asm + +import ( + "context" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" +) + +type x86CallerRuntimeForm struct { + name string + bits int + move string + stack string +} + +func x86CallerRuntimeForms() []x86CallerRuntimeForm { + var forms []x86CallerRuntimeForm + for _, width := range []struct { + bits int + move string + }{{8, "MOVB"}, {16, "MOVW"}, {32, "MOVL"}, {64, "MOVQ"}} { + for _, stack := range []string{"Leaf", "Balanced", "SavedPointer"} { + forms = append(forms, x86CallerRuntimeForm{ + name: fmt.Sprintf("kernel%d%s", width.bits, stack), + bits: width.bits, move: width.move, stack: stack, + }) + } + } + return forms +} + +func x86CallerRuntimeSource() (goSource, assembly string) { + var declarations, machine strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&declarations, ` +//go:noescape +func %[1]s(a, b, res, length unsafe.Pointer) + +func Call%[1]s(x, y []uint64) uint%[2]d { + var res uint%[2]d + length := len(x) + %[1]s( + unsafe.Pointer(unsafe.SliceData(x)), + unsafe.Pointer(unsafe.SliceData(y)), + unsafe.Pointer(&res), + unsafe.Pointer(&length)) + return res +} +`, form.name, form.bits) + fmt.Fprintf(&machine, "TEXT ·%s(SB),4,$0-32\n", form.name) + machine.WriteString("MOVQ a+0(FP),DI\nMOVQ b+8(FP),SI\nMOVQ res+16(FP),DX\nMOVQ length+24(FP),CX\n") + if form.stack != "Leaf" { + machine.WriteString("BYTE $0x55\nWORD $0x8948\nBYTE $0xe5\nLONG $0xf8e48348\n") + } + if form.stack == "SavedPointer" { + machine.WriteString("PUSHQ DX\nMOVQ DI,DX\nPOPQ DX\n") + } + suffix := strings.TrimPrefix(form.move, "MOV") + fmt.Fprintf(&machine, "%s (DI),AX\n%s 8(SI),BX\nADD%s BX,AX\nMOVQ (CX),BX\nADD%s BX,AX\nADD%s $7,AX\n%s AX,(DX)\n", + form.move, form.move, suffix, suffix, suffix, form.move) + if form.stack != "Leaf" { + machine.WriteString("WORD $0x8948\nBYTE $0xec\nBYTE $0x5d\n") + } + machine.WriteString("BYTE $0xc3\n") + } + return declarations.String(), machine.String() +} + +func x86CallerRuntimePackage(t *testing.T, goSource, assembly string) GoPackage { + t.Helper() + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", goSource, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + info := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + } + typed, err := (&types.Config{ + Importer: x86CallerTestImporter{}, Sizes: types.SizesFor("gc", "amd64"), + }).Check("caller", fset, []*ast.File{syntax}, info) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + return GoPackage{ + Path: "caller", Types: typed, Syntax: []*ast.File{syntax}, + SourceSelection: &GoSourceSelection{ + Fset: fset, TypesInfo: info, Sizes: types.SizesFor("gc", "amd64"), + SelectedGoFiles: []string{"value.go"}, GoSources: map[string][]byte{"value.go": []byte(goSource)}, + SelectedAssemblyFiles: []string{"value_amd64.s"}, Assembly: map[string]*File{"value_amd64.s": file}, + }, + } +} + +func x86CallerActualGoOracle(t *testing.T, goSource, assembly string) { + t.Helper() + var tests strings.Builder + tests.WriteString("package caller\nimport \"testing\"\nfunc TestActualCallers(t *testing.T) {\n") + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&tests, ` + for iteration := 0; iteration < 64; iteration++ { + for _, input := range []uint64{0, 1, 127, 255, 65535, 1<<31, 1<<63, ^uint64(0)} { + x := []uint64{input, 0x1122334455667788} + y := []uint64{0x8877665544332211, input ^ 0x9876543210abcdef} + originalX := [2]uint64{x[0], x[1]} + originalY := [2]uint64{y[0], y[1]} + canary := [4]uint64{11, 23, 37, 41} + want := uint%[1]d(x[0]) + uint%[1]d(y[1]) + uint%[1]d(len(x)) + 7 + got := Call%[2]s(x, y) + if got != want || x[0] != originalX[0] || x[1] != originalX[1] || + y[0] != originalY[0] || y[1] != originalY[1] || canary != [4]uint64{11, 23, 37, 41} { + t.Fatalf("%[2]s got=%%#x want=%%#x x=%%#v y=%%#v canary=%%#v", got, want, x, y, canary) + } + } + } +`, form.bits, form.name) + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": goSource, + "value_amd64.s": assembly, "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + command.WaitDelay = 2 * time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go original caller-region oracle: %v\n%s", err, output) + } +} + +func x86CallerCOracle() string { + var source strings.Builder + source.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&source, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + } + source.WriteString("int main(void) {\n") + for _, form := range x86CallerRuntimeForms() { + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 64; iteration++) { + const uint64_t inputs[] = {0, 1, 127, 255, 65535, UINT64_C(1)<<31, UINT64_C(1)<<63, UINT64_MAX}; + for (unsigned index = 0; index < sizeof(inputs)/sizeof(inputs[0]); index++) { + uint64_t x[] = {inputs[index], UINT64_C(0x1122334455667788)}; + uint64_t y[] = {UINT64_C(0x8877665544332211), inputs[index] ^ UINT64_C(0x9876543210abcdef)}; + uint64_t original_x[2], original_y[2], length = 2; + unsigned char output[48], before[48]; + memcpy(original_x, x, sizeof(x)); + memcpy(original_y, y, sizeof(y)); + for (unsigned byte = 0; byte < sizeof(output); byte++) { + output[byte] = byte + iteration; + } + memset(output + 16, 0, %[1]d); + memcpy(before, output, sizeof(output)); + const uint64_t want = (x[0] + y[1] + length + 7) & UINT%[2]d_MAX; + %[3]s(x, y, output + 16, &length); + for (unsigned byte = 0; byte < sizeof(output); byte++) { + unsigned char expected = before[byte]; + if (byte >= 16 && byte < 16 + %[1]d) { + expected = want >> (8*(byte-16)); + } + if (output[byte] != expected) { + return 1; + } + } + if (memcmp(x, original_x, sizeof(x)) || memcmp(y, original_y, sizeof(y)) || length != 2) { + return 2; + } + } + } +`, form.bits/8, form.bits, form.name) + } + source.WriteString("\treturn 0;\n}\n") + return source.String() +} + +func TestCrossLinuxRuntimeMatrixX86GoCallerRegions(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("caller-region execution requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + goSource, assembly := x86CallerRuntimeSource() + x86CallerActualGoOracle(t, goSource, assembly) + pkg := x86CallerRuntimePackage(t, goSource, assembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "caller_regions", triple, + translation.Module.String(), x86CallerCOracle(), runner) +} diff --git a/amd64_go_caller_regions.go b/amd64_go_caller_regions.go new file mode 100644 index 000000000..991da4dbd --- /dev/null +++ b/amd64_go_caller_regions.go @@ -0,0 +1,519 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "go/types" + "reflect" + "strings" +) + +// GoSourceSelection carries a compiler's complete, actual selected package, +// not a user-supplied noalias assertion. The inventories must come from the +// same load/profile as Types, Syntax and TypesInfo. Every selected source is +// mandatory; a per-translation subset cannot establish a closed caller graph. +// Assembly contains the corresponding fully preprocessed source files. +type GoSourceSelection struct { + Fset *token.FileSet + TypesInfo *types.Info + Sizes types.Sizes + SelectedGoFiles []string + GoSources map[string][]byte + SelectedAssemblyFiles []string + Assembly map[string]*File +} + +type x86CallerRegionKind uint8 + +const ( + x86CallerSliceInput x86CallerRegionKind = iota + 1 + x86CallerFreshResult + x86CallerLocalLength +) + +type x86CallerArgumentRegion struct { + kind x86CallerRegionKind + width int64 + slot int64 +} + +// Only the actual Go producer can construct this private proof. It does not +// establish a source stack, an instruction effect or a return continuation. +// The subsequent whole-source CFG proof must consume the exact signature, +// parameter slots and output envelope before it can admit a raw return. +type x86GoCallerRegionProof struct { + symbol string + arguments []x86CallerArgumentRegion + calls int + goSources map[string][32]byte + assembly map[string][32]byte + signature FuncSig + source [32]byte +} + +func x86GoCallerRegionCandidate(fn Func, sig FuncSig) bool { + if sig.Ret != Void || len(sig.Args) == 0 { + return false + } + for _, argument := range sig.Args { + if argument != Ptr { + return false + } + } + decoded, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + return false // The normal decoder still reports the actual failure. + } + for _, ins := range decoded.Instrs { + if ins.x86RawNearReturn != nil { + return true + } + } + return false +} + +func deriveX86GoCallerRegions(pkg GoPackage, selected GoSourceSelection, fn Func, sig FuncSig, arch string) (*x86GoCallerRegionProof, error) { + fail := func(reason string) (*x86GoCallerRegionProof, error) { + return nil, fmt.Errorf("%w: bounded raw-return caller source: %s", ErrProbeNeedsContext, reason) + } + if arch != "amd64" || pkg.Types == nil || selected.Fset == nil || selected.TypesInfo == nil || !x86CallerTargetSizesValid(selected.Sizes) || + len(sig.ArgRegs) != 0 || sig.Ret != Void || len(sig.Frame.Results) != 0 || + sig.ARMEntry != nil || sig.ARM64GoRegisterABI != nil || sig.ARM64ClosureABI != nil || sig.WASMNative || sig.WASMContext != "" || + fn.FrameSize != 0 || strings.Contains(fn.Sym, "<") || len(sig.Args) == 0 || len(sig.Args) > 16 { + return fail("needs complete ordinary AMD64 ABI0 pointer/void types and source") + } + name := strings.TrimPrefix(sig.Name, pkg.Types.Path()+".") + declaration, ok := pkg.Types.Scope().Lookup(name).(*types.Func) + if !ok || name == sig.Name || declaration.Exported() { + return fail("entry must be an unexported actual package declaration") + } + actual, ok := declaration.Type().(*types.Signature) + if !ok || actual.Recv() != nil || actual.Variadic() || actual.TypeParams().Len() != 0 || + actual.Results().Len() != 0 || actual.Params().Len() != len(sig.Args) || len(sig.Frame.Params) != len(sig.Args) || + fn.ArgSize != int64(8*len(sig.Args)) { + return fail("signature is not the complete pointer-only ABI0 declaration") + } + for index, argument := range sig.Args { + bound := false + for _, slot := range sig.Frame.Params { + bound = bound || slot.Offset == int64(index*8) && slot.Index == index && slot.Type == Ptr && len(frameSlotFields(slot)) == 0 + } + if argument != Ptr || !types.Identical(actual.Params().At(index).Type(), types.Typ[types.UnsafePointer]) || + !bound || !x86RawReturnFPBound(int64(index*8), 8, sig, arch) { + return fail("parameter types or exact FP slots differ from the declaration") + } + } + if err := x86ValidateCallerSelection(pkg, selected); err != nil { + return fail(err.Error()) + } + if err := x86RejectAssemblyCallers(selected, fn, name, sig.Name, arch); err != nil { + return fail(err.Error()) + } + + proof := &x86GoCallerRegionProof{ + symbol: sig.Name, signature: sig, + goSources: make(map[string][32]byte), assembly: make(map[string][32]byte), + } + // Slices and maps supplied by the compiler remain mutable. The producer + // owns an independent signature and byte fingerprint, never an alias of + // an operand or layout that a later normalization may change. + proof.signature.Args = append([]LLVMType(nil), sig.Args...) + proof.signature.Frame.Params = append([]FrameSlot(nil), sig.Frame.Params...) + var err error + proof.source, err = x86CallerSourceFingerprint(fn) + if err != nil { + return fail("translated source cannot be pinned: " + err.Error()) + } + for file, source := range selected.GoSources { + proof.goSources[file] = sha256.Sum256(source) + } + for file, assembly := range selected.Assembly { + fingerprint, err := x86CallerSourceFingerprint(assembly) + if err != nil { + return fail("selected assembly cannot be pinned: " + err.Error()) + } + proof.assembly[file] = fingerprint + } + allowedUses := map[*ast.Ident]bool{} + declarations := 0 + for _, file := range pkg.Syntax { + for _, group := range file.Comments { + for _, comment := range group.List { + if strings.Contains(comment.Text, "go:linkname") { + for _, field := range strings.Fields(comment.Text) { + if field == name || field == sig.Name { + return fail("linkname prevents package-local caller closure") + } + } + } + } + } + for _, node := range file.Decls { + function, ok := node.(*ast.FuncDecl) + if !ok { + continue + } + if selected.TypesInfo.Defs[function.Name] == declaration { + if function.Body != nil { + return fail("assembly declaration has a Go body") + } + declarations++ + continue + } + if function.Body == nil { + continue + } + for statementIndex, statement := range function.Body.List { + expression, ok := statement.(*ast.ExprStmt) + if !ok { + continue + } + call, ok := expression.X.(*ast.CallExpr) + if !ok { + continue + } + identifier, ok := call.Fun.(*ast.Ident) + if !ok || selected.TypesInfo.Uses[identifier] != declaration { + continue + } + regions, err := x86DeriveCallerArguments(function, statementIndex, call, selected.TypesInfo) + if err != nil { + return fail(err.Error()) + } + for index := range regions { + regions[index].slot = int64(index * 8) + } + if proof.arguments != nil && !reflect.DeepEqual(proof.arguments, regions) { + return fail("different callers do not share the same parameter/write envelope") + } + proof.arguments = regions + proof.calls++ + allowedUses[identifier] = true + } + } + } + if declarations != 1 || proof.calls == 0 { + return fail("missing bodyless declaration or closed direct caller") + } + closed := true + for _, file := range pkg.Syntax { + ast.Inspect(file, func(node ast.Node) bool { + identifier, ok := node.(*ast.Ident) + if ok && selected.TypesInfo.Uses[identifier] == declaration && !allowedUses[identifier] { + closed = false // Address-taking, indirect and nested calls never disappear. + } + return true + }) + } + if !closed { + return fail("entry has a non-direct, nested or unregistered caller") + } + return proof, nil +} + +func x86CallerTargetSizesValid(sizes types.Sizes) bool { + if sizes == nil || sizes.Sizeof(types.Typ[types.UnsafePointer]) != 8 { + return false + } + for _, kind := range []types.BasicKind{ + types.Int8, types.Uint8, types.Int16, types.Uint16, types.Int32, types.Uint32, + types.Int64, types.Uint64, types.Int, types.Uint, types.Float32, types.Float64, + } { + object := types.NewVar(token.NoPos, nil, "", types.Typ[kind]) + if sizes.Sizeof(object.Type()) != x86CallerNumericWidth(object) { + return false + } + } + return true +} + +func x86ValidateCallerSelection(pkg GoPackage, selected GoSourceSelection) error { + if len(selected.SelectedGoFiles) == 0 || len(selected.SelectedGoFiles) != len(selected.GoSources) || + len(pkg.Syntax) != len(selected.SelectedGoFiles) || len(selected.SelectedAssemblyFiles) == 0 || + len(selected.SelectedAssemblyFiles) != len(selected.Assembly) { + return fmt.Errorf("complete selected Go and assembly inventory is required") + } + seen := map[string]bool{} + for _, name := range selected.SelectedGoFiles { + source, ok := selected.GoSources[name] + if name == "" || seen[name] || !ok { + return fmt.Errorf("missing/duplicate selected Go source") + } + seen[name] = true + var syntax *ast.File + for _, file := range pkg.Syntax { + if selected.Fset.Position(file.Pos()).Filename == name { + if syntax != nil { + return fmt.Errorf("duplicate selected Go syntax") + } + syntax = file + } + } + if syntax == nil { + return fmt.Errorf("selected Go source has no actual typed syntax") + } + identitiesMatch := true + ast.Inspect(syntax, func(node ast.Node) bool { + if identifier, ok := node.(*ast.Ident); ok { + object := selected.TypesInfo.ObjectOf(identifier) + if object != nil && object.Name() != identifier.Name { + identitiesMatch = false + } + } + return true + }) + if !identitiesMatch { + return fmt.Errorf("selected Go identifier differs from its actual typechecked identity") + } + fset := token.NewFileSet() + parsed, err := parser.ParseFile(fset, name, source, parser.ParseComments) + if err != nil { + return fmt.Errorf("selected Go source is not the typed source: %w", err) + } + var original, typed bytes.Buffer + if err := format.Node(&original, fset, parsed); err != nil { + return err + } + if err := format.Node(&typed, selected.Fset, syntax); err != nil { + return err + } + if !bytes.Equal(original.Bytes(), typed.Bytes()) { + return fmt.Errorf("selected Go bytes differ from the actual typed syntax") + } + } + seen = map[string]bool{} + for _, name := range selected.SelectedAssemblyFiles { + if name == "" || seen[name] || selected.Assembly[name] == nil || selected.Assembly[name].Arch != ArchAMD64 { + return fmt.Errorf("missing/duplicate selected assembly source") + } + seen[name] = true + } + return nil +} + +func x86RejectAssemblyCallers(selected GoSourceSelection, target Func, localName, resolvedName, arch string) error { + entries := 0 + for _, file := range selected.Assembly { + if len(file.UnlinkedPrelude) != 0 { + return fmt.Errorf("unregistered assembly prelude prevents caller closure") + } + for _, data := range file.Data { + base, _, reference := parseSBRef(strings.TrimPrefix(data.Addr, "$")) + if reference && (base == target.Sym || base == "·"+localName || base == resolvedName) { + return fmt.Errorf("assembly DATA exposes the Go entry") + } + } + for _, fn := range file.Funcs { + if len(fn.X86RawText) != 0 { + return fmt.Errorf("address-sensitive raw assembly prevents caller closure") + } + if fn.Sym == target.Sym { + if !reflect.DeepEqual(fn, target) { + return fmt.Errorf("selected assembly entry differs from the translated source") + } + entries++ + } + decoded, err := decodeX86RawDirectives(fn, arch) + if err != nil { + return fmt.Errorf("selected assembly caller effects are not known: %w", err) + } + for _, ins := range decoded.Instrs { + for _, argument := range ins.Args { + symbol := argument.Sym + if argument.Kind == OpMem { + symbol = argument.Mem.Sym + } + symbol = strings.TrimPrefix(strings.TrimSpace(symbol), "$") + base, _, reference := parseSBRef(symbol) + if reference && (base == target.Sym || base == "·"+localName || base == resolvedName) && ins.Op != OpTEXT { + return fmt.Errorf("selected assembly calls or exposes the Go entry") + } + } + if ins.Op == OpCALL { + return fmt.Errorf("unknown assembly call cannot close package-local callers") + } + if ins.Op == OpJMP { + if len(ins.Args) != 1 { + return fmt.Errorf("incomplete assembly branch prevents caller closure") + } + argument := ins.Args[0] + if argument.Kind != OpLabel && argument.Kind != OpIdent && + !(argument.Kind == OpMem && argument.Mem.Base == PC && argument.Mem.Index == "" && argument.Mem.Sym == "") { + return fmt.Errorf("indirect or external assembly branch prevents caller closure") + } + } + } + } + } + if entries != 1 { + return fmt.Errorf("translated entry is not unique in the complete selected assembly") + } + return nil +} + +func x86DeriveCallerArguments(function *ast.FuncDecl, index int, call *ast.CallExpr, info *types.Info) ([]x86CallerArgumentRegion, error) { + if call.Ellipsis.IsValid() || len(call.Args) == 0 { + return nil, fmt.Errorf("direct call has incomplete argument flow") + } + parameters := map[types.Object]bool{} + if function.Type.Params != nil { + for _, field := range function.Type.Params.List { + for _, name := range field.Names { + parameters[info.Defs[name]] = true + } + } + } + locals := map[types.Object]x86CallerArgumentRegion{} + for _, statement := range function.Body.List[:index] { + switch statement := statement.(type) { + case *ast.DeclStmt: + declaration, ok := statement.Decl.(*ast.GenDecl) + if !ok || declaration.Tok != token.VAR { + continue + } + for _, spec := range declaration.Specs { + value, ok := spec.(*ast.ValueSpec) + if !ok || len(value.Values) != 0 { + continue + } + for _, name := range value.Names { + object := info.Defs[name] + if width := x86CallerNumericWidth(object); width != 0 { + locals[object] = x86CallerArgumentRegion{kind: x86CallerFreshResult, width: width} + } + } + } + case *ast.AssignStmt: + if statement.Tok != token.DEFINE || len(statement.Lhs) != 1 || len(statement.Rhs) != 1 { + continue + } + name, ok := statement.Lhs[0].(*ast.Ident) + length, call := statement.Rhs[0].(*ast.CallExpr) + if !ok || !call || len(length.Args) != 1 { + continue + } + builtin, ok := length.Fun.(*ast.Ident) + slice, sliceOK := length.Args[0].(*ast.Ident) + if ok && sliceOK && info.Uses[builtin] == types.Universe.Lookup("len") && parameters[info.Uses[slice]] { + if _, ok := info.Uses[slice].Type().Underlying().(*types.Slice); ok { + locals[info.Defs[name]] = x86CallerArgumentRegion{kind: x86CallerLocalLength, width: 8} + } + } + } + } + var regions []x86CallerArgumentRegion + allowedAddresses := map[*ast.Ident]bool{} + localObjects := map[types.Object]bool{} + results := 0 + for _, argument := range call.Args { + conversion, ok := argument.(*ast.CallExpr) + if !ok || len(conversion.Args) != 1 || conversion.Ellipsis.IsValid() || + !info.Types[conversion.Fun].IsType() || !types.Identical(info.TypeOf(conversion.Fun), types.Typ[types.UnsafePointer]) { + return nil, fmt.Errorf("argument is not an actual unsafe.Pointer conversion") + } + switch value := conversion.Args[0].(type) { + case *ast.UnaryExpr: + identifier, ok := value.X.(*ast.Ident) + if value.Op != token.AND || !ok { + return nil, fmt.Errorf("result/length address is not a fresh direct local") + } + object := info.Uses[identifier] + region, ok := locals[object] + if !ok || localObjects[object] { + return nil, fmt.Errorf("result/length local is missing, aliased or not fresh") + } + regions = append(regions, region) + localObjects[object] = true + allowedAddresses[identifier] = true + if region.kind == x86CallerFreshResult { + results++ + } + case *ast.CallExpr: + selector, ok := value.Fun.(*ast.SelectorExpr) + if !ok || info.Uses[selector.Sel] != types.Unsafe.Scope().Lookup("SliceData") || len(value.Args) != 1 { + return nil, fmt.Errorf("input is not actual unsafe.SliceData") + } + identifier, ok := value.Args[0].(*ast.Ident) + if !ok || !parameters[info.Uses[identifier]] { + return nil, fmt.Errorf("slice input is not an independent typed caller parameter") + } + slice, ok := info.Uses[identifier].Type().Underlying().(*types.Slice) + if !ok { + return nil, fmt.Errorf("input is not a typed slice") + } + width := x86CallerNumericWidth(types.NewVar(token.NoPos, nil, "", slice.Elem())) + if width == 0 { + return nil, fmt.Errorf("slice element layout is not a supported numerical input") + } + regions = append(regions, x86CallerArgumentRegion{kind: x86CallerSliceInput, width: width}) + default: + return nil, fmt.Errorf("unsafe arithmetic or non-direct argument flow") + } + } + if results == 0 { + return nil, fmt.Errorf("caller has no independent fresh result envelope") + } + valid := true + ast.Inspect(function.Body, func(node ast.Node) bool { + if address, ok := node.(*ast.UnaryExpr); ok && address.Op == token.AND { + if identifier, ok := address.X.(*ast.Ident); ok && localObjects[info.Uses[identifier]] && !allowedAddresses[identifier] { + valid = false + } + } + if assignment, ok := node.(*ast.AssignStmt); ok && assignment.Tok != token.DEFINE { + for _, target := range assignment.Lhs { + if identifier, ok := target.(*ast.Ident); ok && localObjects[info.Uses[identifier]] { + valid = false + } + } + } + if increment, ok := node.(*ast.IncDecStmt); ok { + if identifier, ok := increment.X.(*ast.Ident); ok && localObjects[info.Uses[identifier]] { + valid = false + } + } + return true + }) + if !valid { + return nil, fmt.Errorf("fresh local address escaped or its value was reassigned") + } + return regions, nil +} + +func x86CallerSourceFingerprint(source any) ([32]byte, error) { + // The parser's exported source operands/bytes contain only JSON-encodable + // values. Private decoder certificates are deliberately not source input. + encoded, err := json.Marshal(source) + if err != nil { + return [32]byte{}, err + } + return sha256.Sum256(encoded), nil +} + +func x86CallerNumericWidth(object types.Object) int64 { + if object == nil { + return 0 + } + basic, ok := object.Type().(*types.Basic) + if !ok { + return 0 + } + switch basic.Kind() { + case types.Uint8, types.Int8: + return 1 + case types.Uint16, types.Int16: + return 2 + case types.Uint32, types.Int32, types.Float32: + return 4 + case types.Uint64, types.Int64, types.Float64, types.Int, types.Uint: + return 8 + default: + return 0 + } +} diff --git a/amd64_go_caller_regions_test.go b/amd64_go_caller_regions_test.go new file mode 100644 index 000000000..327e72646 --- /dev/null +++ b/amd64_go_caller_regions_test.go @@ -0,0 +1,319 @@ +package plan9asm + +import ( + "crypto/sha256" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" +) + +type x86CallerTestImporter struct{} + +func (x86CallerTestImporter) Import(path string) (*types.Package, error) { + if path == "unsafe" { + return types.Unsafe, nil + } + panic("unexpected caller-region fixture import: " + path) +} + +const x86CallerFixtureSource = `package caller +import "unsafe" + +//go:noescape +func kernel(a, b, res, length unsafe.Pointer) + +func Call(x, y []float32) float32 { + var res float32 + length := len(x) + kernel( + unsafe.Pointer(unsafe.SliceData(x)), + unsafe.Pointer(unsafe.SliceData(y)), + unsafe.Pointer(&res), + unsafe.Pointer(&length)) + return res +} +` + +const x86CallerFixtureAssembly = `TEXT ·kernel(SB),4,$0-32 +MOVQ a+0(FP),DI +MOVQ b+8(FP),SI +MOVQ res+16(FP),DX +MOVQ length+24(FP),CX +BYTE $0x55 +WORD $0x8948 +BYTE $0xe5 +LONG $0xf8e48348 +MOVL $7,AX +MOVL AX,(DX) +WORD $0x8948 +BYTE $0xec +BYTE $0x5d +BYTE $0xc3 +` + +func x86CallerFixture(t *testing.T, source string) (GoPackage, GoSourceSelection, Func, FuncSig) { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "caller.go", source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + info := &types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + } + pkg, err := (&types.Config{ + Importer: x86CallerTestImporter{}, + Sizes: types.SizesFor("gc", "amd64"), + }).Check("caller", fset, []*ast.File{file}, info) + if err != nil { + t.Fatal(err) + } + assembly, err := Parse(ArchAMD64, x86CallerFixtureAssembly) + if err != nil { + t.Fatal(err) + } + goPkg := GoPackage{Path: "caller", Types: pkg, Syntax: []*ast.File{file}} + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(goPkg, assembly, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + selected := GoSourceSelection{ + Fset: fset, + TypesInfo: info, + Sizes: types.SizesFor("gc", "amd64"), + SelectedGoFiles: []string{"caller.go"}, + GoSources: map[string][]byte{"caller.go": []byte(source)}, + SelectedAssemblyFiles: []string{"kernel_amd64.s"}, + Assembly: map[string]*File{"kernel_amd64.s": assembly}, + } + return goPkg, selected, assembly.Funcs[0], sigs["caller.kernel"] +} + +func TestX86GoCallerRegionsDeriveActualTypedArguments(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64") + if err != nil { + t.Fatal(err) + } + if proof == nil || proof.symbol != "caller.kernel" || len(proof.arguments) != 4 || proof.calls != 1 { + t.Fatalf("missing actual caller/parameter binding: %+v", proof) + } + wantKinds := []x86CallerRegionKind{x86CallerSliceInput, x86CallerSliceInput, x86CallerFreshResult, x86CallerLocalLength} + wantWidths := []int64{4, 4, 4, 8} + for index, argument := range proof.arguments { + if argument.kind != wantKinds[index] || argument.width != wantWidths[index] || argument.slot != int64(index*8) { + t.Fatalf("argument %d: %+v", index, argument) + } + } + // A caller producer by itself must not turn this into a raw-return pass. + file, err := Parse(ArchAMD64, x86CallerFixtureAssembly) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{sig.Name: sig}, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }); err == nil { + t.Fatal("region metadata without a whole-source control proof became a pass") + } +} + +func TestX86GoCallerRegionsBindNumericalResultWidthsAndEveryCaller(t *testing.T) { + for _, result := range []struct { + name string + width int64 + }{ + {name: "uint8", width: 1}, {name: "int8", width: 1}, + {name: "uint16", width: 2}, {name: "int16", width: 2}, + {name: "uint32", width: 4}, {name: "int32", width: 4}, {name: "float32", width: 4}, + {name: "uint64", width: 8}, {name: "int64", width: 8}, {name: "float64", width: 8}, + } { + t.Run(result.name, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", result.name) + caller := source[strings.Index(source, "func Call("):] + source += strings.Replace(caller, "func Call(", "func SecondCall(", 1) + pkg, selected, fn, sig := x86CallerFixture(t, source) + proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64") + if err != nil { + t.Fatal(err) + } + if proof.calls != 2 || proof.arguments[2].width != result.width { + t.Fatalf("actual typed callers/result envelope: %+v", proof) + } + }) + } +} + +func TestX86GoCallerRegionsRejectIncompleteAndChangedSelection(t *testing.T) { + for _, change := range []string{ + "missing types", "missing type info", "missing Go bytes", "missing Go AST", "changed Go bytes", + "extra Go bytes", "missing ASM", "extra ASM", "changed signature", "wrong parameter index", "custom registers", "wrong architecture", "missing sizes", "wrong actual sizes", + } { + t.Run(change, func(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + arch := "amd64" + switch change { + case "missing types": + pkg.Types = nil + case "missing type info": + selected.TypesInfo = nil + case "missing Go bytes": + delete(selected.GoSources, "caller.go") + case "missing Go AST": + pkg.Syntax = nil + case "changed Go bytes": + selected.GoSources["caller.go"] = []byte(strings.Replace(x86CallerFixtureSource, "&res", "unsafe.SliceData(x)", 1)) + case "extra Go bytes": + selected.GoSources["unselected.go"] = []byte("package caller\n") + case "missing ASM": + delete(selected.Assembly, "kernel_amd64.s") + case "extra ASM": + selected.Assembly["unselected.s"] = selected.Assembly["kernel_amd64.s"] + case "changed signature": + sig.Args = []LLVMType{I64, Ptr, Ptr, Ptr} + case "wrong parameter index": + sig.Frame.Params[0].Index, sig.Frame.Params[1].Index = 1, 0 + case "custom registers": + sig.ArgRegs = []Reg{AX, BX, CX, DX} + case "wrong architecture": + arch = "arm64" + case "missing sizes": + selected.Sizes = nil + case "wrong actual sizes": + selected.Sizes = types.SizesFor("gc", "386") + } + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, arch); err == nil || proof != nil { + t.Fatalf("incomplete/changed actual source got a caller certificate: %+v, %v", proof, err) + } + }) + } +} + +func TestX86GoCallerRegionsRejectUnclosedCallsAndUnsafeRegions(t *testing.T) { + for _, source := range []string{ + x86CallerFixtureSource + "\nvar addressTaken = kernel\n", + strings.Replace(x86CallerFixtureSource, "//go:noescape", "//go:linkname kernel external.kernel", 1), + strings.Replace(x86CallerFixtureSource, "kernel(\n", "indirect := kernel\n\tindirect(\n", 1), + strings.Replace(x86CallerFixtureSource, "unsafe.Pointer(&res)", "unsafe.Pointer(unsafe.SliceData(x))", 1), + strings.Replace(x86CallerFixtureSource, "unsafe.Pointer(&res)", "unsafe.Pointer(uintptr(unsafe.Pointer(&res)) + 1)", 1), + strings.Replace(x86CallerFixtureSource, "length := len(x)", "length := 7", 1), + strings.Replace(x86CallerFixtureSource, "length := len(x)", "length := len(x)\n\tlength++", 1), + strings.Replace(x86CallerFixtureSource, "length := len(x)", "length := len(x)\n\tEscape(&res)", 1) + "\nfunc Escape(*float32) {}\n", + strings.Replace(x86CallerFixtureSource, "kernel(\n", "func() { kernel(\n", 1), + } { + // The last source is intentionally replaced with a valid closure below. + if strings.Contains(source, "func() { kernel") { + source = strings.Replace(source, "unsafe.Pointer(&length))", "unsafe.Pointer(&length)) }()", 1) + } + pkg, selected, fn, sig := x86CallerFixture(t, source) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("unclosed/unsafe source got a caller certificate:\n%s\n%+v, %v", source, proof, err) + } + } +} + +func TestX86GoCallerRegionsRejectAssemblyCallersAndAddressTaking(t *testing.T) { + for _, body := range []string{ + "CALL ·kernel(SB)", + "MOVQ $·kernel(SB),AX", + "JMP ·kernel(SB)", + "JMP external·unknown(SB)", + "BYTE $0xe8\nLONG $0", + "JMP AX", + } { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + sibling, err := Parse(ArchAMD64, "TEXT ·Sibling(SB),4,$0-0\n"+body+"\nRET\n") + if err != nil { + t.Fatal(err) + } + selected.SelectedAssemblyFiles = append(selected.SelectedAssemblyFiles, "sibling_amd64.s") + selected.Assembly["sibling_amd64.s"] = sibling + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("unregistered assembly caller got a certificate: %s: %+v, %v", body, proof, err) + } + } +} + +func TestX86GoCallerRegionsRejectAssemblyMemorySymbolExposure(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + sibling, err := Parse(ArchAMD64, "TEXT ·Sibling(SB),4,$0-0\nMOVQ (AX),BX\nRET\n") + if err != nil { + t.Fatal(err) + } + // The actual parser usually spells SB references as OpSym. A compiler's + // selected File may also carry a symbolic memory operand; closure must + // inspect that representation rather than silently treating it as numeric. + sibling.Funcs[0].Instrs[1].Args[0].Mem.Sym = "·kernel(SB)" + selected.SelectedAssemblyFiles = append(selected.SelectedAssemblyFiles, "sibling_amd64.s") + selected.Assembly["sibling_amd64.s"] = sibling + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("assembly memory exposed entry without closing its callers: %+v, %v", proof, err) + } +} + +func TestX86GoCallerRegionsKeepIndependentSourceAndSignatureSnapshots(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64") + if err != nil { + t.Fatal(err) + } + before := proof.source + fn.Instrs[1].Args[0].FPOffset = 8 + sig.Args[0] = I64 + sig.Frame.Params[0].Offset = 8 + selected.GoSources["caller.go"][0] = 'X' + after, err := x86CallerSourceFingerprint(fn) + if err != nil { + t.Fatal(err) + } + if proof.source != before || proof.source == after || + proof.signature.Args[0] != Ptr || proof.signature.Frame.Params[0].Offset != 0 || + proof.goSources["caller.go"] == sha256.Sum256(selected.GoSources["caller.go"]) { + t.Fatal("mutable compiler inputs changed an already-produced witness") + } +} + +func TestX86CallerSourceFingerprintRejectsOpaqueInput(t *testing.T) { + if _, err := x86CallerSourceFingerprint(make(chan int)); err == nil { + t.Fatal("opaque input acquired a source fingerprint") + } +} + +func TestX86GoCallerRegionsRejectEditedASTWithStaleTypeInfo(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + changed := false + ast.Inspect(pkg.Syntax[0], func(node ast.Node) bool { + if changed { + return false + } + address, ok := node.(*ast.UnaryExpr) + if !ok || address.Op != token.AND { + return true + } + identifier, ok := address.X.(*ast.Ident) + if ok && identifier.Name == "res" { + identifier.Name = "length" + changed = true + } + return true + }) + selected.GoSources["caller.go"] = []byte(strings.Replace(x86CallerFixtureSource, "&res", "&length", 1)) + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("edited AST retained stale typechecked local identity: %+v, %v", proof, err) + } +} + +func TestX86GoCallerRegionsRejectAssemblyDataAddressTaking(t *testing.T) { + pkg, selected, fn, sig := x86CallerFixture(t, x86CallerFixtureSource) + selected.Assembly["kernel_amd64.s"].Data = []DataStmt{{ + Sym: "entryAddress", Width: 8, Addr: "·kernel(SB)", + }} + if proof, err := deriveX86GoCallerRegions(pkg, selected, fn, sig, "amd64"); err == nil || proof != nil { + t.Fatalf("assembly DATA exposed a closed entry: %+v, %v", proof, err) + } +} diff --git a/amd64_go_caller_sad.go b/amd64_go_caller_sad.go new file mode 100644 index 000000000..5cad9ca42 --- /dev/null +++ b/amd64_go_caller_sad.go @@ -0,0 +1,35 @@ +package plan9asm + +func x86CallerSADStep(ins *Instr, form x86PackedSADForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + if form.memory >= 0 { + // A selected dword or zero mask does not shrink the native access. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + var known uint64 + for output := 0; output < form.byteWidth; output += 2 { + a, b := form.wordInputs(output) + if first&a == a && second&b == b { + known |= uint64(3) << uint(output) + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_sad_test.go b/amd64_go_caller_sad_test.go new file mode 100644 index 000000000..363137290 --- /dev/null +++ b/amd64_go_caller_sad_test.go @@ -0,0 +1,239 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerSADOriginalRawEffect(t *testing.T) { + // Unchanged instruction from Weaviate v1.39.6 hamming_avx256_amd64.s:77. + // The independent fixture binds its own complete selected typed callers; + // it does not borrow the external kernel's unresolved loop/frame contract. + body := "VMOVDQU (DI),Y2\nVMOVDQU (SI),Y5\nLONG $0xeaf6d5c5\nVMOVSS X5,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86GoCallerSADCompleteMemoryMaskAndAliasForms(t *testing.T) { + for _, row := range x86SADRows("amd64") { + // This caller fixture binds ordinary slice data, not arbitrary SB or + // segment sources. Those accepted table forms have separate objects. + if strings.Contains(row, "(FS)") || strings.Contains(row, "(GS)") || strings.Contains(row, "(SB)") { + continue + } + // Rebind only register/address tokens, never the immediate selector. + row = strings.NewReplacer( + "16(AX)(BX*2)", "(DI)", "(AX)", "(DI)", + "X15", "X2", "Y15", "Y2", "Z15", "Z2", + "X31", "X1", "Y31", "Y1", "Z31", "Z1", + "X30", "X2", "Y30", "Y2", "Z30", "Z2", + "X7", "X2", "Y7", "Y2", "Z7", "Z2", + ).Replace(row) + args := strings.Split(strings.SplitN(row, " ", 2)[1], ",") + index := args[len(args)-1][1:] + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + "\nVMOVSS X" + index + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestX86GoCallerSADMemoryCertificatesBindCompleteInstruction(t *testing.T) { + for _, row := range []string{"PSADBW (DI),X0", "MPSADBW $5,(DI),X0", "VPSADBW (DI),Y0,Y2", "VMPSADBW $37,(DI),Y0,Y2", "VDBPSADBW.Z $228,(DI),Z0,K1,Z2"} { + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS X2,(DX)" + if !strings.HasPrefix(row, "V") { + body = strings.Replace(body, "VMOVSS X2", "VMOVSS X0", 1) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86PackedSADProperties(ins, "amd64") + if !known || err != nil || form.memory < 0 { + continue + } + found = true + access, exists := ins.x86CallerMemory[form.memory] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatal("SAD lost its exact native full-read witness") + } + for _, change := range []string{"opcode", "immediate", "width", "address", "destination", "direction", "mask", "address size"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch change { + case "opcode": + mutated.Op = "VPMADDUBSW" + case "immediate": + if form.spec.kind != amd64PackedSADBlockSum { + mutated.Args[0].Imm++ + } else { + mutated.Args = append([]Operand{{Kind: OpImm, Imm: 1}}, mutated.Args...) + } + case "width": + access.width-- + mutated.x86CallerMemory[form.memory] = access + case "address": + mutated.Args[form.memory].Mem.Off++ + case "destination": + mutated.Args[form.destination].Reg = "X7" + case "direction": + access.write = true + mutated.x86CallerMemory[form.memory] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K7" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K1"}) + } + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, form.memory) { + t.Fatalf("%s stale %s permission survived", row, change) + } + } + } + if !found { + t.Fatal("no consumed SAD memory instruction") + } + } +} + +func TestX86GoCallerSADRetainsUnprovedSourcesAndEnvelopes(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Y1\nVPSADBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Y0\nVPSADBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + "unknown legacy": "MOVUPS (DI),X0\nMPSADBW $5,X0,X1\nVMOVSS X1,(DX)", + "unknown upper": "MOVUPS (DI),X0\nMOVUPS (SI),X1\nVPSADBW Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVDBPSADBW.Z $228,Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nVDBPSADBW $228,Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVDBPSADBW.Z $228,Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask full read": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVDBPSADBW.Z $0,(DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "selected full read": "VMOVUPS (DI),X0\nMPSADBW $0,(CX),X0\nVMOVSS X0,(DX)", + "unknown address": "VMOVUPS (DI),Y0\nVPSADBW (BX),Y0,Y1\nVMOVSS X1,(DX)", + "private SP": "VMOVUPS (DI),X0\nPSADBW (SP),X0\nVMOVSS X0,(DX)", + "private alias": "VMOVUPS (DI),X0\nMOVQ BP,BX\nMPSADBW $0,(BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),X0\nPSADBW X0,X0\nVMOVSS X0,(DI)", + "undefined join": "VMOVUPS (DI),Y0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Y1\njoined:\nVPSADBW Y0,Y1,Y2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("SAD became unproved source permission: %v", err) + } + }) + } +} + +func TestX86GoCallerSADIndependentByteDependencies(t *testing.T) { + // All selectors and widths are compared to perturbations of an independent + // numeric reference, not expectations generated by the production masks. + for _, kind := range []string{"PSADBW", "MPSADBW", "VDBPSADBW"} { + for _, width := range []int{16, 32, 64} { + if kind == "MPSADBW" && width == 64 { + continue + } + for control := 0; control < 256; control++ { + form := x86PackedSADForm{spec: amd64PackedSADSpecs[Op(kind)], immediate: uint8(control)} + form.byteWidth = width + for output := 0; output < width; output += 2 { + var first, second uint64 + for input := 0; input < width; input++ { + a, b := make([]byte, width), make([]byte, width) + a[input] = 1 + if x86SADReference(kind, width, control, a, b)[output/2] != 0 { + first |= uint64(1) << uint(input) + } + a[input], b[input] = 0, 1 + if x86SADReference(kind, width, control, a, b)[output/2] != 0 { + second |= uint64(1) << uint(input) + } + } + a, b := form.wordInputs(output) + if first != a || second != b { + t.Fatalf("%s width%d imm%d word%d: %x/%x != %x/%x", kind, width, control, output/2, a, b, first, second) + } + } + } + } + } + for _, row := range []string{"PSADBW X0,X1", "VPSADBW Y0,Y1,Y2", "MPSADBW $5,X0,X1", "VMPSADBW $37,Y0,Y1,Y2", "VDBPSADBW.Z $228,Z0,Z1,K1,Z2"} { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSADProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, defined := range []uint64{0, 15, 255, 65535, 0xffffffff, ^uint64(0)} { + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": defined, "Z1": defined, "Z2": 0}, masks: map[Reg]bool{"K1": true}} + if !x86CallerSADStep(&ins, form, state, nil) { + t.Fatal("register effect failed") + } + got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] + var want uint64 + for output := 0; output < form.byteWidth; output += 2 { + first, second := form.wordInputs(output) + if defined&first == first && defined&second == second { + want |= uint64(3) << uint(output) + } + } + if !form.legacy { + want |= ^x86CallerVectorByteMask(int64(form.byteWidth)) + } else { + want |= defined &^ x86CallerVectorByteMask(int64(form.byteWidth)) + } + if got != want { + t.Fatal(fmt.Sprintf("ordered %s: %x != %x", row, got, want)) + } + } + } +} + +func TestX86GoCallerSADCompleteNamedFamily(t *testing.T) { + for _, row := range []struct { + instruction string + result string + }{ + {"PSADBW X0,X1", "X1"}, + {"VPSADBW Y0,Y1,Y2", "X2"}, + {"MPSADBW $5,X0,X1", "X1"}, + {"VMPSADBW $37,Y0,Y1,Y2", "X2"}, + {"VDBPSADBW $228,Z0,Z1,K1,Z2", "X2"}, + } { + t.Run(row.instruction, func(t *testing.T) { + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row.instruction + "\nVMOVSS " + row.result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_saved_g.go b/amd64_go_caller_saved_g.go new file mode 100644 index 000000000..f64e8f339 --- /dev/null +++ b/amd64_go_caller_saved_g.go @@ -0,0 +1,41 @@ +package plan9asm + +import "strings" + +// Incoming g is opaque, not an initialized number or caller data pointer. +// Only an exact native-width private PUSH/POP may save/restore its identity. +// All non-stack uses still consume the actual caller's byte/provenance proof. +func x86SourceGRegister(register Reg) bool { + root, valid := amd64FullRegBase(register) + return valid && root == "R14" +} + +func x86SourceGOperand(instruction Instr) bool { + for _, operand := range instruction.Args { + if operand.Kind == OpReg && x86SourceGRegister(operand.Reg) || + operand.Kind == OpMem && (x86SourceGRegister(operand.Mem.Base) || x86SourceGRegister(operand.Mem.Index)) { + return true + } + } + return false +} + +func x86CallerGDefinition(instruction Instr, state *x86SourceStackState) { + op := Op(normalizeInstructionOpcode(instruction.Op)) + if _, stack := amd64StackWidthSpecs[op]; stack { + return // Stack transfer already consumed its exact opaque/numeric cell. + } + if op == OpTEXT || op == OpLABEL || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) || + strings.HasPrefix(string(op), "CMP") || strings.HasPrefix(string(op), "TEST") { + return + } + if count := len(instruction.Args); count != 0 { + destination := instruction.Args[count-1] + if destination.Kind == OpReg && x86SourceGRegister(destination.Reg) { + // Called only after the registered family's complete must-defined + // and pointer effects succeeded. No instruction can define the + // original g; every source return needs an exact saved token POP. + state.gOriginal = false + } + } +} diff --git a/amd64_go_caller_saved_g_runtime_test.go b/amd64_go_caller_saved_g_runtime_test.go new file mode 100644 index 000000000..52c769c4c --- /dev/null +++ b/amd64_go_caller_saved_g_runtime_test.go @@ -0,0 +1,119 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerSavedGRuntimeAssembly(assembly, mode string) string { + // Reuse independent W/L/Q LEA mathematics, including preservation of the + // initialized high bytes. The incoming g never supplies numerical data. + assembly = strings.ReplaceAll(assembly, "AX", "R14") + save, restore := "WORD $0x5641\n", "WORD $0x5e41\n" + if mode == "named" { + save, restore = "PUSHQ R14\n", "POPQ R14\n" + } else if mode == "nested" { + save = "PUSHQ R14\nMOVQ $91,R14\nPUSHQ R14\n" + restore = "POPQ R14\nPOPQ R14\n" + } + assembly = strings.ReplaceAll(assembly, "LONG $0xf8e48348\n", "LONG $0xf8e48348\n"+save) + return strings.ReplaceAll(assembly, "WORD $0x8948\nBYTE $0xec", restore+"WORD $0x8948\nBYTE $0xec") +} + +func x86CallerSavedGOriginalObserver() (goTest, assembly string) { + var source strings.Builder + source.WriteString(` +// This native observer is a separate test-only oracle, never a numerical +// input or declaration used to authorize the translated caller's source. +func savedGContext() (gvalue, spvalue, bpvalue uint64) +func TestOriginalSavedGContext(t *testing.T) { + for iteration := 0; iteration < 64; iteration++ { +`) + for _, form := range x86CallerLEARuntimeForms() { + fmt.Fprintf(&source, ` + { + x, y := []uint64{11,23}, []uint64{29,31} + beforeG, beforeSP, beforeBP := savedGContext() + _ = Call%s(x,y) + afterG, afterSP, afterBP := savedGContext() + if beforeG == 0 || beforeG != afterG || beforeSP != afterSP || beforeBP != afterBP { + t.Fatalf("%s changed native caller context g=%%x/%%x SP=%%x/%%x BP=%%x/%%x", + beforeG,afterG,beforeSP,afterSP,beforeBP,afterBP) + } + } +`, form.name, form.name) + } + source.WriteString(" }\n}\n") + return source.String(), `TEXT ·savedGContext(SB),4,$0-24 +MOVQ R14,AX +MOVQ AX,gvalue+0(FP) +MOVQ SP,AX +MOVQ AX,spvalue+8(FP) +MOVQ BP,AX +MOVQ AX,bpvalue+16(FP) +RET +` +} + +func TestCrossLinuxRuntimeMatrixX86GoCallerPrivateSavedG(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("saved-g execution requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + for _, mode := range []string{"raw", "named", "nested"} { + t.Run(mode, func(t *testing.T) { + goSource, original, goTest, cOracle := x86CallerLEARuntimeSources() + assembly := x86CallerSavedGRuntimeAssembly(original, mode) + observer, observerAssembly := x86CallerSavedGOriginalObserver() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": goSource, + "value_amd64.s": assembly, "value_test.go": goTest + observer, + "context_test_amd64.s": observerAssembly, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + stage, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(stage, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", + "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(stage, command); err != nil { + t.Fatalf("actual Go saved-g numerical/context oracle: %v\n%s", err, output) + } + + pkg := x86CallerRuntimePackage(t, goSource, assembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + triple := testTargetTriple(runtime.GOOS, "amd64") + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "saved_g", triple, + translation.Module.String(), cOracle, runner) + }) + } +} diff --git a/amd64_go_caller_saved_g_test.go b/amd64_go_caller_saved_g_test.go new file mode 100644 index 000000000..d917be64b --- /dev/null +++ b/amd64_go_caller_saved_g_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GoCallerPrivateSavedGOriginalRawPushPop(t *testing.T) { + for _, form := range []struct{ name, body string }{ + // The original dot_byte_256 / hamming_512 two-byte raw words. + {"original raw", "WORD $0x5641\nMOVL $7,R14\nMOVL R14,AX\nWORD $0x5e41"}, + {"named", "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nPOPQ R14"}, + {"nested numerical", "PUSHQ R14\nMOVQ $19,R14\nPUSHQ R14\nMOVL $7,R14\nPOPQ R14\nMOVL R14,AX\nPOPQ R14"}, + {"nested original", "PUSHQ R14\nPUSHQ R14\nMOVL $7,R14\nPOPQ R14\nPOPQ R14\nMOVL $7,AX"}, + {"defined byte", "PUSHQ R14\nMOVL $0,AX\nMOVB $7,R14\nMOVB R14,AX\nPOPQ R14"}, + {"defined word", "PUSHQ R14\nMOVL $0,AX\nMOVW $7,R14\nMOVW R14,AX\nPOPQ R14"}, + {"defined zero", "PUSHQ R14\nXORQ R14,R14\nMOVL R14,AX\nPOPQ R14"}, + {"actual caller pointer", "PUSHQ R14\nMOVQ DI,R14\nMOVL (R14),AX\nPOPQ R14"}, + {"identical joins", "PUSHQ R14\nMOVL $7,R14\nCMPQ R14,$0\nJE joined\nMOVL $7,R14\njoined:\nMOVL R14,AX\nPOPQ R14"}, + {"both CFG arms restore", "WORD $0x5641\nMOVL $7,R14\nCMPQ R14,$0\nJE other\nWORD $0x5e41\nJMP joined\nother:\nWORD $0x5e41\njoined:\nMOVL $7,AX"}, + } { + t.Run(form.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX", form.body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } +} + +func TestX86GoCallerPrivateSavedGDoesNotExposeOpaqueIdentity(t *testing.T) { + for _, form := range []struct{ name, body string }{ + {"entry number", "MOVL R14,AX"}, + {"entry address", "MOVL (R14),AX"}, + {"entry flags", "CMPQ R14,$0\nMOVL $7,AX"}, + {"no restoration", "MOVL $7,R14\nMOVL R14,AX"}, + {"zero is not g", "XORQ R14,R14\nMOVL R14,AX"}, + {"saved then discarded", "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nADDQ $8,SP"}, + {"saved token to ordinary GP", "PUSHQ R14\nPOPQ BX\nMOVL $7,AX"}, + {"restored token read", "PUSHQ R14\nMOVL $7,R14\nPOPQ R14\nMOVL R14,AX"}, + {"restored token address", "PUSHQ R14\nMOVQ DI,R14\nPOPQ R14\nMOVL (R14),AX"}, + {"narrow original save", "PUSHW R14\nMOVW $7,R14\nMOVW R14,AX\nPOPW R14"}, + {"narrow restoration", "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nPOPW R14\nADDQ $6,SP"}, + {"unknown upper numerical bytes", "PUSHQ R14\nMOVW $7,R14\nMOVQ R14,AX\nPOPQ R14"}, + {"byte copy cannot define full result", "PUSHQ R14\nMOVB $7,R14\nMOVB R14,AX\nPOPQ R14"}, + {"word copy cannot define full result", "PUSHQ R14\nMOVW $7,R14\nMOVW R14,AX\nPOPQ R14"}, + {"opaque saved stack read", "PUSHQ R14\nMOVQ 0(SP),AX\nPOPQ R14"}, + {"opaque saved stack overwrite", "PUSHQ R14\nMOVQ $0,0(SP)\nPOPQ R14\nMOVL $7,AX"}, + {"unknown native call", "PUSHQ R14\nCALL ·unknown(SB)\nPOPQ R14\nMOVL $7,AX"}, + {"one unbalanced CFG arm", "PUSHQ R14\nMOVL $7,R14\nCMPQ R14,$0\nJE joined\nPOPQ R14\njoined:\nMOVL $7,AX"}, + {"one CFG arm loses g", "WORD $0x5641\nMOVL $7,R14\nCMPQ R14,$0\nJE other\nWORD $0x5e41\nJMP joined\nother:\nADDQ $8,SP\njoined:\nMOVL $7,AX"}, + {"g is not the input pointer", "PUSHQ R14\nMOVQ DI,R14\nMOVQ R14,(DX)\nPOPQ R14\nMOVL $7,AX"}, + } { + t.Run(form.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX", form.body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("opaque or unrestored g acquired source continuation: %v", err) + } + }) + } +} + +func TestX86GoCallerPrivateSavedGRequiresActualCallerSelection(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX", + "PUSHQ R14\nMOVL $7,R14\nMOVL R14,AX\nPOPQ R14", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + pkg.SourceSelection = nil + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("bare declaration acquired numerical g permission: %v", err) + } +} + +func TestX86GoCallerPrivateSavedGRejectsUnprovedLeafFallback(t *testing.T) { + for _, body := range []string{ + "MOVQ R14,AX", + "MOVQ $7,R14\nMOVQ R14,AX", + "XORQ R14,R14\nMOVQ R14,AX", + } { + t.Run(body, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture("amd64", + body+"\nMOVQ AX,ret+8(FP)\nBYTE $0xc3") + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("text API trusted unproved incoming/clobbered g: %v", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if module != (llvm.Module{}) { + module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("module API trusted unproved incoming/clobbered g: %v", err) + } + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + }) + if translation != nil { + translation.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("declaration API trusted unproved incoming/clobbered g: %v", err) + } + }) + } +} diff --git a/amd64_go_caller_signed_integer_float.go b/amd64_go_caller_signed_integer_float.go new file mode 100644 index 000000000..45a4623b4 --- /dev/null +++ b/amd64_go_caller_signed_integer_float.go @@ -0,0 +1,20 @@ +package plan9asm + +func x86CallerSignedIntegerFloatStep(ins *Instr, f x86SignedIntegerFloatForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.sourceBits/8), false) { + return false + } + } else if !x86CallerNumericRead(ins.Args[0], int64(f.sourceBits/8), state) { + return false + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + low := x86CallerVectorByteMask(int64(f.spec.resultBits / 8)) + if f.spec.legacy { + state.vectors[destination] |= low + } else { + upper := state.vectors[x86CallerVectorRoot(ins.Args[f.upper].Reg)] + state.vectors[destination] = ^x86CallerVectorByteMask(16) | low | upper&(x86CallerVectorByteMask(16)&^low) + } + return true +} diff --git a/amd64_go_caller_signed_integer_float_runtime_test.go b/amd64_go_caller_signed_integer_float_runtime_test.go new file mode 100644 index 000000000..f87f2ed5e --- /dev/null +++ b/amd64_go_caller_signed_integer_float_runtime_test.go @@ -0,0 +1,69 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestCrossLinuxRuntimeMatrixX86SignedIntegerScalarFloatActualCaller(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + for _, op := range []string{"CVTSL2SS", "CVTSL2SD", "CVTSQ2SS", "CVTSQ2SD", "VCVTSI2SSL", "VCVTSI2SDL", "VCVTSI2SSQ", "VCVTSI2SDQ"} { + t.Run(op, func(t *testing.T) { + goSource, bits, resultBits := x86CallerFixtureSource, 32, 32 + load, store, args := "MOVL (DI),AX", "MOVSS X1,(DX)", "AX,X1" + if strings.Contains(op, "Q") { + bits, load = 64, "MOVQ (DI),AX" + } + if strings.Contains(op, "SD") { + resultBits, store = 64, "MOVSD X1,(DX)" + goSource = strings.Replace(goSource, ") float32 {", ") float64 {", 1) + goSource = strings.Replace(goSource, "var res float32", "var res float64", 1) + } + if strings.HasPrefix(op, "V") { + args = "AX,X0,X1" + } + body := load + "\n" + op + " " + args + "\n" + store + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, goSource, assembly) + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "kernel_amd64.s", GOARCH: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) }, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + main := x86SignedIntegerFloatSSEOracle + fmt.Sprintf(` +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +extern void kernel(const void*,const void*,void*,const void*) __asm__(LABEL("caller.kernel")); +int main(void){ + for(unsigned sample=0;sample<256;sample++)for(unsigned mode=0;mode<4;mode++){ + uint64_t a[2]={UINT64_C(0xd31733a6287fbbc3)*sample+UINT64_C(0x1000001),UINT64_C(0x789abcdef1234567)}; + uint64_t b[2]={12,13},aCopy[2],bCopy[2];memcpy(aCopy,a,sizeof(a));memcpy(bCopy,b,sizeof(b)); + uint8_t out[16];memset(out,0xc3,sizeof(out));uint64_t length=4; + unsigned csr=0x9fc5|(mode<<13),after;uint64_t want=sse_convert(a[0],%d,%d,csr);unsigned expectedCSR=sse_csr; + __asm__ volatile("ldmxcsr %%0"::"m"(csr):"memory");kernel(a,b,out+1,&length); + __asm__ volatile("stmxcsr %%0":"=m"(after));uint64_t got=0;memcpy(&got,out+1,%d); + if(got!=want||after!=expectedCSR){fprintf(stderr,"actual caller %s input=%%llx got=%%llx want=%%llx csr=%%x want=%%x\n",(unsigned long long)a[0],(unsigned long long)got,(unsigned long long)want,after,expectedCSR);return 1;} + for(unsigned i=0;i%d)&&out[i]!=0xc3)return 2; + if(memcmp(a,aCopy,sizeof(a))||memcmp(b,bCopy,sizeof(b))||length!=4)return 3; + } + unsigned csr=0x1f80;__asm__ volatile("ldmxcsr %%0"::"m"(csr));return 0; +} +`, bits, resultBits, resultBits/8, op, resultBits/8) + compileAndRunRuntimeTestForTarget(t, llc, clang, "signed-float-caller", triple, x86SignedIntegerFloatOptimize(t, translation.Module.String()), main, prefix) + }) + } +} diff --git a/amd64_go_caller_signed_integer_float_test.go b/amd64_go_caller_signed_integer_float_test.go new file mode 100644 index 000000000..708f54922 --- /dev/null +++ b/amd64_go_caller_signed_integer_float_test.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerSignedIntegerFloatCompleteFamily(t *testing.T) { + for _, row := range x86SignedIntegerFloatRows("amd64") { + row = strings.ReplaceAll(strings.Replace(row, "8(BX)", "(DI)", 1), "(BX)(SI*4)", "(DI)") + body := "VMOVUPS (SI),Z0\nVMOVUPS (SI),Z1\n" + row + "\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s: %v", row, err) + } + } +} + +func TestX86GoCallerSignedIntegerFloatByteKnowledge(t *testing.T) { + for _, row := range x86SignedIntegerFloatRows("amd64") { + if strings.Contains(row, "(BX)") { + continue + } + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + f, _, err := x86SignedIntegerFloatProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for known := uint8(0); ; known++ { + state := x86SourceStackState{known: map[Reg]uint8{AX: known}, callers: map[Reg]x86CallerPointerValue{}, vectors: map[Reg]uint64{"Z0": 0x5500, "Z1": 0xaaff0000}, masks: map[Reg]bool{}} + accepted := x86CallerSignedIntegerFloatStep(&ins, f, &state, nil) + mask := uint8((1 << uint(f.sourceBits/8)) - 1) + if accepted != (known&mask == mask) { + t.Fatalf("%s GP knowledge %x: %t", row, known, accepted) + } + if accepted { + low := x86CallerVectorByteMask(int64(f.spec.resultBits / 8)) + want := uint64(0xaaff0000) | low + if !f.spec.legacy { + want = ^uint64(0xffff) | low | (0x5500 & (uint64(0xffff) &^ low)) + } + if state.vectors["Z1"] != want { + t.Fatalf("%s invented upper byte knowledge: %x want %x", row, state.vectors["Z1"], want) + } + } + if known == 255 { + break + } + } + } +} + +func TestX86GoCallerSignedIntegerFloatUnknownPermissionsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "pointer bits": "VCVTSI2SSL DI,X0,X1\nVMOVSS X1,(DX)", + "stack bits": "CVTSQ2SS SP,X1\nVMOVSS X1,(DX)", + "g bits": "VCVTSI2SSL R14,X0,X1\nVMOVSS X1,(DX)", + "undefined GP": "VCVTSI2SSQ R11,X0,X1\nVMOVSS X1,(DX)", + "partial GP": "MOVB $7,R11\nCVTSL2SS R11,X1\nVMOVSS X1,(DX)", + "source SP": "VCVTSI2SSL (SP),X0,X1\nVMOVSS X1,(DX)", + "unknown memory": "VCVTSI2SSL (R11),X0,X1\nVMOVSS X1,(DX)", + "pointer FP": "VCVTSI2SSL a+0(FP),X0,X1\nVMOVSS X1,(DX)", + "unknown upper passthrough": "VCVTSI2SSL AX,X0,X1\nVEXTRACTPS $3,X1,AX\nMOVL AX,(DX)", + "CFG undefined GP": "CMPQ AX,$0\nJE join\nMOVL $7,R11\njoin:\nVCVTSI2SSL R11,X0,X1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown permission became success: %v", err) + } + }) + } + // A numerical low result is independent of undefined passthrough bytes. + for _, op := range []string{"CVTSL2SS", "VCVTSI2SSL"} { + args := "AX,X1" + if strings.HasPrefix(op, "V") { + args = "AX,X0,X1" + } + body := fmt.Sprintf("%s %s\nVMOVSS X1,(DX)", op, args) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + } +} diff --git a/amd64_go_caller_signed_packed_float.go b/amd64_go_caller_signed_packed_float.go new file mode 100644 index 000000000..35f30ff85 --- /dev/null +++ b/amd64_go_caller_signed_packed_float.go @@ -0,0 +1,39 @@ +package plan9asm + +func x86CallerSignedPackedFloatStep(ins *Instr, f x86SignedPackedFloatForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + // Virtual MMX has no complete public caller x87/tag/physical alias contract. + if f.mmx || f.mask >= 0 && !state.masks[ins.Args[f.mask].Reg] { + return false + } + if f.memory >= 0 { + source := ins.Args[f.memory] + if source.Kind == OpFP || source.Kind == OpMem && + (source.Mem.Base == SP || source.Mem.Base == BP || source.Mem.Index == BP) { + return false + } + // The full potential tuple, even for inactive masks, is read-only input. + // The actual native opcode retains lane fault suppression; initialized + // masks do not authorize missing bytes or writable input pointers. + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.memoryBytes), false) { + return false + } + } else { + input := state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] + mask := x86CallerVectorByteMask(int64(f.inputBytes)) + if input&mask != mask { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + width := x86CallerVectorByteMask(int64(f.byteWidth)) + old := state.vectors[destination] + if f.mask >= 0 && !f.zeroing && old&width != width { + return false + } + if f.spec.legacy { + state.vectors[destination] = old&^width | width + } else { + state.vectors[destination] = ^uint64(0) + } + return true +} diff --git a/amd64_go_caller_signed_packed_float_test.go b/amd64_go_caller_signed_packed_float_test.go new file mode 100644 index 000000000..26dfafee9 --- /dev/null +++ b/amd64_go_caller_signed_packed_float_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86SignedPackedIntegerFloatCallerCompleteInitializedForms(t *testing.T) { + for _, op := range []string{"CVTPL2PS", "CVTPL2PD", "VCVTDQ2PS", "VCVTDQ2PD"} { + banks := []string{"X"} + if strings.HasPrefix(op, "V") { + banks = []string{"X", "Y", "Z"} + } + for _, bank := range banks { + src := bank + if strings.HasSuffix(op, "PD") { + src = "X" + if bank == "Z" { + src = "Y" + } + } + modes := []string{""} + if strings.HasPrefix(op, "V") { + modes = []string{"", "K1,", "K7,"} + } + for _, input := range []string{src + "0", "(DI)"} { + for _, mode := range modes { + name := op + if mode == "K7," { + name += ".Z" + } + row := fmt.Sprintf("%s %s,%s%s1", name, input, mode, bank) + t.Run(row, func(t *testing.T) { + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + "\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } + } + } +} + +func TestX86SignedPackedIntegerFloatCallerRetainsFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown input": "VCVTDQ2PS Y0,Y1\nVMOVSS X1,(DX)", + "partial dword": "MOVSS (DI),X2\nMOVSS X2,X0\nCVTPL2PS X0,X1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVCVTDQ2PS Z0,K1,Z1\nVMOVSS X1,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVCVTDQ2PS.Z Z0,K1,Z1\nVMOVSS X1,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVCVTDQ2PS.Z Z0,K1,Z1\nVMOVSS X1,(DX)", + "unknown base": "VCVTDQ2PD (BX),Y1\nVMOVSS X1,(DX)", + "private stack": "VCVTDQ2PS (SP),Y1\nVMOVSS X1,(DX)", + "FP pointer data": "CVTPL2PD a+0(FP),X1\nVMOVSS X1,(DX)", + "zero mask envelope": "MOVL $0,AX\nKMOVQ AX,K1\nVCVTDQ2PS.Z (DX),K1,Z1\nVMOVSS X1,(DX)", + "source write": "VCVTDQ2PS (DI),Y1\nVMOVSS X1,(DI)", + "wide result": "VCVTDQ2PD (DI),Y1\nVMOVUPS Y1,(DX)", + "join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),Y0\njoined:\nVCVTDQ2PS Y0,Y1\nVMOVSS X1,(DX)", + "unknown upper alias": "VMOVUPS (DI),X0\nCVTPL2PS X0,X1\nVCVTDQ2PS Y1,Y2\nVMOVSS X2,(DX)", + "MMX transport": "MOVQ (DI),M0\nCVTPL2PS M0,X1\nVMOVSS X1,(DX)", + "unsigned unknown upper": "MOVOU (DI),X0\nVCVTUDQ2PS Y0,Y1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved conversion got caller permission: %v", err) + } + }) + } +} + +func TestX86SignedPackedIntegerFloatGPTransferDefinesCompleteXInput(t *testing.T) { + // MOVQ GP -> X initializes the low qword and zeroes all remaining X + // bytes. In contrast, the register MOVSS negative above preserves the + // unknown upper twelve bytes of its destination. + body := "MOVQ AX,X0\nCVTPL2PS X0,X1\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("complete GP/X input lost its caller-defined bytes", err) + } +} + +func TestX86SignedPackedIntegerFloatIndependentByteEffects(t *testing.T) { + for _, row := range x86SignedPackedFloatRows("amd64") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + f, _, err := x86SignedPackedFloatProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + if f.memory >= 0 || f.mmx { + continue + } + for missing := -1; missing < f.inputBytes; missing++ { + input := x86CallerVectorByteMask(int64(f.inputBytes)) + if missing >= 0 { + input &^= uint64(1) << uint(missing) + } + for _, old := range []uint64{0, ^uint64(0)} { + state := x86SourceStackState{vectors: map[Reg]uint64{x86CallerVectorRoot(ins.Args[0].Reg): input}, masks: map[Reg]bool{"K1": true, "K3": true, "K7": true}} + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + if destination != x86CallerVectorRoot(ins.Args[0].Reg) { + state.vectors[destination] = old + } else { + old = input + } + valid := missing < 0 && (f.mask < 0 || f.zeroing || old&x86CallerVectorByteMask(int64(f.byteWidth)) == x86CallerVectorByteMask(int64(f.byteWidth))) + if got := x86CallerSignedPackedFloatStep(&ins, f, &state, nil); got != valid { + t.Fatalf("%s missing%d merge%016x got%t want%t", row, missing, old, got, valid) + } + if valid { + want := ^uint64(0) + if f.spec.legacy { + want = old | x86CallerVectorByteMask(16) + } + if state.vectors[destination] != want { + t.Fatal("alias/upper byte effect", row) + } + } + } + } + } +} diff --git a/amd64_go_caller_unpack.go b/amd64_go_caller_unpack.go new file mode 100644 index 000000000..801381f3d --- /dev/null +++ b/amd64_go_caller_unpack.go @@ -0,0 +1,47 @@ +package plan9asm + +func x86CallerPackedUnpackStep(ins *Instr, form x86PackedUnpackForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + first := uint64(0) + if form.memory >= 0 { + // Legacy m128 needs native alignment, not just a slice-data identity. + // No initialized mask shrinks an E4NF read's complete native extent. + if form.legacy || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + first = width + } else { + first = state.vectors[x86CallerVectorRoot(ins.Args[form.first].Reg)] & width + } + second := state.vectors[x86CallerVectorRoot(ins.Args[form.second].Reg)] & width + known := uint64(0) + laneBytes := form.spec.laneBits / 8 + start := 0 + if form.spec.high { + start = 8 + } + for group := 0; group < form.byteWidth; group += 16 { + for offset := 0; offset < 8; offset += laneBytes { + for byteIndex := 0; byteIndex < laneBytes; byteIndex++ { + source := uint(group + start + offset + byteIndex) + destination := uint(group + 2*offset + byteIndex) + known |= (second >> source & 1) << destination + known |= (first >> source & 1) << (destination + uint(laneBytes)) + } + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.mask >= 0 && !form.zeroing { + known &= old + } + if form.legacy { + state.vectors[destination] = old&^width | known + } else { + state.vectors[destination] = ^width | known + } + return true +} diff --git a/amd64_go_caller_unpack_test.go b/amd64_go_caller_unpack_test.go new file mode 100644 index 000000000..f7311d298 --- /dev/null +++ b/amd64_go_caller_unpack_test.go @@ -0,0 +1,278 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86CallerUnpackOpcodes() []string { + return []string{ + "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLWL", "PUNPCKHWL", + "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PUNPCKHQDQ", + "UNPCKLPS", "UNPCKHPS", "UNPCKLPD", "UNPCKHPD", + "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLWD", "VPUNPCKHWD", + "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", + "VUNPCKLPS", "VUNPCKHPS", "VUNPCKLPD", "VUNPCKHPD", + } +} + +func TestX86GoCallerUnpackCompleteInitializedGrammar(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + for _, original := range x86UnpackAcceptedForms(opcode, "amd64") { + t.Run(original, func(t *testing.T) { + row := strings.NewReplacer("8(AX)", "8(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(original) + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + "\nVMOVSS X2,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + needsContext := strings.Contains(row, "M7") || !strings.HasPrefix(opcode, "V") && strings.Contains(row, "(DI)") + if needsContext && !errors.Is(err, ErrProbeNeedsContext) || !needsContext && err != nil { + t.Fatalf("Context=%t: %v", needsContext, err) + } + }) + } + } +} + +func TestX86GoCallerUnpackRetainsUnknownAndMemoryFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),X1\nVPUNPCKLBW X0,X1,X2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),X0\nVPUNPCKLBW X0,X1,X2\nVMOVSS X2,(DX)", + "unknown selected high bytes": "MOVSS (DI),X0\nMOVSS X0,X1\nVUNPCKHPS X0,X1,X2\nVMOVSS X2,(DX)", + "unknown destructive input": "VMOVUPS (DI),X0\nPUNPCKLBW X0,X1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ AX,K1\nVPUNPCKLBW X0,X1,K1,X2\nVMOVSS X2,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nVPUNPCKLBW.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),X0\nVMOVUPS (SI),X1\nKMOVQ DI,K1\nVPUNPCKLBW.Z X0,X1,K1,X2\nVMOVSS X2,(DX)", + "zero mask still full E4NF read": "VMOVUPS (DI),X0\nMOVL $0,AX\nKMOVQ AX,K1\nVPUNPCKLDQ.Z (DX),X0,K1,X1\nVMOVSS X1,(DX)", + "zero mask broadcast still full read": "VMOVUPS (DI),X0\nMOVL $0,AX\nKMOVQ AX,K1\nVPUNPCKLQDQ.BCST.Z (DX),X0,K1,X1\nVMOVSS X1,(DX)", + "legacy alignment absent": "VMOVUPS (DI),X0\nPUNPCKLBW (DI),X0\nVMOVSS X0,(DX)", + "private stack": "VMOVUPS (DI),X0\nVPUNPCKLBW (SP),X0,X0\nVMOVSS X0,(DX)", + "unknown address": "VMOVUPS (DI),X0\nVPUNPCKLBW (BX),X0,X0\nVMOVSS X0,(DX)", + "read beyond scalar length": "VMOVUPS (DI),X0\nVPUNPCKLDQ (CX),X0,X0\nVMOVSS X0,(DX)", + "source write": "VMOVUPS (DI),X0\nVPUNPCKLBW X0,X0,X0\nVMOVSS X0,(DI)", + "wide result": "VMOVUPS (DI),X0\nVPUNPCKLBW X0,X0,X0\nVMOVUPS X0,(DX)", + "undefined CFG join": "VMOVUPS (DI),X0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),X1\njoined:\nVPUNPCKLBW X0,X1,X2\nVMOVSS X2,(DX)", + "MMX is a separate bank": "PUNPCKLBW M0,M1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved interleave acquired caller permission: %v", err) + } + }) + } +} + +func TestX86GoCallerUnpackIndependentByteKnowledge(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + widths := []int{16} + if strings.HasPrefix(opcode, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, destination := range []int{0, 1, 2} { + if !strings.HasPrefix(opcode, "V") && destination != 1 { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if !strings.HasPrefix(opcode, "V") && mode != "unmasked" { + continue + } + row := opcode + " " + bank + "0," + bank + "1" + if strings.HasPrefix(opcode, "V") { + if mode != "unmasked" { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + if mode == "zero" { + row = strings.Replace(row, opcode, opcode+".Z", 1) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedUnpackProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("%s: %v", row, err) + } + for seed := 0; seed < 64; seed++ { + initial := []uint64{0x19ca786fe32a907b, 0xf6210c89d8ba3547, 0x4a3b27e806c591df} + for index := range initial { + initial[index] ^= uint64(seed+index) * 0x102030405060708 + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": initial[0], "Z1": initial[1], "Z2": initial[2]}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedUnpackStep(&ins, form, &state, nil) { + t.Fatalf("register effect refused %s", row) + } + // Calculate each output byte from the source spelling and + // Intel's independent per-128-bit interleave definition. + var want uint64 + lane := x86UnpackTestLaneBytes(opcode) + high := strings.Contains(opcode, "UNPCKH") + for output := 0; output < 64; output++ { + known := false + if output >= width { + known = strings.HasPrefix(opcode, "V") || initial[destination]>>uint(output)&1 != 0 + } else { + pair := (output % 16) / (2 * lane) + sourceByte := output/16*16 + pair*lane + output%lane + if high { + sourceByte += 8 + } + sourceRegister := 1 + if output/lane%2 != 0 { + sourceRegister = 0 + } + known = initial[sourceRegister]>>uint(sourceByte)&1 != 0 + if mode == "merge" { + known = known && initial[destination]>>uint(output)&1 != 0 + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if got := state.vectors[Reg(fmt.Sprintf("Z%d", destination))]; got != want { + t.Fatalf("%s seed=%d: byte knowledge %#x want %#x", row, seed, got, want) + } + } + } + } + } + } +} + +func TestX86GoCallerUnpackCompleteFamilyThreeAPIs(t *testing.T) { + var body strings.Builder + body.WriteString("VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\n") + for _, opcode := range x86CallerUnpackOpcodes() { + if strings.HasPrefix(opcode, "V") { + fmt.Fprintf(&body, "%s Z0,Z1,Z2\nVMOVSS X2,(DX)\n", opcode) + } else { + fmt.Fprintf(&body, "%s X0,X1\nVMOVSS X1,(DX)\n", opcode) + } + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body.String(), 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "unpack.ll", "unpack.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, triple, "unpack-module.ll", "unpack-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerUnpackMemoryCertificatesBindFullE4NFSource(t *testing.T) { + for _, row := range []string{"VPUNPCKLBW (DI),X0,X2", "VUNPCKHPD.BCST.Z (DI),Z0,K1,Z2"} { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n"+row+"\nVMOVSS X2,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedUnpackProperties(ins, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, index) { + t.Fatal("missing full-width native read certificate") + } + for _, change := range []string{"opcode", "width", "address", "destination", "direction", "broadcast", "mask"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VPUNPCKHBW" + case "width": + access.width-- + mutated.x86CallerMemory[index] = access + case "address": + mutated.Args[index].Mem.Off++ + case "destination": + mutated.Args[form.destination].Reg = "X3" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "broadcast": + mutated.Op = Op(string(mutated.Op) + ".BCST") + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s mutation reused a stale interleave certificate", change) + } + } + } + if !found { + t.Fatal("no interleave instruction consumed its caller proof") + } + } +} + +func TestX86GoCallerUnpackCompleteFamilyOriginalSource(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + t.Run(opcode, func(t *testing.T) { + operands, destination := "X0,X1", "X1" + if strings.HasPrefix(opcode, "V") { + operands, destination = "Y0,Y1,Y2", "X2" + } + body := fmt.Sprintf("VMOVUPS (DI),Y0\nVMOVUPS (SI),Y1\n%s %s\nVMOVSS %s,(DX)", opcode, operands, destination) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_vector_moves.go b/amd64_go_caller_vector_moves.go new file mode 100644 index 000000000..f1adf9d02 --- /dev/null +++ b/amd64_go_caller_vector_moves.go @@ -0,0 +1,472 @@ +package plan9asm + +import "fmt" + +func x86CallerVectorMoveKnown(ins Instr) bool { + if _, known, err := x86ByteShuffleProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if form, known, err := x86PackedConversionProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86CLMULProperties(ins.Op, ins, "amd64"); known { + return err == nil && form.memory < 0 + } + if _, known, err := x86LegacyPackedMoveProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if _, known, err := x86ScalarLaneProperties(ins, "amd64"); known { + return err == nil + } + if _, known, err := x86AESProperties(ins.Op, ins, "amd64"); known { + return err == nil + } + if form, known, err := x86HorizontalIntegerProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86GPSIMDTransferProperties(ins, "amd64"); known { + return err == nil && !form.mmx && !form.spec.rawOnly + } + if form, known, err := x86ImmediateShuffleProperties(ins, "amd64"); known { + return err == nil && !form.spec.mmx + } + if _, known, err := x86MaskVectorProperties(ins, "amd64"); known { + return err == nil + } + if form, known, err := x86PackedNarrowProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86PackedWordMultiplyProperties(ins.Op, ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86DwordProductProperties(ins.Op, ins, "amd64"); known { + return err == nil && !form.mmx + } + if _, known, err := x86PackedSADProperties(ins, "amd64"); known { + return err == nil + } + if form, known, err := x86PackedSubtractProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if _, recognized, err := x86FloatCompareProperties(ins, "amd64"); recognized { + return err == nil + } + if form, known, err := x86PackedWrappingAddProperties(ins, "amd64"); known { + return err == nil && !form.mmx + } + if form, known, err := x86PackedMultiplyAddProperties(ins.Op, ins, "amd64"); known { + return err == nil && !form.mmx + } + if f, known, err := x86SignedPackedFloatProperties(ins, "amd64"); known { + return err == nil && !f.mmx + } + if _, recognized, err := x86FloatMulDivProperties(ins, "amd64"); recognized { + return err == nil + } + if _, recognized, err := x86PackedZeroExtendProperties(ins, "amd64"); recognized { + return err == nil + } + if _, recognized, err := x86VectorZeroProperties(ins, "amd64"); recognized { + return err == nil + } + if _, recognized, err := x86SignedIntegerFloatProperties(ins, "amd64"); recognized { + return err == nil + } + if f, recognized, err := x86InsertExtractProperties(ins, "amd64"); recognized { + return err == nil && !(f.mask >= 0 && f.memoryWrite) + } + if form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64"); recognized { + return err == nil && !form.mmx + } + if _, recognized, err := x86HorizontalFloatProperties(ins, "amd64"); recognized { + return err == nil + } + if form, recognized, err := x86PackedUnpackProperties(ins, "amd64"); recognized { + return err == nil && !form.mmx + } + if _, recognized, err := x86FloatAddSubProperties(ins, "amd64"); recognized { + return err == nil + } + if form, recognized, err := x86PackedLogicalProperties(ins, "amd64"); recognized { + return err == nil && !form.mmx + } + if amd64MaskMoveOperandsValid(ins, "amd64") { + return true + } + if form, recognized, err := amd64PackedVectorMoveProperties(ins, "amd64"); recognized { + return err == nil && !form.undefinedOpcode() + } + if form, recognized, err := amd64FMA3Properties(ins, "amd64"); recognized { + return err == nil && !form.spec.rawOnly + } + form, recognized, err := amd64ScalarVectorMoveProperties(ins, "amd64") + // The shared lowerer also models raw-only FP16 moves. Their caller-bank + // permission needs its own complete source/runtime family, not inheritance + // from the reviewed Go-named single/double-precision forms. + return recognized && err == nil && !form.spec.rawOnly +} + +func x86CloneCallerVectorState(vectors map[Reg]uint64, masks map[Reg]bool) (map[Reg]uint64, map[Reg]bool) { + vectorCopy := make(map[Reg]uint64, len(vectors)) + for reg, known := range vectors { + vectorCopy[reg] = known + } + maskCopy := make(map[Reg]bool, len(masks)) + for reg, known := range masks { + maskCopy[reg] = known + } + return vectorCopy, maskCopy +} + +func x86CallerVectorRoot(reg Reg) Reg { + index, _ := amd64VectorRegisterIndex(reg, amd64VectorByteWidth(reg)) + return Reg(fmt.Sprintf("Z%d", index)) +} + +func x86CallerVectorByteMask(width int64) uint64 { + if width <= 0 || width > 64 { + return 0 + } + if width == 64 { + return ^uint64(0) + } + return uint64(1)<= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + width := int64(form.spec.bits / 8) + lowMask := x86CallerVectorByteMask(width) + if form.memory >= 0 { + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, width, form.memoryWrite) { + return false + } + if form.memoryWrite { + source := x86CallerVectorRoot(ins.Args[form.low].Reg) + return state.vectors[source]&lowMask == lowMask + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + if form.spec.legacy || form.legacyZero { + if form.memory >= 0 || form.literal { + // A legacy memory load zeroes the rest of XMM but preserves the + // upper Y/Z aliases. Register copies preserve the old upper XMM. + state.vectors[destination] = old | x86CallerVectorByteMask(16) + } else { + source := x86CallerVectorRoot(ins.Args[form.low].Reg) + state.vectors[destination] = old&^lowMask | state.vectors[source]&lowMask + } + return true + } + known := ^uint64(0) + if form.memory < 0 && !form.literal { + low := state.vectors[x86CallerVectorRoot(ins.Args[form.low].Reg)] & lowMask + upperMask := x86CallerVectorByteMask(16) &^ lowMask + upper := state.vectors[x86CallerVectorRoot(ins.Args[form.upper].Reg)] & upperMask + known = ^x86CallerVectorByteMask(16) | low | upper + } + if form.mask >= 0 && !form.zeroing { + // The mask is initialized, not assumed all-one: either active data + // or the old low element may survive. Higher lanes follow the move + // form's upper merge/zero rules independently of that mask bit. + known = known&^lowMask | known&old&lowMask + } + state.vectors[destination] = known + return true +} + +func x86CallerMaskMoveStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + bits := amd64MaskMoveSpecs[ins.Op] + width := int64(bits / 8) + source, destination := ins.Args[0], ins.Args[1] + if source.Kind == OpMem { + if !x86CallerOrdinaryMemoryStep(ins, 0, state, proof, width, false) { + return false + } + } else if amd64IsKOperand(source) { + if !state.masks[source.Reg] { + return false + } + } else if !x86CallerNumericRead(source, width, state) { + return false // In particular, GP pointer bits cannot become mask data. + } + if destination.Kind == OpMem { + return x86CallerOrdinaryMemoryStep(ins, 1, state, proof, width, true) + } + if amd64IsKOperand(destination) { + state.masks[destination.Reg] = true + return true + } + // All KMOV GP destinations zero-extend the transferred value, including + // B/W/D, so the complete GP result is numeric; SP/BP/g stay forbidden. + return x86CallerNumericWrite(destination.Reg, 8, state) +} + +func x86CallerMoveMemoryShape(ins Instr, index int) (width int64, write, valid bool) { + return x86InstructionMemoryShape(ins, index, "amd64") +} + +func x86InstructionMemoryShape(ins Instr, index int, arch string) (width int64, write, valid bool) { + if form, known, err := x86ByteShuffleProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86CLMULProperties(ins.Op, ins, arch); known { + return int64(form.byteWidth), false, err == nil && form.memory >= 0 && index == form.memory + } + if form, known, err := x86ExplicitIMULProperties(ins.Op, ins, arch); known { + sourceIndex := 0 + if form.three { + sourceIndex = 1 + } + return int64(form.bits / 8), false, err == nil && index == sourceIndex && isAMD64MemoryOperand(form.source) + } + if form, known, err := x86AESProperties(ins.Op, ins, arch); known { + return int64(form.byteWidth), false, err == nil && form.memory >= 0 && form.memory == index + } + if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + return int64(form.bits / 8), false, err == nil && index == 0 && isAMD64MemoryOperand(form.source) + } + if form, known, err := x86ScalarLaneProperties(ins, arch); known { + return int64(form.bits / 8), !form.insert, err == nil && form.memory == index + } + if form, known, err := x86HorizontalIntegerProperties(ins, arch); known { + return int64(form.byteWidth), false, err == nil && !form.mmx && form.spec.vector && form.memory == index + } + if form, known, err := x86GPSIMDTransferProperties(ins, arch); known { + return int64(form.bits / 8), form.memoryWrite, err == nil && !form.mmx && form.memory == index + } + if form, recognized, err := x86LegacyPackedMoveProperties(ins.Op, ins, arch); recognized { + return 16, form.memoryWrite, err == nil && x86Mode32VectorRegisterContext(ins, arch) == nil && form.memory == index + } + if form, recognized, err := x86LegacyIntegerArithmeticProperties(ins.Op, ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if index < 0 || index >= len(ins.Args) || !isAMD64MemoryOperand(ins.Args[index]) { + return 0, false, false + } + if form, known, err := x86ImmediateShuffleProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.spec.legacy && form.memory == index + } + if form, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedWordMultiplyProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if form, known, err := x86DwordProductProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if form, known, err := x86PackedAverageProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedIntegerCompareProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); recognized { + return 1, true, err == nil && index == 0 + } + if form, known, err := x86PackedNarrowProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedSADProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedSubtractProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, known, err := x86PackedWrappingAddProperties(ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if form, known, err := x86PackedMultiplyAddProperties(ins.Op, ins, arch); known { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if f, known, err := x86PackedConversionProperties(ins, arch); known { + return int64(f.memoryBytes), false, err == nil && !f.mmx && f.memory == index + } + if f, known, err := x86SignedPackedFloatProperties(ins, arch); known { + return int64(f.memoryBytes), false, err == nil && !f.mmx && f.memory == index + } + if form, recognized, err := x86PackedZeroExtendProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, recognized, err := x86FloatCompareProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if f, recognized, err := x86SignedIntegerFloatProperties(ins, arch); recognized { + return int64(f.sourceBits / 8), false, err == nil && f.memory == index + } + if form, recognized, err := x86FloatMulDivProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if f, recognized, err := x86InsertExtractProperties(ins, arch); recognized { + return int64(f.spec.chunk), f.memoryWrite, err == nil && f.memory == index && !(f.mask >= 0 && f.memoryWrite) + } + if form, recognized, err := x86PackedShiftProperties(ins.Op, ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if form, recognized, err := x86HorizontalFloatProperties(ins, arch); recognized { + return int64(form.byteWidth), false, err == nil && form.spec.vector && form.memory == index + } + if form, recognized, err := x86PackedUnpackProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.mmx && !form.legacy && form.memory == index + } + if form, recognized, err := x86FloatAddSubProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && form.memory == index + } + if form, recognized, err := x86PackedLogicalProperties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.mmx && form.memory == index + } + if width = x86RawReturnScalarMoveBytes(Op(normalizeInstructionOpcode(ins.Op))); width != 0 && len(ins.Args) == 2 { + return width, index == 1, true + } + if form, recognized, err := x86ScalarExtensionProperties(ins.Op, ins, arch); recognized { + return int64(form.spec.sourceBits / 8), false, err == nil && index == 0 + } + if amd64MaskMoveOperandsValid(ins, arch) { + return int64(amd64MaskMoveSpecs[ins.Op] / 8), index == 1, true + } + if form, recognized, err := amd64PackedVectorMoveProperties(ins, arch); recognized { + return int64(form.byteWidth), form.memoryWrite, err == nil && !form.undefinedOpcode() && form.memory == index + } + if form, recognized, err := amd64FMA3Properties(ins, arch); recognized { + return int64(form.memoryBytes), false, err == nil && !form.spec.rawOnly && form.memory == index + } + form, recognized, err := amd64ScalarVectorMoveProperties(ins, arch) + if !recognized || err != nil || form.memory != index { + return 0, false, false + } + return int64(form.spec.bits / 8), form.memoryWrite, true +} + +func x86CallerPackedVectorMoveStep(ins *Instr, form amd64PackedVectorMoveForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.undefinedOpcode() || form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + widthMask := x86CallerVectorByteMask(int64(form.byteWidth)) + var known uint64 + if form.memory >= 0 { + // A slice-data/nonalias witness is not a SIMD alignment promise. + // Aligned register copies are valid, but aligned memory requires a + // separate actual-address proof before borrowing this continuation. + if form.spec.aligned || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.byteWidth), form.memoryWrite) { + return false + } + if form.memoryWrite { + return state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)]&widthMask == widthMask + } + known = widthMask + } else { + known = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] & widthMask + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + // No initialized mask is assumed all-one. Both potential inputs matter. + known = form.definedActiveBytes(known, state.vectors[destination]) + state.vectors[destination] = ^widthMask | known + return true +} diff --git a/amd64_go_caller_vector_moves_test.go b/amd64_go_caller_vector_moves_test.go new file mode 100644 index 000000000..0f74db41c --- /dev/null +++ b/amd64_go_caller_vector_moves_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerVectorMovesFreshScalarResults(t *testing.T) { + for _, width := range []struct { + suffix, typ string + }{{"SS", "float32"}, {"SD", "float64"}} { + legacy, vector := "MOV"+width.suffix, "VMOV"+width.suffix + for _, form := range []struct { + name, body string + }{ + {name: "legacy load/copy/store", body: fmt.Sprintf("%s (DI),X0\n%s X0,X1\n%s X1,(DX)", legacy, legacy, legacy)}, + {name: "vector load/merge/store", body: fmt.Sprintf("%s (DI),X0\n%s X0,X0,X1\n%s X1,(DX)", vector, vector, vector)}, + {name: "masked load zero", body: fmt.Sprintf("KMOVW AX,K1\n%s.Z (DI),K1,X0\n%s X0,(DX)", vector, vector)}, + {name: "masked store", body: fmt.Sprintf("KMOVW AX,K1\n%s (DI),X0\n%s X0,K1,(DX)", vector, vector)}, + {name: "masked merge", body: fmt.Sprintf("KMOVW AX,K1\n%s (DI),X0\n%s (SI),X1\n%s (DI),X2\n%s X0,X1,K1,X2\n%s X2,(DX)", vector, vector, vector, vector, vector)}, + {name: "masked merge zero", body: fmt.Sprintf("KMOVW AX,K1\n%s (DI),X0\n%s (SI),X1\n%s.Z X0,X1,K1,X2\n%s X2,(DX)", vector, vector, vector, vector)}, + {name: "extended vector register", body: fmt.Sprintf("%s (DI),X31\n%s X31,(DX)", vector, vector)}, + {name: "complete numeric overwrite", body: fmt.Sprintf("%s X0,X1,X2\n%s (DI),X2\n%s X2,(DX)", vector, vector, vector)}, + } { + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + t.Run(width.suffix+"/"+form.name+"/"+triple, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", width.typ) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", form.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } + } +} + +func TestX86GoCallerMaskMovesAllWidthsAndDirections(t *testing.T) { + for _, width := range []struct{ suffix, typ, move string }{ + {"B", "uint8", "MOVB"}, {"W", "uint16", "MOVW"}, + {"D", "uint32", "MOVL"}, {"Q", "uint64", "MOVQ"}, + } { + maskMove := "KMOV" + width.suffix + for _, form := range []struct{ name, body string }{ + {"GP to mask to memory", maskMove + " AX,K1\n" + maskMove + " K1,(DX)"}, + {"input to mask to memory", maskMove + " (DI),K1\n" + maskMove + " K1,(DX)"}, + {"length to mask to memory", maskMove + " (CX),K1\n" + maskMove + " K1,(DX)"}, + {"mask copy to memory", maskMove + " AX,K1\n" + maskMove + " K1,K2\n" + maskMove + " K2,(DX)"}, + {"mask to GP to memory", maskMove + " AX,K1\n" + maskMove + " K1,BX\n" + width.move + " BX,(DX)"}, + } { + t.Run(width.suffix+"/"+form.name, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", width.typ) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", form.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } +} + +func TestX86GoCallerVectorMovesDoNotWashPointersOrInventInitializedBytes(t *testing.T) { + for _, example := range []struct{ name, body, result string }{ + {"unknown mask", "VMOVSS.Z (DI),K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"unknown mask source", "KMOVW BX,K1\nVMOVSS.Z (DI),K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"pointer to mask", "KMOVW DI,K1\nVMOVSS.Z (DI),K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"fixed g to mask", "KMOVQ R14,K1\nKMOVQ K1,(DX)", "uint64"}, + {"mask to fixed g", "KMOVW AX,K1\nKMOVQ K1,R14\nMOVL AX,(DX)", "float32"}, + {"mask to SP", "KMOVW AX,K1\nKMOVQ K1,SP\nMOVL AX,(DX)", "float32"}, + {"mask to BP", "KMOVW AX,K1\nKMOVQ K1,BP\nMOVL AX,(DX)", "float32"}, + {"mask replaces input pointer", "KMOVW AX,K1\nKMOVQ K1,DI\nVMOVSS (DI),X0\nVMOVSS X0,(DX)", "float32"}, + {"GP pointer as vector data", "MOVQ DI,X0\nMOVSD X0,(DX)", "float64"}, + {"private SP as vector data", "MOVQ SP,X0\nMOVSD X0,(DX)", "float64"}, + {"fixed g as vector data", "MOVQ R14,X0\nMOVSD X0,(DX)", "float64"}, + {"FP pointer as vector data", "MOVSD a+0(FP),X0\nMOVSD X0,(DX)", "float64"}, + {"saved BP as vector data", "MOVSD 0(SP),X0\nMOVSD X0,(DX)", "float64"}, + {"unknown vector bytes", "VMOVSS X0,(DX)", "float32"}, + {"unknown low source", "VMOVSS (DI),X0\nVMOVSS X1,X0,X2\nVMOVSS X2,(DX)", "float32"}, + {"legacy unknown upper bytes", "MOVSS (DI),X0\nMOVSS X0,X1\nMOVSD X1,(DX)", "float64"}, + {"masked merge unknown destination", "KMOVW AX,K1\nVMOVSS (DI),X0\nVMOVSS (SI),X1\nVMOVSS X0,X1,K1,X2\nVMOVSS X2,(DX)", "float32"}, + {"wide result", "VMOVSD (DI),X0\nVMOVSD X0,(DX)", "float32"}, + {"result one byte overlap", "VMOVSS (DI),X0\nVMOVSS X0,1(DX)", "float32"}, + {"result negative offset", "VMOVSS (DI),X0\nVMOVSS X0,-1(DX)", "float32"}, + {"result indexed", "VMOVSS (DI),X0\nVMOVSS X0,(DX)(AX*1)", "float32"}, + {"input write", "VMOVSS (DI),X0\nVMOVSS X0,(DI)", "float32"}, + {"length write", "VMOVSS (DI),X0\nVMOVSS X0,(CX)", "float32"}, + {"unknown input base", "VMOVSS (BX),X0\nVMOVSS X0,(DX)", "float32"}, + {"mask input write", "KMOVW AX,K1\nKMOVW K1,(DI)", "uint16"}, + {"mask wide result", "KMOVQ AX,K1\nKMOVQ K1,(DX)", "uint32"}, + {"literal result one byte overlap", "MOVSS $1.25,X0\nMOVSS X0,1(DX)", "float32"}, + {"literal unknown mask", "VMOVSS.Z $1.25,K1,X0\nVMOVSS X0,(DX)", "float32"}, + {"literal mask from pointer", "KMOVW DI,K1\nVMOVSS.Z $1.25,K1,X0\nVMOVSS X0,(DX)", "float32"}, + } { + t.Run(example.name, func(t *testing.T) { + source := strings.ReplaceAll(x86CallerFixtureSource, "float32", example.result) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", example.body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, source, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved vector/mask state acquired raw-RET continuation: %v", err) + } + }) + } +} + +func TestX86GoCallerVectorMemoryCertificatesBindMaskWidthAndEveryOperand(t *testing.T) { + for _, body := range []string{ + "KMOVW AX,K1\nVMOVSS (DI),X0\nVMOVSS X0,K1,(DX)", + "KMOVD AX,K1\nKMOVD K1,(DX)", + } { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + found := false + for _, instruction := range normalized.Funcs[0].Instrs { + for index, access := range instruction.x86CallerMemory { + if !access.write || !x86CallerMemoryCertified(instruction, index) { + continue + } + found = true + for _, change := range []string{"opcode", "source", "address", "address-bits", "width", "direction", "mask"} { + mutated := instruction + mutated.Args = append([]Operand(nil), instruction.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VMOVSD" + case "source": + mutated.Args[0].Reg = "X3" + case "address": + mutated.Args[index].Mem.Off++ + case "address-bits": + mutated.x86AddressBits = 32 + case "width": + access.width++ + mutated.x86CallerMemory[index] = access + case "direction": + access.write = false + mutated.x86CallerMemory[index] = access + case "mask": + if len(mutated.Args) != 3 { + continue + } + mutated.Args[1].Reg = "K2" + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s borrowed a %s memory certificate", change, instruction.Op) + } + } + } + } + if !found { + t.Fatal("fixture produced no complete vector/mask write certificate") + } + } +} diff --git a/amd64_go_caller_vector_zero_test.go b/amd64_go_caller_vector_zero_test.go new file mode 100644 index 000000000..881aacc13 --- /dev/null +++ b/amd64_go_caller_vector_zero_test.go @@ -0,0 +1,124 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GoCallerVectorZeroOriginalSource(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Z0\nVZEROUPPER\nVMOVSS X0,(DX)", + "VZEROALL\nVMOVSS X0,(DX)", + "VMOVUPS (DI),Z0\nWORD $0xf8c5\nBYTE $0x77\nVMOVSS X0,(DX)", + "WORD $0xfcc5\nBYTE $0x77\nVMOVSS X0,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", "_"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86GoCallerVectorZeroInitializedBytesAndUnaffectedState(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, opcode := range []Op{"VZEROUPPER", "VZEROALL"} { + for _, initial := range []uint64{0, 1, 0xffff, 0xaaaaaaaaaaaaaaaa, ^uint64(0)} { + state := x86SourceStackState{ + vectors: map[Reg]uint64{}, masks: map[Reg]bool{"K1": true, "K2": false}, + known: map[Reg]uint8{AX: 0xff}, numericFlags: true, gOriginal: true, + } + for register := 0; register < 32; register++ { + state.vectors[Reg(fmt.Sprintf("Z%d", register))] = initial + } + known, masks := x86CloneCallerKnown(state.known), map[Reg]bool{"K1": true, "K2": false} + form, recognized, err := x86VectorZeroProperties(Instr{Op: opcode}, arch) + if !recognized || err != nil || !x86CallerVectorZeroStep(form, &state) { + t.Fatalf("%s/%s grammar/effect: %t/%v", arch, opcode, recognized, err) + } + limit, added := 16, ^uint64(0xffff) + if arch == "386" { + limit = 8 + } + if opcode == "VZEROALL" { + added = ^uint64(0) + } + for register := 0; register < 32; register++ { + want := initial + if register < limit { + want |= added + } + if state.vectors[Reg(fmt.Sprintf("Z%d", register))] != want { + t.Fatalf("%s/%s invented/lost register%d byte knowledge", arch, opcode, register) + } + } + if !reflect.DeepEqual(state.known, known) || !reflect.DeepEqual(state.masks, masks) || !state.numericFlags || !state.gOriginal { + t.Fatal("vector zero changed GP/mask/flags/g source state") + } + } + } + } +} + +func TestX86GoCallerVectorZeroDoesNotInventPreservedLowOrHighRegisters(t *testing.T) { + for _, body := range []string{ + "VZEROUPPER\nVMOVSS X0,(DX)", + "VZEROALL\nVMOVSS X16,(DX)", + "VZEROUPPER\nVMOVSS X31,(DX)", + "VZEROALL\nKMOVQ K1,AX\nMOVL AX,(DX)", + } { + t.Run(strings.ReplaceAll(body, "\n", "_"), func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown low/high/mask source acquired return proof: %v", err) + } + }) + } +} + +func TestX86GoCallerVectorZeroCompleteFamilyThreeAPIs(t *testing.T) { + const body = "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z16\nVZEROUPPER\nVMOVSS X0,(DX)\nVZEROALL\nVMOVSS X15,(DX)\nVMOVSS X16,(DX)\nWORD $0xfcc5\nBYTE $0x77\nVMOVSS X1,(DX)\n" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "zero-caller.ll", "zero-caller.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, triple, "zero-module.ll", "zero-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_caller_word_multiply_test.go b/amd64_go_caller_word_multiply_test.go new file mode 100644 index 000000000..ef6ebb643 --- /dev/null +++ b/amd64_go_caller_word_multiply_test.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerWordMultiplyCompleteForms(t *testing.T) { + for _, name := range x86WordMultiplyNames { + banks := []string{"X"} + if strings.HasPrefix(name, "V") { + banks = append(banks, "Y", "Z") + } + for _, bank := range banks { + for _, first := range []string{bank + "0", "(DI)"} { + for _, mode := range []int{0, 1, 2} { + if !strings.HasPrefix(name, "V") && mode != 0 { + continue + } + opcode, args, result := name, first+","+bank+"1", "X1" + if strings.HasPrefix(name, "V") { + args, result = first+","+bank+"1", "X2" + if mode != 0 { + args += ",K1" + } + if mode == 2 { + opcode += ".Z" + } + args += "," + bank + "2" + } + row := opcode + " " + args + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + }) + } + } + } + } +} + +func TestX86GoCallerWordMultiplyUnknownAndMemoryPermissionsRemainContext(t *testing.T) { + for name, body := range map[string]string{ + "unknown first": "VMOVUPS (DI),Z1\nVPMULLW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Z0\nVPMULHW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "partial source": "MOVSS (DI),X0\nVMOVUPS (SI),Z1\nVPMULHUW Z0,Z1,Z2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVPMULHRSW.Z Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nMOVL $1,AX\nKMOVQ AX,K1\nVPMULHW Z0,Z1,K1,Z2\nVMOVSS X2,(DX)", + "pointer mask": "VMOVUPS (DI),Z0\nKMOVQ DI,K1\nVPMULLW.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "zero mask lacks region": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMULHW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMULHUW (BX),Z0,Z2\nVMOVSS X2,(DX)", + "stack alias": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMULLW (BX),X0\nVMOVSS X0,(DX)", + "BP alias": "VMOVUPS (DI),Z0\nMOVQ BP,BX\nPMULHW (BX),X0\nVMOVSS X0,(DX)", + "input write": "VMOVUPS (DI),Z0\nVPMULHRSW Z0,Z0,Z2\nVMOVSS X2,(DI)", + "MMX separate bank": "PMULLW M0,M1\nVMOVSS X1,(DX)", + "join missing second": "VMOVUPS (DI),Z0\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z1\njoined:\nVPMULHW Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved word multiply acquired caller permission", err) + } + }) + } +} + +func TestX86GoCallerWordMultiplyFullReadCertificatesAndByteAliases(t *testing.T) { + names := append(append([]string(nil), x86WordMultiplyNames...), "PMULLD", "VPMULLD", "VPMULLQ") + for _, name := range names { + row, result := name+" (DI),X0", "X0" + if strings.HasPrefix(name, "V") { + row, result = name+".Z (DI),Z0,K1,Z2", "X2" + } + body := "VMOVUPS (SI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, known, err := x86LowAndWordMultiplyTestForm(ins) + if !known || err != nil || form.memory < 0 { + continue + } + found = true + access, exists := ins.x86CallerMemory[form.memory] + if !exists || access.width != int64(form.memoryBytes) || access.write || !x86CallerMemoryCertified(ins, form.memory) { + t.Fatal("zero mask shrank the independent possible read envelope") + } + for _, change := range []string{"op", "base", "width", "direction", "destination", "mask", "address size"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{form.memory: access} + switch change { + case "op": + mutated.Op = "MOVOU" + case "base": + mutated.Args[form.memory].Mem.Base = BX + case "width": + access.width-- + mutated.x86CallerMemory[form.memory] = access + case "direction": + access.write = true + mutated.x86CallerMemory[form.memory] = access + case "destination": + mutated.Args[form.destination].Reg = "X7" + case "mask": + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K7"}) + case "address size": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, form.memory) { + t.Fatal("stale original instruction memory certificate survived", change) + } + } + } + if !found { + t.Fatal("no multiply memory proof") + } + } + for _, name := range names { + operands := "X0,X1" + if strings.HasPrefix(name, "V") { + operands = "X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86LowAndWordMultiplyTestForm(ins) + if err != nil { + t.Fatal(err) + } + state := &x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xfffd, "Z1": 0xffff}, masks: map[Reg]bool{}} + if !x86CallerVectorMoveStep(&ins, state, nil) { + t.Fatal("registered pure vector effect missing") + } + want := uint64(0xfffc) // One absent source byte kills its whole word. + if name == "PMULLD" || name == "VPMULLD" { + want = 0xfff0 + } else if name == "VPMULLQ" { + want = 0xff00 + } + if !form.legacy { + want |= ^uint64(0xffff) + } + if got := state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)]; got != want { + t.Fatal(fmt.Sprintf("incorrect word-byte/upper alias transfer %s got=%x want=%x", name, got, want)) + } + } +} + +func x86LowAndWordMultiplyTestForm(ins Instr) (x86PackedBinaryForm, bool, error) { + if form, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, "amd64"); known { + return form.x86PackedBinaryForm, known, err + } + form, known, err := x86PackedWordMultiplyProperties(ins.Op, ins, "amd64") + return form.x86PackedBinaryForm, known, err +} diff --git a/amd64_go_caller_zero_extend.go b/amd64_go_caller_zero_extend.go new file mode 100644 index 000000000..ec2926802 --- /dev/null +++ b/amd64_go_caller_zero_extend.go @@ -0,0 +1,47 @@ +package plan9asm + +func x86CallerPackedZeroExtendStep(ins *Instr, form x86PackedZeroExtendForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mask >= 0 && !state.masks[ins.Args[form.mask].Reg] { + return false + } + var input uint64 + if form.memory >= 0 { + // Mask knowledge grants no address/alignment/envelope permission. + // Keep the complete potential read bound even when lanes can fault- + // suppress. Input regions never become writable scalar results. + if !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.memoryBytes), false) { + return false + } + input = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + input = state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)] + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + old := state.vectors[destination] + known := x86PackedZeroExtendDefinedBytes(form, input, old) + width := x86CallerVectorByteMask(int64(form.byteWidth)) + if form.spec.vector { + state.vectors[destination] = ^width | known + } else { + state.vectors[destination] = old&^width | known + } + return true +} + +// This pure byte dependency grants no memory or caller permission. Upper +// zero bits are known independently of input bytes; mask/alias writes remain +// distinct from the register and memory observations that precede this step. +func x86PackedZeroExtendDefinedBytes(form x86PackedZeroExtendForm, input, old uint64) uint64 { + var known uint64 + inputBytes, outputBytes := form.spec.inputBits/8, form.spec.outputBits/8 + for output := 0; output < form.byteWidth; output++ { + part := output % outputBytes + if part >= inputBytes || input>>uint(output/outputBytes*inputBytes+part)&1 != 0 { + known |= uint64(1) << uint(output) + } + } + if form.mask >= 0 && !form.zeroing { + known &= old // An initialized mask is not an all-active mask promise. + } + return known +} diff --git a/amd64_go_caller_zero_extend_test.go b/amd64_go_caller_zero_extend_test.go new file mode 100644 index 000000000..2d2eca87a --- /dev/null +++ b/amd64_go_caller_zero_extend_test.go @@ -0,0 +1,161 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GoCallerPackedZeroExtendCurrentGap(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", + "VMOVUPS (DI),X0\nVPMOVZXBD X0,Y1\nVMOVSS X1,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("Go-accepted bounded byte widening with initialized input: %v", err) + } +} + +func TestX86GoCallerPackedZeroExtendCompleteInitializedForms(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, original := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, "amd64") { + t.Run(original, func(t *testing.T) { + row := strings.NewReplacer("(AX)", "(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(original) + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (SI),Z2\nKMOVQ AX,K1\nKMOVQ AX,K7\n" + row + if strings.HasSuffix(row, "X1") || strings.HasSuffix(row, "Y1") { + body += "\nVMOVSS X1,(DX)" + } else { + body += "\nVMOVSS X2,(DX)" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } + } +} + +func TestX86GoCallerPackedZeroExtendRetainsUnknownAndEnvelopeFailures(t *testing.T) { + for name, body := range map[string]string{ + "unknown input": "VPMOVZXBD X0,Y1\nVMOVSS X1,(DX)", + "unknown merge": "VMOVUPS (DI),X0\nKMOVQ AX,K1\nVPMOVZXBD X0,K1,Y1\nVMOVSS X1,(DX)", + "unknown mask": "VMOVUPS (DI),X0\nVPMOVZXBD.Z X0,K1,Y1\nVMOVSS X1,(DX)", + "pointer mask": "VMOVUPS (DI),X0\nKMOVQ DI,K1\nVPMOVZXBD.Z X0,K1,Y1\nVMOVSS X1,(DX)", + "unknown base": "VPMOVZXBD (BX),Y1\nVMOVSS X1,(DX)", + "private stack": "VPMOVZXBD (SP),Y1\nVMOVSS X1,(DX)", + "pointer FP data": "VPMOVZXBD a+0(FP),Y1\nVMOVSS X1,(DX)", + "overread length": "VPMOVZXBW (CX),Y1\nVMOVSS X1,(DX)", + "zero mask envelope": "MOVL $0,AX\nKMOVQ AX,K1\nVPMOVZXBW.Z (DX),K1,Y1\nVMOVSS X1,(DX)", + "source write": "VPMOVZXBD (DI),Y1\nVMOVSS X1,(DI)", + "wide result": "VPMOVZXBD (DI),Y1\nVMOVUPS Y1,(DX)", + "undefined CFG join": "CMPQ AX,$0\nJE joined\nVMOVUPS (DI),X0\njoined:\nVPMOVZXBD X0,Y1\nVMOVSS X1,(DX)", + "unknown upper alias": "VMOVUPS (DI),X0\nPMOVZXBD X0,X1\nVPMOVZXBW Y1,Z2\nVMOVSS X2,(DX)", + "signed not inherited": "VMOVUPS (DI),X0\nVPMOVSXBD X0,Y1\nVMOVSS X1,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + // The upper-alias negative must observe an unknown part, not the + // correctly initialized low X bytes of the first widening result. + if name == "unknown upper alias" { + assembly = strings.Replace(assembly, "VMOVSS X2,(DX)", "VEXTRACTI128 $1,Y1,X2\nVMOVSS X2,(DX)", 1) + } + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved zero-extension acquired caller continuation: %v", err) + } + }) + } +} + +func TestX86GoCallerPackedZeroExtendIndependentByteKnowledge(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, vector := range []bool{false, true} { + for _, width := range []int{16, 32, 64} { + if !vector && width != 16 { + continue + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + source := "X0" + if width*ratio.input/ratio.output > 16 { + source = "Y0" + } + for _, destination := range []int{0, 1} { + for mode := 0; mode < 3; mode++ { + if !vector && mode != 0 { + continue + } + name, mask := "PMOVZX"+ratio.suffix, "" + if vector { + name = "V" + name + } + if mode != 0 { + mask = "K1," + if mode == 2 { + name += ".Z" + } + } + row := fmt.Sprintf("%s %s,%s%s%d", name, source, mask, bank, destination) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedZeroExtendProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for seed := 0; seed < 64; seed++ { + input, old := uint64(seed)*0x391072d5e64b8a0f, uint64(seed+7)*0x6a89e14f30725bc1 + initial := map[Reg]uint64{"Z0": input, "Z1": old} + before := initial[Reg(fmt.Sprintf("Z%d", destination))] + result := x86ResultState{flags: x86ByteFlagsAll} + result.writeVector(0, 64, input, false) + result.writeVector(1, 64, old, false) + result.masks[1] = true + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") { + t.Fatal("independent result observer lacks widening byte effects", row) + } + state := x86SourceStackState{vectors: initial, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedZeroExtendStep(&ins, form, &state, nil) { + t.Fatal("register byte effect refused") + } + var want uint64 + for output := 0; output < 64; output++ { + known := false + if output >= width { + known = vector || before>>uint(output)&1 != 0 + } else { + part := output % ratio.output + known = part >= ratio.input || input>>uint(output/ratio.output*ratio.input+part)&1 != 0 + if mode == 1 { + known = known && before>>uint(output)&1 != 0 + } + } + if known { + want |= uint64(1) << uint(output) + } + } + if got := state.vectors[Reg(fmt.Sprintf("Z%d", destination))]; got != want { + t.Fatalf("%s seed%d: defined bytes %#x want %#x", row, seed, got, want) + } + if result.vectorBytes(destination) != want || result.flags != x86ByteFlagsAll || + result.vectorBytes(1-destination) != map[int]uint64{0: input, 1: old}[1-destination] { + t.Fatalf("%s seed%d: result bytes %#x want %#x, or changed another register/FLAGS", row, seed, result.vectorBytes(destination), want) + } + if state.vectors[Reg(fmt.Sprintf("Z%d", 1-destination))] != map[int]uint64{0: input, 1: old}[1-destination] { + t.Fatal("widening changed another SIMD register") + } + } + } + } + } + } + } +} diff --git a/amd64_go_downloaded_frame_imports.go b/amd64_go_downloaded_frame_imports.go new file mode 100644 index 000000000..5ee3eda1b --- /dev/null +++ b/amd64_go_downloaded_frame_imports.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "io" + "path/filepath" + + "github.com/xgo-dev/plan9asm/internal/golegacymod" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" +) + +// The frame factory independently observes exact dependency downloads. The +// outer package query and caller-supplied types cannot supply these origins. +func (source *GoAMD64FrameSource) captureDownloadedDeclarationOrigins(ctx context.Context, output []byte) error { + if source.downloadedModule == nil { + return nil + } + if len(output) > 16<<20 { + return goAMD64FrameContext("declaration import graph exceeds its bound") + } + if err := source.verifyDownloadedOrigin(); err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(output)) + for count := 0; ; count++ { + var selected goAMD64FrameSelection + if err := decoder.Decode(&selected); err == io.EOF { + break + } else if err != nil { + return err + } + if count >= 4096 { + return goAMD64FrameContext("downloaded declaration import graph exceeds its count bound") + } + if selected.Standard { + continue // The SDK source/export consumer validates this role. + } + actual := selected.Module + if actual == nil || len(actual.Replace) != 0 && string(actual.Replace) != "null" { + return goAMD64FrameContext("downloaded dependency has an unbound module/replacement role") + } + if !actual.Main { + key := actual.Path + "@" + actual.Version + if _, exists := source.downloadedImports[key]; !exists { + if source.ready { + return goAMD64FrameContext("actual declaration import graph acquired a new module") + } + original, err := source.observeDownloadedModule(ctx, source.downloadedOrigin, + goAMD64DownloadedIdentity{module: actual.Path, version: actual.Version}, source.env["GOTMPDIR"]) + if err != nil { + return err + } + if source.downloadedImports == nil { + source.downloadedImports = make(map[string]gooriginalsrc.Module) + } + source.downloadedImports[key] = original + } + } + if _, err := source.downloadedDeclarationDirectory(selected); err != nil { + return err + } + } + return source.verifyDownloadedOrigin() +} + +func (source *GoAMD64FrameSource) downloadedDeclarationDirectory(selected goAMD64FrameSelection) (string, error) { + actual := selected.Module + if source.downloadedOrigin == nil || source.downloadedModule == nil || selected.Standard || actual == nil || + len(actual.Replace) != 0 && string(actual.Replace) != "null" { + return "", goAMD64FrameContext("foreign declaration dependency needs authenticated original handoff") + } + original := *source.downloadedModule + version := "" // Go's actual cache-CWD main role is not its download version. + if !actual.Main { + var found bool + original, found = source.downloadedImports[actual.Path+"@"+actual.Version] + if !found { + return "", goAMD64FrameContext("declaration dependency lacks its exact original download") + } + version = original.SourceVersion + } + root, rootErr := filepath.EvalSymlinks(actual.Dir) + metadata, metadataErr := filepath.EvalSymlinks(actual.GoMod) + if rootErr != nil || metadataErr != nil || root != original.Root || + actual.Path != original.ModulePath || actual.Version != version || + actual.GoVersion != gooriginalsrc.SelectedGoVersion(original.GoVersion, actual.Main) || + metadata != original.GoMod && metadata != filepath.Join(original.Root, "go.mod") { + return "", goAMD64FrameContext("declaration dependency differs from its original module metadata") + } + if original.GoModOrigin == golegacymod.ProxyGoModProtocol && (actual.Main || metadata != original.GoMod) { + return "", goAMD64FrameContext("legacy proxy metadata requires its exact dependency role") + } + directory, err := filepath.EvalSymlinks(selected.Dir) + if err != nil { + return "", err + } + path, err := goAMD64DownloadedPackagePath(original, directory) + if err != nil || path != selected.ImportPath { + return "", goAMD64FrameContext("declaration dependency escaped its original package namespace") + } + return directory, nil +} diff --git a/amd64_go_downloaded_frame_imports_test.go b/amd64_go_downloaded_frame_imports_test.go new file mode 100644 index 000000000..da780be22 --- /dev/null +++ b/amd64_go_downloaded_frame_imports_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" +) + +func TestGoAMD64DownloadedDeclarationMetadataRoles(t *testing.T) { + root := t.TempDir() + primary := gooriginalsrc.Module{Root: filepath.Join(root, "main"), ModulePath: "example.invalid/main", GoVersion: "1.20"} + foreign := gooriginalsrc.Module{Root: filepath.Join(root, "dependency"), ModulePath: "example.invalid/dependency", + SourceModule: "example.invalid/dependency", SourceVersion: "v1.0.0", GoVersion: "1.20"} + for _, original := range []*gooriginalsrc.Module{&primary, &foreign} { + if err := os.Mkdir(original.Root, 0700); err != nil { + t.Fatal(err) + } + canonical, err := filepath.EvalSymlinks(original.Root) + if err != nil { + t.Fatal(err) + } + original.Root = canonical + original.GoMod = filepath.Join(canonical, "go.mod") + if err := os.WriteFile(original.GoMod, []byte("module "+original.ModulePath+"\ngo 1.20\n"), 0600); err != nil { + t.Fatal(err) + } + } + for _, axis := range []string{"foreign", "primary", "unknown version", "namespace", "language", "main role", "root", + "metadata", "package directory", "package path", "replacement", "standard", "missing module", "missing owner"} { + t.Run(axis, func(t *testing.T) { + original, version, main := foreign, foreign.SourceVersion, false + if axis == "primary" { + original, version, main = primary, "", true + } + data, err := json.Marshal(map[string]any{ + "Dir": original.Root, "ImportPath": original.ModulePath, + "Module": map[string]any{"Path": original.ModulePath, "Version": version, "Main": main, + "Dir": original.Root, "GoMod": original.GoMod, "GoVersion": original.GoVersion}, + }) + if err != nil { + t.Fatal(err) + } + var selected goAMD64FrameSelection + if err := json.Unmarshal(data, &selected); err != nil { + t.Fatal(err) + } + // This is metadata predicate data, deliberately not a sealed source + // or a real OriginSet. It cannot Verify or supply frame authority. + source := &GoAMD64FrameSource{downloadedOrigin: &gooriginalsrc.OriginSet{}, downloadedModule: &primary, + downloadedImports: map[string]gooriginalsrc.Module{foreign.ModulePath + "@" + foreign.SourceVersion: foreign}} + switch axis { + case "unknown version": + selected.Module.Version = "v2.0.0" + case "namespace": + selected.Module.Path = primary.ModulePath + case "language": + selected.Module.GoVersion = "1.21" + case "main role": + selected.Module.Main = true + case "root": + selected.Module.Dir = primary.Root + case "metadata": + selected.Module.GoMod = primary.GoMod + case "package directory": + selected.Dir = primary.Root + case "package path": + selected.ImportPath += "/unobserved" + case "replacement": + selected.Module.Replace = json.RawMessage(`{}`) + case "standard": + selected.Standard = true + case "missing module": + selected.Module = nil + case "missing owner": + source.downloadedOrigin = nil + } + directory, err := source.downloadedDeclarationDirectory(selected) + if axis == "foreign" || axis == "primary" { + if err != nil || directory != original.Root { + t.Fatalf("consistent actual role lost metadata identity: %q, %v", directory, err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || directory != "" { + t.Fatalf("changed original module role was accepted: %q, %v", directory, err) + } + if err := source.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("metadata predicate manufactured actual frame authority", err) + } + }) + } +} diff --git a/amd64_go_downloaded_frame_source.go b/amd64_go_downloaded_frame_source.go new file mode 100644 index 000000000..8fa4a9e54 --- /dev/null +++ b/amd64_go_downloaded_frame_source.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "time" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/golegacymod" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" + "golang.org/x/mod/module" +) + +// GoAMD64DownloadedFrameCaptureRequest names original exact cache inputs. It +// carries no observation, checker, object, signature or permission callback. +type GoAMD64DownloadedFrameCaptureRequest struct { + Original GoAMD64FrameCaptureRequest + SourceModule, SourceVersion string +} + +type goAMD64DownloadedIdentity struct { + module, version string +} + +// CaptureGoAMD64DownloadedFrameSource independently executes stock Go and +// closes the original ZIP, extracted members, declarations and whole objects. +// Downloaded roles retain their primary module and independently observed +// original import graph. Capture itself grants no memory permission. +func CaptureGoAMD64DownloadedFrameSource(ctx context.Context, pkg GoPackage, request GoAMD64DownloadedFrameCaptureRequest) (*GoAMD64FrameSource, error) { + if module.Check(request.SourceModule, request.SourceVersion) != nil || + module.CanonicalVersion(request.SourceVersion) != request.SourceVersion { + return nil, goAMD64FrameContext("downloaded capture requires a canonical exact source identity") + } + return captureGoAMD64FrameSource(ctx, pkg, request.Original, + &goAMD64DownloadedIdentity{module: request.SourceModule, version: request.SourceVersion}) +} + +func (source *GoAMD64FrameSource) captureDownloadedOrigin(ctx context.Context, exact goAMD64DownloadedIdentity, temporary string) (result error) { + cache, err := filepath.EvalSymlinks(source.env["GOMODCACHE"]) + if err != nil { + return err + } + origins, err := gooriginalsrc.NewOriginSet(cache, source.request.GoBinary) + if err != nil { + return err + } + original, err := source.observeDownloadedModule(ctx, origins, exact, temporary) + if err != nil { + return err + } + if _, err := goAMD64DownloadedPackagePath(original, source.request.Directory); err != nil { + return err + } + source.downloadedOrigin, source.downloadedModule = origins, &original + source.downloadedImports = map[string]gooriginalsrc.Module{original.SourceModule + "@" + original.SourceVersion: original} + return source.verifyDownloadedOrigin() +} + +func (source *GoAMD64FrameSource) legacyDownloadedModule() bool { + return source.downloadedModule != nil && source.downloadedModule.GoModOrigin == golegacymod.ProxyGoModProtocol +} + +func (source *GoAMD64FrameSource) runDownloadedList(ctx context.Context, args []string) ([]byte, error) { + if source.numericalBinding != nil { + return nil, goAMD64FrameContext("legacy dependency query needs its separate numerical-command binding") + } + original := source.downloadedModule + path, err := goAMD64DownloadedPackagePath(*original, source.request.Directory) + if err != nil { + return nil, err + } + env, err := gooriginalsrc.DependencyQueryEnvironment(source.request.Environment) + if err != nil { + return nil, err + } + arguments := append([]string(nil), args...) + for index, argument := range arguments { + switch argument { + case ".": + arguments[index] = path + } + } + return gooriginalsrc.RunDependencyList(ctx, source.env["GOTMPDIR"], + original.SourceModule, original.SourceVersion, arguments, func(directory string, arguments []string) ([]byte, []byte, error) { + observed, err := goasmcmd.Observe(ctx, directory, env, source.request.GoBinary, arguments...) + return observed.Stdout, observed.Stderr, err + }) +} + +func (source *GoAMD64FrameSource) observeDownloadedModule(ctx context.Context, origins *gooriginalsrc.OriginSet, + exact goAMD64DownloadedIdentity, temporary string, +) (_ gooriginalsrc.Module, result error) { + if module.Check(exact.module, exact.version) != nil || module.CanonicalVersion(exact.version) != exact.version { + return gooriginalsrc.Module{}, goAMD64FrameContext("downloaded dependency requires a canonical exact source identity") + } + boundary, err := gooriginalsrc.NewDownloadBoundary(temporary, "plan9asm-frame-download-", "example.invalid/plan9asm-frame-download") + if err != nil { + return gooriginalsrc.Module{}, err + } + defer func() { + result = errors.Join(result, boundary.Close()) + }() + // This private command boundary is not a copied main/source module. Its + // sole purpose is to prevent stock Go mutating an ancestor go.mod/go.sum. + if err := boundary.Prepare(); err != nil { + return gooriginalsrc.Module{}, err + } + commandContext, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + data, err := goasmcmd.Run(commandContext, boundary.Directory(), source.request.Environment, + source.request.GoBinary, "mod", "download", "-json", exact.module+"@"+exact.version) + err = errors.Join(err, commandContext.Err()) + if err != nil { + return gooriginalsrc.Module{}, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var receipt gooriginalsrc.DownloadReceipt + if err := decoder.Decode(&receipt); err != nil { + return gooriginalsrc.Module{}, err + } + var extra any + if decoder.Decode(&extra) != io.EOF { + return gooriginalsrc.Module{}, goAMD64FrameContext("actual exact download receipt has incomplete or additional JSON") + } + var proxyMetadata *golegacymod.ProxyGoModProof + if _, err := os.Lstat(filepath.Join(receipt.Dir, "go.mod")); os.IsNotExist(err) { + // Source capture pins GOPROXY=off, but authenticated public checksum + // metadata remains independently readable. This neither creates a ZIP + // member nor turns a legacy dependency into an original main module. + proxyMetadata, _, err = golegacymod.CaptureCachedProxyGoMod(commandContext, + source.env["GOMODCACHE"], "off", exact.module, exact.version, + receipt.Sum, receipt.GoModSum, receipt.GoMod, nil) + if err != nil { + return gooriginalsrc.Module{}, err + } + } + original, err := origins.ObserveLegacy(ctx, exact.module, exact.version, receipt, proxyMetadata) + if err != nil { + return gooriginalsrc.Module{}, errors.Join(goAMD64FrameContext("exact original archive/source is not closed"), err) + } + return original, nil +} + +func (source *GoAMD64FrameSource) verifyDownloadedOrigin() error { + if source.downloadedOrigin == nil && source.downloadedModule == nil { + return nil + } + if source.downloadedOrigin == nil || source.downloadedModule == nil { + return goAMD64FrameContext("missing exact original download/source role") + } + if _, err := goAMD64DownloadedPackagePath(*source.downloadedModule, source.request.Directory); err != nil { + return err + } + if err := source.downloadedOrigin.Verify(); err != nil { + return errors.Join(goAMD64FrameContext("exact original source was changed or revoked"), err) + } + return nil +} + +// The original archive namespace and the declared Go package namespace are +// distinct. Derive the latter only from the guarded module's own go.mod and +// the actual canonical package directory, never from an import-path prefix. +func goAMD64DownloadedPackagePath(original gooriginalsrc.Module, directory string) (string, error) { + relative, err := filepath.Rel(original.Root, directory) + if err != nil || !filepath.IsAbs(original.Root) || !filepath.IsAbs(directory) || !filepath.IsLocal(relative) { + return "", goAMD64FrameContext("downloaded package must remain inside its actual exact primary cache root") + } + path := original.ModulePath + if relative != "." { + path += "/" + filepath.ToSlash(relative) + } + if module.CheckImportPath(path) != nil { + return "", goAMD64FrameContext("downloaded package has an invalid original module-relative identity") + } + return path, nil +} diff --git a/amd64_go_downloaded_package_path_test.go b/amd64_go_downloaded_package_path_test.go new file mode 100644 index 000000000..6ce9dcbd1 --- /dev/null +++ b/amd64_go_downloaded_package_path_test.go @@ -0,0 +1,37 @@ +package plan9asm + +import ( + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" +) + +func TestGoAMD64DownloadedPackageIdentity(t *testing.T) { + root := filepath.Join(t.TempDir(), "archive.example", "mirror@v1.0.0") + original := gooriginalsrc.Module{Root: root, ModulePath: "example.invalid/declared"} + for _, test := range []struct { + name, directory, want string + }{ + {"root alias", root, "example.invalid/declared"}, + {"nested alias", filepath.Join(root, "internal", "kernel"), "example.invalid/declared/internal/kernel"}, + {"parent", filepath.Dir(root), ""}, + {"prefix sibling", root + "-other", ""}, + {"traversal", filepath.Join(root, "..", "other"), ""}, + {"relative", "internal/kernel", ""}, + {"invalid import", filepath.Join(root, "bad path"), ""}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := goAMD64DownloadedPackagePath(original, test.directory) + if test.want == "" { + if err == nil || got != "" { + t.Fatalf("unbound path acquired an original package identity: %q, %v", got, err) + } + return + } + if err != nil || got != test.want { + t.Fatalf("original declared package = %q, %v; want %q", got, err, test.want) + } + }) + } +} diff --git a/amd64_go_frame_complete_source_test.go b/amd64_go_frame_complete_source_test.go new file mode 100644 index 000000000..45cd487a5 --- /dev/null +++ b/amd64_go_frame_complete_source_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Original Go compilation is the acceptance oracle. Additional ordinary Go +// declarations do not themselves grant caller memory or fixed-local backing. +func TestGoAMD64FrameSourceCompleteOrdinaryGoDeclarations(t *testing.T) { + // Reuse only this root's owned compiler cache, not source identities. + // Every case still executes actual Go and consumes its selected export. + t.Setenv("GOCACHE", t.TempDir()) + for _, fixture := range []struct { + name, imports, declarations string + }{ + {"body", "", "func Reference(a uint64) uint64 { return a + 1 }"}, + {"values", "", "const Bias uint64 = 1\nvar Counter uint64"}, + {"unused-type", "", "type Record struct { Bytes []byte; Value uint64 }"}, + {"unused-interface", "", "type XOF interface { Read([]byte) (int, error); Clone() XOF }"}, + {"unused-function", "", "type Callback func([]byte) error"}, + {"unused-map", "", "type Cache map[string][]byte"}, + {"unused-channel", "", "type Queue chan []byte"}, + {"unused-float", "", "type Scalar float64"}, + {"unused-recursive", "", "type Node struct { Next *Node; Value uint64 }"}, + {"unused-generic", "", "type Box[T any] struct { Value T }"}, + {"unused-interface-alias", "", "type XOF = interface { Read([]byte) (int, error) }"}, + {"sdk-import", "import \"math/bits\"", "func Reference(a uint64) uint64 { return bits.RotateLeft64(a, 1) }"}, + {"unrelated-body-transport", "", "func Reference(a float64) float64 { return a + 1 }"}, + {"combined", "import \"math/bits\"", "const Bias = 1\nvar Counter uint64\nfunc Reference(a uint64) uint64 { return bits.RotateLeft64(a, Bias) }"}, + } { + t.Run(fixture.name, func(t *testing.T) { + _, request := goAMD64FrameSourceFixture(t) + text := "package framefixture\n" + fixture.imports + "\nfunc Value(a uint64) uint64\n" + fixture.declarations + "\n" + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte(text), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOTOOLCHAIN=local", + "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOPROXY=off") + command := exec.CommandContext(ctx, request.GoBinary, + "list", "-deps", "-export", "-compiled", "-json", "-mod=readonly", ".") + command.Dir, command.Env = request.Directory, env + output, diagnostics, err := runtimeOracleMachineOutput(ctx, command) + if err != nil { + t.Fatalf("original Go rejected complete ordinary source: %v\n%s\n%s", err, output, diagnostics) + } + var selected goAMD64FrameSelection + exports := make(map[string]string) + decoder := json.NewDecoder(bytes.NewReader(output)) + for { + var member goAMD64FrameSelection + if err := decoder.Decode(&member); err == io.EOF { + break + } else if err != nil || member.ImportPath == "" || len(member.Error)+len(member.DepsErrors) != 0 { + t.Fatalf("original Go compiler graph is incomplete: %+v: %v", member, err) + } + exports[member.ImportPath] = member.Export + if member.Module != nil && member.Module.Main { + if selected.ImportPath != "" { + t.Fatal("original Go selected more than one owned package") + } + selected = member + } + } + if selected.ImportPath != "example.invalid/framefixture" || len(selected.CompiledGoFiles) != 1 || len(selected.SFiles) != 1 { + t.Fatalf("original Go omitted the complete Go/ASM source pair: %+v", selected) + } + export, err := os.ReadFile(selected.Export) + if err != nil || len(export) == 0 { + t.Fatalf("original Go omitted compiled package/assembly export: %v", err) + } + t.Logf("ORIGINAL_GO_COMPLETE_FRAME_SOURCE role=%s go_sha=%x export_sha=%x", fixture.name, + sha256.Sum256([]byte(text)), sha256.Sum256(export)) + set := token.NewFileSet() + syntax, err := parser.ParseFile(set, filepath.Join(request.Directory, "value.go"), text, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + actualImporter := importer.ForCompiler(set, "gc", func(path string) (io.ReadCloser, error) { + return os.Open(exports[path]) + }) + typed, err := (&types.Config{Importer: actualImporter, Sizes: types.SizesFor("gc", "amd64")}).Check( + selected.ImportPath, set, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + pkg := GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{syntax}} + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatalf("Go-accepted ordinary declarations lost original frame observation: %v", err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + function, found := source.functions[pkg.Path+".Value"] + if !found || function.locals != 168 || function.args != 16 || function.transport != goAMD64FrameNumericTransport { + t.Fatalf("complete source changed the independently captured numeric ASM frame: %+v", function) + } + if len(source.functions) != 1 || len(source.declarations) != 1 { + t.Fatal("unrelated ordinary Go declarations became ASM frame capabilities") + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/amd64_go_frame_data.go b/amd64_go_frame_data.go new file mode 100644 index 000000000..d677aa36b --- /dev/null +++ b/amd64_go_frame_data.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "path/filepath" + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "github.com/xgo-dev/plan9asm/internal/gotoolchain" +) + +func (source *GoAMD64FrameSource) frameDataObject(expanded string, object []byte) (*gostaticstorage.AMD64AssemblyData, error) { + if !gotoolchain.AuditedGo127(source.env["GOVERSION"]) || source.numericalBinding != nil { + return nil, goAMD64FrameContext("mixed DATA needs its independent actual Go object producer") + } + builtin, err := source.read(filepath.Join(source.env["GOROOT"], "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + return nil, err + } + observed, err := gostaticstorage.ObserveAMD64AssemblyData([]byte(expanded), object, builtin, + source.selection.ImportPath, source.request.Target) + if err != nil { + return nil, goAMD64FrameContext("complete original DATA/source object: %v", err) + } + return &observed, nil +} + +// Consume exact complete DATA blocks, never arbitrary listing lines. TEXT and +// Windows metadata remain in the original order for their independent parser. +// This observation grants no permission to use a DATA address or load its bytes. +func goAMD64FrameDataListing(listing []byte, observed *gostaticstorage.AMD64AssemblyData) ([]byte, error) { + if observed == nil || len(observed.Layouts) == 0 || len(listing) == 0 || listing[len(listing)-1] != '\n' { + return nil, goAMD64FrameContext("mixed DATA listing lacks a complete object/source inventory") + } + blocks := make(map[string][]string, len(observed.Layouts)) + var headers []string + for _, layout := range observed.Layouts { + block, err := goAMD64FrameDataBlock(layout) + if err != nil { + return nil, err + } + if _, duplicate := blocks[block[0]]; duplicate { + return nil, goAMD64FrameContext("mixed DATA listing has an ambiguous complete symbol header") + } + blocks[block[0]] = block + headers = append(headers, block[0]) + } + lines := strings.Split(string(listing), "\n") + var result strings.Builder + next := 0 + for index := 0; index < len(lines)-1; index++ { + block, found := blocks[lines[index]] + if !found { + result.WriteString(lines[index] + "\n") + continue + } + if next >= len(headers) || block[0] != headers[next] || + index+len(block) > len(lines)-1 || !reflect.DeepEqual(lines[index:index+len(block)], block) { + return nil, goAMD64FrameContext("mixed DATA listing bytes, extent or ordered relocations differ") + } + delete(blocks, block[0]) + next++ + index += len(block) - 1 + } + if len(blocks) != 0 { + return nil, goAMD64FrameContext("mixed DATA listing omits an original definition") + } + return []byte(result.String()), nil +} + +func goAMD64FrameDataBlock(layout gostaticstorage.ASMGlobalLayout) ([]string, error) { + definition := layout.Definition + kinds := map[uint8]string{3: "SRODATA", 4: "SRODATAFIPS", 5: "SNOPTRDATA", 6: "SNOPTRDATAFIPS", + 7: "SDATA", 8: "SDATAFIPS", 9: "SBSS", 10: "SNOPTRBSS", 11: "STLSBSS"} + kind := kinds[definition.Kind] + if kind == "" || definition.Name == "" || definition.Name != layout.Name || definition.ABI != layout.ABI { + return nil, goAMD64FrameContext("mixed DATA header lacks its exact source/object identity") + } + header := definition.Name + " " + kind + " " + if layout.Static { + header += "static " + } + if definition.Flags&1 != 0 { + header += "dupok " + } + // Go's GloblPos consumes DUPOK/RODATA/NOPTR/TLSBSS only; unrelated + // TEXT flags, including CFunc, do not change the DATA listing header. + header += fmt.Sprintf("size=%d align=%#x", definition.Size, definition.Align) + block := []string{header} + for offset := 0; offset < len(definition.Data); offset += 16 { + var row strings.Builder + fmt.Fprintf(&row, "\t%#04x", offset) + end := offset + 16 + if end > len(definition.Data) { + end = len(definition.Data) + } + for _, value := range definition.Data[offset:end] { + fmt.Fprintf(&row, " %02x", value) + } + row.WriteString(strings.Repeat(" ", offset+16-end) + " ") + for _, value := range definition.Data[offset:end] { + if value < ' ' || value > '~' { + value = '.' + } + row.WriteByte(value) + } + block = append(block, row.String()) + } + for _, relocation := range definition.Relocations { + if relocation.Type != 1 || !relocation.Target.ABIKnown { + return nil, goAMD64FrameContext("mixed DATA initializer has an unbound relocation") + } + block = append(block, fmt.Sprintf("\trel %d+%d t=R_ADDR %s+%d", + relocation.Offset, relocation.Size, relocation.Target.Name, relocation.Add)) + } + return block, nil +} + +func goAMD64FrameDataRelocationLine(relocation gostaticstorage.TextRelocation, observed *gostaticstorage.AMD64AssemblyData) string { + if relocation.Type != 14 || relocation.Size != 4 || !relocation.Target.ABIKnown { + return "" + } + packageIndices := map[gostaticstorage.TextSymbolNamespace]uint32{ + gostaticstorage.TextSymbolSelf: 2147483643, + gostaticstorage.TextSymbolHashed64: 2147483646, + gostaticstorage.TextSymbolHashed: 2147483645, + gostaticstorage.TextSymbolNonpkgDefinition: 2147483647, + } + for _, layout := range observed.Layouts { + definition := layout.Definition + if definition.Name == relocation.Target.Name && definition.ABI == relocation.Target.ABI && + packageIndices[definition.Namespace] == relocation.Target.PackageIndex && + definition.Relative == relocation.Target.SymbolIndex { + return fmt.Sprintf("rel %d+%d t=R_PCREL %s+%d", + relocation.Offset, relocation.Size, relocation.Target.Name, relocation.Add) + } + } + return "" +} + +func goAMD64FrameDataTextJoin(functions map[string]goAMD64FrameFunction, observed *gostaticstorage.AMD64AssemblyData, + directory string, origins map[string]int) error { + if len(functions) != len(observed.Member.Texts) { + return goAMD64FrameContext("mixed DATA object/listing complete TEXT inventories differ") + } + for _, text := range observed.Member.Texts { + fn, found := functions[text.Name] + if !found || text.ABI != 0 || text.Kind != 1 || text.Flags != 16 || text.Flags2 != 0 || !bytes.Equal(text.Code, fn.code) { + return goAMD64FrameContext("mixed DATA object/listing TEXT ABI, flags or complete bytes differ") + } + if err := goAMD64WindowsUnwindSourcePCs(text, fn, directory, origins); err != nil { + return err + } + } + return nil +} diff --git a/amd64_go_frame_data_boundaries_go127_test.go b/amd64_go_frame_data_boundaries_go127_test.go new file mode 100644 index 000000000..fc78ed9dd --- /dev/null +++ b/amd64_go_frame_data_boundaries_go127_test.go @@ -0,0 +1,53 @@ +//go:build go1.27 + +package plan9asm + +import ( + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestGoAMD64FrameDataWholeMaskAESBoundaries(t *testing.T) { + var rows []string + for _, op := range []string{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + rows = append(rows, op+" literal<>+1(SB),K1", op+" K1,literal<>+1(SB)") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, op := range []string{"VAESENC", "VAESENCLAST", "VAESDEC", "VAESDECLAST"} { + rows = append(rows, op+" literal<>+1(SB),"+bank+"0,"+bank+"1") + if bank != "Z" { + // Force the distinct EVEX.128/256 rows, not only VEX. + rows = append(rows, op+" literal<>+1(SB),"+bank+"0,"+bank+"16") + } + } + } + rows = append(rows, "VAESIMC literal<>+1(SB),X0", + "VAESKEYGENASSIST $255,literal<>+1(SB),X0", "VAESKEYGENASSIST $-128,literal<>+1(SB),X0") + for _, row := range rows { + t.Run(row, func(t *testing.T) { + code, pcs := x86NumericalBoundaryGoBytes(t, "amd64", row+"\nRET") + if len(pcs) != 2 { + t.Fatal("independent original instruction inventory changed", pcs) + } + decode := decodedX86MaskMoveInstruction + if strings.HasPrefix(row, "VAES") { + decode = decodedX86AESInstruction + } + if _, _, known, err := decode(code, 64); !known || err == nil || !strings.Contains(err.Error(), "RIP-relative") { + t.Fatal("ordinary raw decoding borrowed a physical RIP source", known, err) + } + boundaries, scalar, err := x86NumericalNativeBoundaries(code, 64) + if err != nil || len(boundaries) != 2 || boundaries[0].Length != int(pcs[1]) || + len(scalar) != 1 || scalar[pcs[1]].Op != x86asm.RET { + t.Fatal("whole original mask/AES boundary lost physical RIP", boundaries, scalar, err) + } + for cut := 1; cut < int(pcs[1]); cut++ { + if values, _, err := x86NumericalNativeBoundaries(code[:cut], 64); err == nil || values != nil { + t.Fatal("truncated original mask/AES acquired a boundary", cut, err) + } + } + }) + } +} diff --git a/amd64_go_frame_data_go127_test.go b/amd64_go_frame_data_go127_test.go new file mode 100644 index 000000000..19a6919bf --- /dev/null +++ b/amd64_go_frame_data_go127_test.go @@ -0,0 +1,185 @@ +//go:build go1.27 + +package plan9asm + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +// DATA coexistence is an original-object observation, not a new memory, +// address, native-entry or fixed-local permission. +func TestGoAMD64FrameSourceMixedDataOriginalObjects(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + for _, used := range []bool{false, true} { + name := "unused" + if used { + name = "referenced" + } + t.Run(target+"/"+name, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + assembly := "DATA literal<>+0(SB)/8,$0x102030405060708\n" + + "DATA literal<>+16(SB)/4,$\"abcd\"\n" + + "GLOBL literal<>(SB),8,$32\n" + + "DATA mutable<>+0(SB)/8,$0x55667788\n" + + "GLOBL mutable<>(SB),16,$8\n" + + "GLOBL zero<>(SB),16,$24\n" + + "DATA pointer<>+0(SB)/8,$literal<>+8(SB)\n" + + "GLOBL pointer<>(SB),8,$8\n" + + "TEXT ·Value(SB),4,$160-16\n" + + "MOVQ a+0(FP),AX\nMOVQ AX,16(SP)\n" + if used { + assembly += "MOVQ literal<>+8(SB),CX\n" + } + assembly += "MOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS="+strings.Split(target, "/")[0], "GOARCH=amd64", "CGO_ENABLED=0", + "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=") + listing, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, "-o", filepath.Join(t.TempDir(), "original.o"), request.AssemblyFile) + if err != nil { + t.Fatalf("independent original Go mixed TEXT/DATA control: %v\n%s", err, listing) + } + t.Logf("original Go listing:\n%s", listing) + source := requireGoAMD64FrameCapture(t, pkg, request) + if len(source.functions) != 1 || source.functions[pkg.Path+".Value"].locals != 168 { + t.Fatal("mixed DATA lost the original complete frame inventory") + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + goAMD64FrameMixedDataJoinGuards(t, pkg, source, used) + changed := strings.Replace(assembly, "0x102030405060708", "0x102030405060709", 1) + path := filepath.Join(request.Directory, request.AssemblyFile) + if err := os.WriteFile(path, []byte(changed), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed original DATA retained the frame observation", err) + } + if err := os.WriteFile(path, []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("restored DATA revived a revoked frame observation", err) + } + }) + } + } +} + +func goAMD64FrameMixedDataJoinGuards(t *testing.T, pkg GoPackage, source *GoAMD64FrameSource, used bool) { + t.Helper() + objectPath := filepath.Join(source.owned, source.request.AssemblyFile+".o") + object, err := source.read(objectPath) + if err != nil { + t.Fatal(err) + } + listing, err := source.read(objectPath + ".listing") + if err != nil { + t.Fatal(err) + } + original, err := source.read(filepath.Join(source.request.Directory, source.request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(source.request.AssemblyFile, original) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + observed, err := source.frameDataObject(expanded, object) + if err != nil { + t.Fatal(err) + } + var unwind *gostaticstorage.WindowsAMD64AssemblyUnwind + if source.request.Target == "windows/amd64" { + value, err := source.observeWindowsUnwind(object) + if err != nil { + t.Fatal(err) + } + unwind = &value + } + check := func(input []byte) error { + filtered, err := goAMD64FrameDataListing(input, observed) + if err != nil { + return err + } + functions, err := goAMD64FrameListingWithDataObject(filtered, file, pkg, source.origins, observed, unwind) + if err != nil { + return err + } + return goAMD64FrameDataTextJoin(functions, observed, source.request.Directory, source.origins) + } + if err := check(listing); err != nil { + t.Fatal("complete unmodified source/object/listing control", err) + } + changes := map[string][2]string{ + "DATA kind": {"literal SRODATA", "literal SDATA"}, + "DATA static": {"literal SRODATA static", "literal SRODATA"}, + "DATA dupok": {"literal SRODATA static", "literal SRODATA static dupok"}, + "DATA size": {"size=32 align=0x0", "size=33 align=0x0"}, + "DATA alignment": {"size=32 align=0x0", "size=32 align=0x8"}, + "DATA byte": {"08 07 06 05", "09 07 06 05"}, + "DATA ASCII": {"abcd\n", "abce\n"}, + "DATA offset": {"0x0010 61 62 63 64", "0x0011 61 62 63 64"}, + "DATA relocation": {"R_ADDR literal+8", "R_ADDR literal+16"}, + "DATA target": {"R_ADDR literal+8", "R_ADDR mutable+8"}, + "DATA BSS extent": {"zero SNOPTRBSS static size=24", "zero SNOPTRBSS static size=25"}, + "TEXT bytes": {"55 48 89 e5", "90 48 89 e5"}, + } + if used { + changes["TEXT relocation"] = [2]string{"R_PCREL literal+8", "R_PCREL literal+16"} + changes["TEXT target"] = [2]string{"R_PCREL literal+8", "R_PCREL mutable+8"} + } + for name, change := range changes { + t.Run(name, func(t *testing.T) { + changed := bytes.Replace(listing, []byte(change[0]), []byte(change[1]), 1) + if bytes.Equal(changed, listing) { + t.Fatal("listing mutation did not touch the independent Go control") + } + if err := check(changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed complete listing acquired an observation", err) + } + }) + } + for _, suffix := range []string{"UNKNOWN_NATIVE_RECORD\n", "\trel 0+8 t=R_ADDR literal+8\n", "zero SNOPTRBSS static size=24 align=0x0\n"} { + if err := check(append(bytes.Clone(listing), suffix...)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("extra DATA/listing record was ignored", err) + } + } + if used { + relocation := observed.Member.Texts[0].Relocations[0] + for _, mutate := range []func(*gostaticstorage.TextRelocation){ + func(r *gostaticstorage.TextRelocation) { r.Type = 1 }, + func(r *gostaticstorage.TextRelocation) { r.Size = 8 }, + func(r *gostaticstorage.TextRelocation) { r.Target.ABIKnown = false }, + func(r *gostaticstorage.TextRelocation) { r.Target.ABI = 1 }, + func(r *gostaticstorage.TextRelocation) { r.Target.PackageIndex = 2147483647 }, + func(r *gostaticstorage.TextRelocation) { r.Target.SymbolIndex++ }, + } { + changed := relocation + mutate(&changed) + if changed == relocation || goAMD64FrameDataRelocationLine(changed, observed) != "" { + t.Fatal("DATA relocation lost exact object-local identity", changed) + } + } + } +} diff --git a/amd64_go_frame_fixture_flags_test.go b/amd64_go_frame_fixture_flags_test.go new file mode 100644 index 000000000..2a495be80 --- /dev/null +++ b/amd64_go_frame_fixture_flags_test.go @@ -0,0 +1,34 @@ +package plan9asm + +import ( + "errors" + "os" + "testing" +) + +func TestGoAMD64FrameFixtureSeparatesHarnessBuildTags(t *testing.T) { + // The Windows LLVM harness needs this tag; the independent, pure-Go + // source fixture does not import LLVM and must have its own build role. + t.Setenv("GOFLAGS", "-tags=byollvm") + for _, target := range []string{"linux/amd64", "windows/amd64"} { + t.Run(target, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + if os.Getenv("GOFLAGS") != "-tags=byollvm" { + t.Fatal("fixture changed its parent harness environment") + } + // An explicit caller request is still nonordinary. This fixture + // separation cannot relax production profile/source validation. + request.Environment = append(request.Environment, "GOFLAGS=-tags=foreign") + if changed, err := CaptureGoAMD64FrameSource(ctx, pkg, request); changed != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("explicit foreign source profile was accepted", err) + } + }) + } +} diff --git a/amd64_go_frame_identity_test.go b/amd64_go_frame_identity_test.go new file mode 100644 index 000000000..6e3da0cc9 --- /dev/null +++ b/amd64_go_frame_identity_test.go @@ -0,0 +1,83 @@ +package plan9asm + +import ( + "context" + "errors" + "os" + "reflect" + "testing" +) + +func TestGoOriginalInputGuardCannotSupplyFrameAuthority(t *testing.T) { + request := GoAMD64FrameCaptureRequest{Environment: []string{"GOFLAGS="}} + first, second := newGoOriginalInputGuard(request), newGoOriginalInputGuard(request) + request.Environment[0] = "GOFLAGS=-overlay=foreign.json" + if first.request.Environment[0] != "GOFLAGS=" || second.request.Environment[0] != "GOFLAGS=" { + t.Fatal("query-only guard retained caller environment storage") + } + first.inputs["first-only"] = [32]byte{1} + if len(second.inputs) != 0 { + t.Fatal("independent query-only guards shared input maps") + } + for _, source := range []*GoAMD64FrameSource{first, second} { + if source.ready || source.owned != "" || len(source.functions) != 0 || len(source.declarations) != 0 || + source.downloadedOrigin != nil || source.downloadedModule != nil { + t.Fatal("input initialization created an object or original-source capability") + } + if !errors.Is(source.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatal("query-only guard verified as a frame") + } + if _, err := source.BindFixedLocal(context.Background(), GoPackage{}, nil, Options{}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("query-only guard admitted fixed-local memory", err) + } + if err := source.Close(); err != nil { + t.Fatal("original query guard could not close its empty resource role", err) + } + } +} + +func TestGoAMD64FrameSourceCopyCannotBorrowOrCleanOriginal(t *testing.T) { + for _, operation := range []string{"Verify", "Close"} { + t.Run(operation, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }() + if err := source.Verify(ctx); err != nil { + t.Fatal("actual original public factory did not verify", err) + } + owned := source.owned + before, err := os.Lstat(owned) + if err != nil { + t.Fatal(err) + } + // Deliberately model the public whole-value operation without hiding + // a mutex copy behind a constructor or serializing private fields. + copied := new(GoAMD64FrameSource) + reflect.ValueOf(copied).Elem().Set(reflect.ValueOf(source).Elem()) + if operation == "Verify" { + err = copied.Verify(ctx) + } else { + err = copied.Close() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("whole-value copied frame %s borrowed its original owner role: %v", operation, err) + } + after, err := os.Lstat(owned) + if err != nil || !os.SameFile(before, after) { + t.Error("copied frame operation removed or replaced original artifacts", err) + } + if err := source.Verify(ctx); err != nil { + t.Error("copied frame operation revoked its original object", err) + } + }) + } +} diff --git a/amd64_go_frame_macro_test.go b/amd64_go_frame_macro_test.go new file mode 100644 index 000000000..c1dba737d --- /dev/null +++ b/amd64_go_frame_macro_test.go @@ -0,0 +1,180 @@ +package plan9asm + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestGoAMD64FrameSourceConditionalMacrosMatchActualObjects(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for targetIndex, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(target+"/"+profile, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + request.Environment = append(request.Environment, "GOAMD64="+profile) + assembly := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + header := `#ifndef PROFILE_GUARD +#define PROFILE_GUARD +#ifdef GOAMD64_v2 +#define PROFILE_VALUE 7 +#else +#define PROFILE_VALUE 11 +#endif +#ifdef GOOS_linux +#define TARGET_VALUE 1 +#else +#ifdef GOOS_darwin +#define TARGET_VALUE 2 +#else +#define TARGET_VALUE 3 +#endif +#endif +#endif +` + headerFile := filepath.Join(request.Directory, "profile.h") + if err := os.WriteFile(headerFile, []byte(header), 0600); err != nil { + t.Fatal(err) + } + prefix := `#include "profile.h" +#include "profile.h" +#ifdef UNDEFINED_EXTERNAL_MACRO +#include "missing_generated.h" +#endif +` + text := prefix + strings.Replace(string(original), "MOVQ AX,ret+8(FP)", "MOVQ $(PROFILE_VALUE+TARGET_VALUE),AX\nMOVQ AX,ret+8(FP)", 1) + if err := os.WriteFile(assembly, []byte(text), 0600); err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + function, found := source.functions[pkg.Path+".Value"] + if !found { + t.Fatal("complete original function is missing") + } + want := int64(11 + targetIndex + 1) + if profile == "v2" { + want -= 4 + } + matched := 0 + for code := function.code; len(code) != 0; { + instruction, err := x86asm.Decode(code, 64) + if err != nil || instruction.Len == 0 { + t.Fatal("decode actual conditional Go object", err) + } + _, immediate := instruction.Args[1].(x86asm.Imm) + // Current Go encodes these small MOVQ constants as a + // zero-extending EAX write; Go 1.20 retains RAX's prefix. + valueRegister := instruction.Args[0] == x86asm.RAX || instruction.Args[0] == x86asm.EAX + if instruction.Op == x86asm.MOV && valueRegister && immediate { + if instruction.Args[1] != x86asm.Imm(want) { + t.Fatal("conditional branch differs from independent profile/target", instruction, want) + } + matched++ + } + code = code[instruction.Len:] + } + if matched != 1 { + t.Fatal("missing or repeated independently observed conditional result", matched) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + expanded, err := source.preprocess(request.AssemblyFile, []byte(text)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + triple := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"}[targetIndex] + options := Options{Goarch: "amd64", TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }, + Sigs: map[string]FuncSig{function.signature.Name: function.signature}, + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "conditional.ll", "conditional.o", ir) + compileLLVMToObject(t, llc, triple, "conditional-opt.ll", "conditional-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(headerFile, []byte(header+"// late mutation\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); err == nil { + t.Fatal("changed conditional header retained source authority") + } + if err := os.WriteFile(headerFile, []byte(header), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); err == nil { + t.Fatal("restored header revived a revoked observation") + } + t.Logf("actual %s/%s selected MOVQ $%d,AX", target, profile, want) + }) + } + } +} + +func TestGoAMD64FrameSourceConditionalMacrosGrantNoNumericEffects(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + pkg, request := goAMD64FrameSourceFixture(t) + name := filepath.Join(request.Directory, request.AssemblyFile) + text, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + // Macro registration closes only source selection. The fixed-local + // consumer still has no complete ordinary numeric ADDQ effect family. + body := "#ifdef GOARCH_amd64\n" + strings.Replace(string(text), "MOVQ AX,ret+8(FP)", "ADDQ $7,AX\nMOVQ AX,ret+8(FP)", 1) + "#endif\n" + if err := os.WriteFile(name, []byte(body), 0600); err != nil { + t.Fatal(err) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + expanded, err := source.preprocess(request.AssemblyFile, []byte(body)) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + sig := source.functions[pkg.Path+".Value"].signature + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) }, + Sigs: map[string]FuncSig{sig.Name: sig}, + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + if _, err := source.BindFixedLocal(ctx, pkg, file, options); !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "effect: ADDQ") { + t.Fatal("conditional macro observation created a numerical frame effect", err) + } +} + +func TestGoAMD64FrameSourceCPPRequiresActualRegistration(t *testing.T) { + source := newGoOriginalInputGuard(GoAMD64FrameCaptureRequest{Directory: t.TempDir()}) + source.env = map[string]string{"GOOS": "linux", "GOARCH": "amd64", "GOAMD64": "v2"} + if _, err := source.preprocess("value.s", []byte("#ifdef GOAMD64_v2\nTEXT value(SB),4,$0-0\nRET\n#endif\n")); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("environment strings impersonated registered actual macro sources", err) + } +} diff --git a/amd64_go_frame_numerical_source.go b/amd64_go_frame_numerical_source.go new file mode 100644 index 000000000..edc44c01d --- /dev/null +++ b/amd64_go_frame_numerical_source.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "path/filepath" + "reflect" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "golang.org/x/arch/x86/x86asm" +) + +func verifyGoAMD64FrameNumericalRequest(ctx context.Context, pkg GoPackage, request GoAMD64FrameCaptureRequest) error { + binding := request.NumericalReads + if err := binding.verify(); err != nil { + return err + } + if ctx != binding.context || request.Target != "linux/amd64" || request.Target != binding.source.request.Target || + request.GoBinary != binding.source.request.GoBinary || request.Directory != binding.source.request.Directory || + request.AssemblyFile != binding.source.request.AssemblyFile || pkg.Path != binding.packageInput.Path || + pkg.Types != binding.originalTypes || pkg.SourceSelection != nil || !numericalSameImports(pkg.Imports, binding.originalImports) { + return goAMD64FrameContext("numerical frame join has foreign complete source/package/tool/profile/context") + } + environment := func(values []string) (map[string]string, bool) { + result := make(map[string]string, len(values)) + for _, pair := range values { + key, value, valid := strings.Cut(pair, "=") + if _, duplicate := result[key]; !valid || key == "" || duplicate { + return nil, false + } + result[key] = value + } + return result, true + } + provided, valid := environment(request.Environment) + original, originalValid := environment(binding.source.request.Environment) + if !valid || !originalValid || !reflect.DeepEqual(provided, original) { + return goAMD64FrameContext("numerical frame join command environment differs") + } + binding.source.mu.Lock() + err := numericalDeclarationSyntax(binding.source, pkg) + binding.source.mu.Unlock() + return err +} + +func (source *GoAMD64FrameSource) revokeNumericalFrame() { + source.mu.Lock() + source.ready = false + source.mu.Unlock() +} + +// No Frame lock survives an external numerical/owner verification. Close can +// linearize during the independent frame round, but its final guard must fail. +func (source *GoAMD64FrameSource) verifyNumericalFrameInputs(ctx context.Context) error { + source.mu.Lock() + request, binding, seal, pkg, ready, origin := source.request, source.numericalBinding, source.numericalSeal, source.numericalPackage, source.ready, source.numericalOrigin + selection := source.selection + inputs := make(map[string][32]byte, len(source.inputs)) + for name, sum := range source.inputs { + inputs[name] = sum + } + environment := make(map[string]string, len(source.env)) + for name, value := range source.env { + environment[name] = value + } + source.mu.Unlock() + if binding == nil && request.NumericalReads == nil && origin == nil { + return nil // Preserve every existing unbound/default role. + } + if ctx == nil { + return goAMD64FrameContext("numerical frame verification requires a live context") + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("verify numerical frame context: %w", err) + } + if !ready || origin != source || binding == nil || request.NumericalReads != binding || binding.seal != seal { + return goAMD64FrameContext("numerical frame generation/token is closed, missing or changed") + } + if err := verifyGoAMD64FrameNumericalRequest(binding.context, pkg, request); err != nil { + return err + } + binding.source.mu.Lock() + actualSelection := binding.source.guard.selection + actualEnvironment := binding.source.guard.env + var mismatch bool + for _, key := range []string{"GOROOT", "GOTOOLDIR", "GOVERSION", "GOOS", "GOARCH", "GOAMD64", "GOFLAGS", "GOEXPERIMENT", "GOCACHE"} { + mismatch = mismatch || environment[key] != actualEnvironment[key] + } + for name, data := range binding.source.native.Sources { + sum, exists := inputs[name] + mismatch = mismatch || !exists || sum != sha256.Sum256(data) + } + binding.source.mu.Unlock() + if mismatch || !reflect.DeepEqual(selection, actualSelection) || len(selection.SFiles) != 1 || selection.SFiles[0] != request.AssemblyFile { + return goAMD64FrameContext("independent complete numerical/frame source selection or tool bytes differ") + } + // Catch Close that linearized while the independent owner checks ran. + source.mu.Lock() + live := source.ready && source.request.NumericalReads == binding && source.numericalBinding == binding + source.mu.Unlock() + if !live { + return goAMD64FrameContext("numerical frame closed during complete verification") + } + return nil +} + +func (source *GoAMD64FrameSource) numericalFrameObject(object string, file *File) ([]gostaticstorage.TextRecord, error) { + data, err := source.read(object) + if err != nil { + return nil, err + } + builtin, err := source.read(filepath.Join(source.env["GOROOT"], "src/cmd/internal/goobj/builtinlist.go")) + if err != nil { + return nil, err + } + texts, err := gostaticstorage.ObserveX86AssemblyText(data, builtin, source.request.Target) + if err != nil { + return nil, err + } + binding := source.numericalBinding + if err := binding.verify(); err != nil { + return nil, err + } + fingerprint, err := x86CallerSourceFingerprint(file) + if err != nil || fingerprint != binding.file { + return nil, goAMD64FrameContext("independent numerical frame parsed source differs") + } + if err := goAMD64FrameNumericalObjectJoin(texts, binding.source.native.OriginalText, binding.reads); err != nil { + return nil, err + } + return texts, nil +} + +// Both TEXT collections came from independently decoded actual original +// objects. Whole records (including positions and auxiliaries) must agree; +// every relocation must also be exactly one certified scalar read's field. +func goAMD64FrameNumericalObjectJoin(texts, numerical []gostaticstorage.TextRecord, reads []numericalSBReadFact) error { + if len(texts) == 0 || !reflect.DeepEqual(texts, numerical) { + return goAMD64FrameContext("independent complete TEXT/code/source/ABI0/relocations differ") + } + readIndex := 0 + for _, text := range texts { + if text.ABI != 0 || text.Size != int64(len(text.Code)) { + return goAMD64FrameContext("numerical frame has foreign ABI0 or incomplete code") + } + lastPC, lastInstruction := int64(-1), -1 + for _, relocation := range text.Relocations { + if readIndex >= len(reads) { + return goAMD64FrameContext("missing complete numerical frame read") + } + read := reads[readIndex] + if read.Function != text.Name || !reflect.DeepEqual(read.Relocation, relocation) || + read.PC <= lastPC || read.Instruction <= lastInstruction || read.PC >= text.Size || + read.Variable.NativeName != relocation.Target.Name { + return goAMD64FrameContext("foreign, duplicate or reordered numerical frame read/owner") + } + native, err := x86asm.Decode(text.Code[read.PC:], 64) + if err != nil || native.Op == 0 || native.Len <= 0 || read.PC+int64(native.Len) > text.Size || + !bytes.Equal(read.Code, text.Code[read.PC:read.PC+int64(native.Len)]) || native.MemBytes != int(read.Width) { + return goAMD64FrameContext("numerical frame instruction boundary/code/width differs") + } + actual, offset, err := numericalInstructionRelocation(text, read.PC, native, "amd64") + if err != nil || !reflect.DeepEqual(actual, relocation) || offset != read.Offset { + return goAMD64FrameContext("numerical frame PC displacement/type/target/ABI/addend differs") + } + lastPC, lastInstruction = read.PC, read.Instruction + readIndex++ + } + } + if readIndex != len(reads) { + return goAMD64FrameContext("numerical frame omitted a certified original read") + } + return nil +} + +func goAMD64FrameNumericalRelocationLine(relocation gostaticstorage.TextRelocation) string { + if relocation.Type != 14 || relocation.Size != 4 || !relocation.Target.ABIKnown || relocation.Target.ABI != 0 { + return "" // No other listing relocation has been authorized. + } + line := fmt.Sprintf("rel %d+%d t=R_PCREL %s", relocation.Offset, relocation.Size, relocation.Target.Name) + if relocation.Add != 0 { + line += fmt.Sprintf("%+d", relocation.Add) + } + return line +} diff --git a/amd64_go_frame_profile_test.go b/amd64_go_frame_profile_test.go new file mode 100644 index 000000000..e20b71b9c --- /dev/null +++ b/amd64_go_frame_profile_test.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +func TestGoAMD64FrameSourceActualCPUProfiles(t *testing.T) { + // Share only this root's owned build cache. Go 1.27 builds its objdump + // helper there; rebuilding that tool for every profile adds no coverage. + // Every fixture retains its own source/module/temp directories and still + // runs actual Go selection, export, object and post-capture verification. + t.Setenv("GOCACHE", t.TempDir()) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range []struct{ name, triple string }{ + {"linux/amd64", "x86_64-unknown-linux-gnu"}, + {"darwin/amd64", "x86_64-apple-darwin"}, + {"windows/amd64", "x86_64-pc-windows-msvc"}, + } { + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(target.name+"/"+profile, func(t *testing.T) { + pkg, request, file, options := fixedLocalFrameFixture(t, "PADDQ 16(SP),X0") + request.Target = target.name + request.Environment = append(request.Environment, "GOAMD64="+profile) + options.TargetTriple = target.triple + assembly := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + // Go must select the exact feature-tagged original file before + // the frame factory independently observes its native object. + tagged := fmt.Sprintf("//go:build amd64.%s\n\n%s", profile, original) + if err := os.WriteFile(assembly, []byte(tagged), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append(append([]string(nil), request.Environment...), + "GOOS="+strings.Split(target.name, "/")[0], "GOARCH=amd64", + "CGO_ENABLED=0", "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=") + output, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, + "list", "-json", "-compiled", "-export", "-mod=readonly", ".") + if err != nil { + t.Fatalf("independent actual Go profile control: %v\n%s", err, output) + } + var selected goAMD64FrameSelection + if err := json.Unmarshal(output, &selected); err != nil || selected.Export == "" || + len(selected.SFiles) != 1 || selected.SFiles[0] != request.AssemblyFile { + t.Fatalf("actual Go omitted profile-selected assembly/export: %v\n%s", err, output) + } + source := requireGoAMD64FrameCapture(t, pkg, request) + if source.env["GOAMD64"] != profile || source.env["GOOS"]+"/"+source.env["GOARCH"] != target.name { + t.Fatal("capture substituted a different actual CPU/target profile") + } + bound, err := source.BindFixedLocal(ctx, pkg, file, options) + if err != nil { + t.Fatal(err) + } + fixedLocalAPIs(t, pkg, file, bound, func(t *testing.T, ir string, err error) { + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "profile.ll", "profile.o", ir) + compileLLVMToObject(t, llc, target.triple, "profile-opt.ll", "profile-opt.o", x86MulDivRoundOptimize(t, ir)) + }) + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestGoAMD64FrameSourceCPUProfileRevocation(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(profile, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Environment = append(request.Environment, "GOAMD64="+profile) + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + before := append([]string(nil), source.request.Environment...) + replacement := "v1" + if profile == "v1" { + replacement = "v2" + } + source.request.Environment = append(source.request.Environment, "GOAMD64="+replacement) + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a different actual CPU profile retained the original observation", err) + } + source.request.Environment = before + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("restoring the CPU profile revived a revoked observation", err) + } + }) + } +} + +func TestGoAMD64FrameSourceCPUProfilesDoNotGrantGeneratedCPP(t *testing.T) { + t.Setenv("GOCACHE", t.TempDir()) + for _, profile := range []string{"v1", "v2", "v3", "v4"} { + t.Run(profile, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Environment = append(request.Environment, "GOAMD64="+profile) + name := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + conditional := fmt.Sprintf("#ifdef GOAMD64_%s\n#include \"go_asm.h\"\n%s\n#endif\n", profile, original) + if err := os.WriteFile(name, []byte(conditional), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + source.Close() + t.Fatal("conditional generated header acquired an ordinary profile observation") + } + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "qualified metadata") { + t.Fatal("generated CPP still needs its own source mapping, not CPU rejection", err) + } + }) + } +} diff --git a/amd64_go_frame_source.go b/amd64_go_frame_source.go new file mode 100644 index 000000000..9f164966d --- /dev/null +++ b/amd64_go_frame_source.go @@ -0,0 +1,1537 @@ +package plan9asm + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/xgo-dev/plan9asm/internal/goabi" + "github.com/xgo-dev/plan9asm/internal/goasmcmd" + "github.com/xgo-dev/plan9asm/internal/goasmmacros" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "github.com/xgo-dev/plan9asm/internal/gotoolchain" + "golang.org/x/arch/x86/x86asm" +) + +// GoAMD64FrameCaptureRequest describes actual input locations, not permissions. +type GoAMD64FrameCaptureRequest struct { + GoBinary, Directory, AssemblyFile, Target string + Environment []string + // NumericalReads joins one already authenticated original numerical-read + // source. It observes relocations; it grants no additional memory role. + NumericalReads *GoX86NumericalReadBinding +} + +// GoAMD64FrameSource is a sealed actual-object observation. It does not grant +// a native backing, caller region, memory access or return certificate. +type GoAMD64FrameSource struct { + mu sync.Mutex + identity *GoAMD64FrameSource + ready bool + request GoAMD64FrameCaptureRequest + selection goAMD64FrameSelection + env map[string]string + assemblerDefines []string + inputs map[string][32]byte + entries []string + directoryInfo os.FileInfo + owned string + ownedInfo os.FileInfo + functions map[string]goAMD64FrameFunction + declarations map[string]goAMD64FrameDeclaration + imports map[string]goAMD64FrameSelection + origins map[string]int + downloadedOrigin *gooriginalsrc.OriginSet + downloadedModule *gooriginalsrc.Module + downloadedImports map[string]gooriginalsrc.Module + numericalBinding *GoX86NumericalReadBinding + numericalOrigin *GoAMD64FrameSource + numericalSeal [32]byte + numericalPackage GoPackage + numericalCapturing bool + numericalCommands []numericalSBCommand // Actual capture output only; never an authority. +} + +// newGoOriginalInputGuard initializes only private input/cleanup ownership. +// It has no ready object, declaration, source-origin or frame capability. +func newGoOriginalInputGuard(request GoAMD64FrameCaptureRequest) *GoAMD64FrameSource { + request.Environment = append([]string(nil), request.Environment...) + source := &GoAMD64FrameSource{request: request, + inputs: make(map[string][32]byte), origins: make(map[string]int)} + source.identity = source + return source +} + +// Transport is an observation role, never a memory or alias permission. +type goAMD64FrameTransport uint8 + +const ( + goAMD64FrameNumericTransport goAMD64FrameTransport = iota + 1 + goAMD64FrameCompositeTransport +) + +type goAMD64FrameDeclaration struct { + signature FuncSig + transport goAMD64FrameTransport + languageType string // go/types model identity, not a native pointee extent +} + +type goAMD64FrameFunction struct { + objectSymbol, listingSymbol string + locals, args int64 + code []byte + positions map[int]string + sourcePositions map[int][]goAMD64FrameSourcePosition + signature FuncSig + transport goAMD64FrameTransport + languageType string +} + +type goAMD64FrameSourcePosition struct { + file string + line int +} + +// This is decoded only from the actual driver stdout, never from a proof file. +type goAMD64FrameSelection struct { + Dir, ImportPath, Name, Export, ForTest string + GoFiles, CompiledGoFiles, SFiles, HFiles, CgoFiles, Imports, EmbedFiles, EmbedPatterns []string + IgnoredGoFiles, IgnoredOtherFiles []string + Standard bool + ImportMap map[string]string + CFiles, CXXFiles, MFiles, FFiles, SwigFiles, SwigCXXFiles, SysoFiles []string + Module *struct { + Path, Version, Dir, GoMod, GoVersion string + Main bool + Replace json.RawMessage + } + Error json.RawMessage + DepsErrors []json.RawMessage +} + +func (selected goAMD64FrameSelection) nonordinaryRole() bool { + for _, files := range [][]string{selected.CgoFiles, selected.CFiles, selected.CXXFiles, selected.MFiles, + selected.FFiles, selected.SwigFiles, selected.SwigCXXFiles, selected.SysoFiles, selected.EmbedFiles, selected.EmbedPatterns} { + if len(files) != 0 { + return true + } + } + return false +} + +func CaptureGoAMD64FrameSource(ctx context.Context, pkg GoPackage, request GoAMD64FrameCaptureRequest) (*GoAMD64FrameSource, error) { + return captureGoAMD64FrameSource(ctx, pkg, request, nil) +} + +func captureGoAMD64FrameSource(ctx context.Context, pkg GoPackage, request GoAMD64FrameCaptureRequest, exact *goAMD64DownloadedIdentity) (_ *GoAMD64FrameSource, captureErr error) { + if ctx != nil && ctx.Err() != nil { + if request.NumericalReads != nil { + return nil, fmt.Errorf("capture numerical frame context: %w", ctx.Err()) + } + return nil, errors.Join(goAMD64FrameContext("requires live capture context"), ctx.Err()) + } + if ctx == nil || pkg.Types == nil || pkg.Path != pkg.Types.Path() || + !filepath.IsAbs(request.GoBinary) || !filepath.IsAbs(request.Directory) || + filepath.Base(request.AssemblyFile) != request.AssemblyFile || filepath.Ext(request.AssemblyFile) != ".s" { + return nil, goAMD64FrameContext("requires live context, actual package and bounded absolute inputs") + } + parts := strings.Split(request.Target, "/") + if len(parts) != 2 || parts[1] != "amd64" || parts[0] != "linux" && parts[0] != "darwin" && parts[0] != "windows" { + return nil, goAMD64FrameContext("initial object capture supports ordinary AMD64 targets only") + } + var err error + request.Directory, err = filepath.EvalSymlinks(request.Directory) + if err != nil { + return nil, err + } + source := newGoOriginalInputGuard(request) + if request.NumericalReads != nil { + if err := verifyGoAMD64FrameNumericalRequest(ctx, pkg, source.request); err != nil { + return nil, err + } + source.numericalBinding = request.NumericalReads + source.numericalOrigin = source + source.numericalSeal = request.NumericalReads.seal + source.numericalPackage = pkg + source.numericalCapturing = true + } + provided := make(map[string]string) + for _, value := range request.Environment { + key, setting, valid := strings.Cut(value, "=") + if !valid || key == "" { + return nil, goAMD64FrameContext("malformed command environment") + } + provided[key] = setting + if (key == "GOFLAGS" || key == "GOEXPERIMENT") && setting != "" || key == "GOPACKAGESDRIVER" && setting != "" && setting != "off" { + return nil, goAMD64FrameContext("effective overlay, custom flags, experiments or driver require their own source mapping") + } + } + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "TMPDIR", "GOTMPDIR"} { + if !filepath.IsAbs(provided[key]) { + return nil, goAMD64FrameContext("explicit private %s directory is required", key) + } + } + settings := map[string]string{"GOOS": parts[0], "GOARCH": "amd64", "CGO_ENABLED": "0", "GOTOOLCHAIN": "local", "GOENV": "off", "GOWORK": "off", "GOFLAGS": "", "GOPROXY": "off"} + var env []string + for _, value := range source.request.Environment { + key, _, _ := strings.Cut(value, "=") + if _, replace := settings[key]; !replace { + env = append(env, value) + } + } + for key, value := range settings { + env = append(env, key+"="+value) + } + source.request.Environment = env + if err := source.snapshotDirectory(); err != nil { + return nil, err + } + if err := source.observe(ctx); err != nil { + return nil, err + } + if exact == nil { + if err := source.verifyOwnedSourceDirectory(); err != nil { + return nil, err + } + } else { + inside, err := gostaticstorage.SourceDirectoryWithinRoot(request.Directory, source.env["GOROOT"]) + if err != nil || inside { + return nil, errors.Join(goAMD64FrameContext("SDK source cannot acquire an exact downloaded role"), err) + } + if err := source.captureDownloadedOrigin(ctx, *exact, provided["GOTMPDIR"]); err != nil { + return nil, err + } + } + selected, err := source.load(ctx) + if err != nil { + return nil, err + } + source.selection = selected + if source.numericalBinding != nil && (len(selected.SFiles) != 1 || selected.SFiles[0] != request.AssemblyFile) { + return nil, goAMD64FrameContext("numerical frame join requires one complete original assembly selection") + } + moduleRoot := request.Directory + if source.downloadedModule != nil { + moduleRoot = source.downloadedModule.Root + } + mainRole, moduleVersion := true, "" + metadata := filepath.Join(moduleRoot, "go.mod") + if source.legacyDownloadedModule() { + mainRole, moduleVersion = false, source.downloadedModule.SourceVersion + metadata = source.downloadedModule.GoMod + } + if selected.Dir != request.Directory || selected.ImportPath != pkg.Path || selected.ForTest != "" || selected.Module == nil || + selected.Module.Main != mainRole || selected.Module.Version != moduleVersion || selected.Module.Dir != moduleRoot || len(selected.Module.Replace) != 0 && string(selected.Module.Replace) != "null" || + selected.nonordinaryRole() || len(selected.GoFiles) == 0 || len(selected.SFiles) == 0 { + return nil, goAMD64FrameContext("capture needs complete ordinary original module sources") + } + if selected.Module.GoMod != metadata { + return nil, goAMD64FrameContext("actual module metadata is outside the original source directory") + } + if source.downloadedModule != nil { + original := source.downloadedModule + path, err := goAMD64DownloadedPackagePath(*original, selected.Dir) + if err != nil { + return nil, err + } + if selected.Module.Path != original.ModulePath || selected.Module.GoVersion != gooriginalsrc.SelectedGoVersion(original.GoVersion, mainRole) || + selected.ImportPath != path { + return nil, goAMD64FrameContext("exact downloaded capture requires its original primary module package") + } + } + source.imports, err = source.loadOriginalDeclarationImports(ctx) + if err != nil { + return nil, err + } + source.declarations, err = source.typedDeclarations(pkg, false) + if err != nil { + return nil, err + } + rules, err := goasmmacros.Capture(source.env["GOROOT"], source.env["GOVERSION"], selected.ImportPath, source.read) + if err != nil || rules.PackageRole != "ordinary_path" && rules.PackageRole != "legacy_no_experiment_macros" { + return nil, goAMD64FrameContext("ordinary assembler macro source registration: %v", err) + } + source.assemblerDefines, err = GoAssemblerDefinesForEnvironment(source.env["GOOS"], source.env["GOARCH"], source.env) + if err != nil { + return nil, err + } + source.owned, err = os.MkdirTemp(provided["TMPDIR"], "plan9asm-go-frame-object-") + if err != nil { + return nil, err + } + defer func() { + if captureErr != nil { + captureErr = errors.Join(captureErr, source.Close()) + } + }() + source.ownedInfo, err = os.Lstat(source.owned) + if err != nil { + return nil, err + } + canonicalOwned, err := filepath.EvalSymlinks(source.owned) + if err != nil { + return nil, err + } + source.owned = canonicalOwned + if err := source.verifyOwned(); err != nil { + return nil, err + } + source.functions = make(map[string]goAMD64FrameFunction) + found := false + for _, name := range selected.SFiles { + found = found || name == request.AssemblyFile + original, err := source.read(filepath.Join(request.Directory, name)) + if err != nil { + return nil, err + } + expanded, err := source.preprocess(name, original) + if err != nil { + return nil, err + } + file, err := ParseWithDefines(ArchAMD64, expanded, nil) + if err != nil || len(file.Funcs) == 0 || len(file.UnlinkedPrelude) != 0 { + return nil, goAMD64FrameContext("full original assembly lacks complete TEXT: %v", err) + } + object := filepath.Join(source.owned, name+".o") + assemblyInput := name + if source.numericalBinding != nil { + assemblyInput = filepath.Join(request.Directory, name) + } + // The independent actual assembler and source preprocessor consume + // the same registered ordinary profile macros. Generated headers keep + // their separate source mapping; CPU identity grants no memory role. + args := []string{"tool", "asm", "-S", "-p", pkg.Path, + "-I", request.Directory, "-I", filepath.Join(source.env["GOROOT"], "pkg", "include")} + for _, define := range source.assemblerDefines { + args = append(args, "-D", define) + } + args = append(args, "-o", object, assemblyInput) + listing, err := source.run(ctx, args...) + if err != nil { + return nil, err + } + objectBytes, err := source.read(object) + if err != nil || len(objectBytes) == 0 { + if err == nil { + err = goAMD64FrameContext("actual assembler produced an empty object") + } + return nil, err + } + if err := os.WriteFile(object+".listing", listing, 0600); err != nil { + return nil, err + } + if _, err := source.read(object + ".listing"); err != nil { + return nil, err + } + var texts []gostaticstorage.TextRecord + if source.numericalBinding != nil { + texts, err = source.numericalFrameObject(object, file) + if err != nil { + return nil, err + } + } + var functions map[string]goAMD64FrameFunction + if len(file.Data)+len(file.Globl) != 0 { + observed, readErr := source.frameDataObject(expanded, objectBytes) + if readErr != nil { + return nil, readErr + } + filtered, readErr := goAMD64FrameDataListing(listing, observed) + if readErr != nil { + return nil, readErr + } + var unwind *gostaticstorage.WindowsAMD64AssemblyUnwind + if source.request.Target == "windows/amd64" { + value, readErr := source.observeWindowsUnwind(objectBytes) + if readErr != nil { + return nil, readErr + } + unwind = &value + } + functions, err = goAMD64FrameListingWithDataObject(filtered, file, pkg, source.origins, observed, unwind) + if err == nil { + err = goAMD64FrameDataTextJoin(functions, observed, source.request.Directory, source.origins) + } + if err == nil && unwind != nil { + err = goAMD64FrameWindowsUnwindFunctions(functions, source.origins, source.request.Directory, *unwind) + } + } else if source.request.Target == "windows/amd64" && (gotoolchain.AuditedGo127(source.env["GOVERSION"]) || + bytes.Contains(listing, []byte(" SSEHUNWINDINFO "))) { + unwind, readErr := source.observeWindowsUnwind(objectBytes) + if readErr != nil { + return nil, readErr + } + functions, err = goAMD64FrameWindowsUnwindListing(listing, file, pkg, source.origins, + source.request.Directory, unwind) + } else { + functions, err = goAMD64FrameListingWithNumericalObject(listing, file, pkg, source.origins, texts) + } + if err != nil { + return nil, err + } + if source.numericalBinding != nil && !bytes.Equal(listing, source.numericalBinding.source.native.OriginalListing) { + return nil, goAMD64FrameContext("independent complete original numerical/frame listing differs") + } + for _, text := range texts { + if !bytes.Equal(functions[text.Name].code, text.Code) { + return nil, goAMD64FrameContext("independent complete object/listing code bytes differ") + } + } + if err := source.verifyObjectListing(ctx, object, functions); err != nil { + return nil, err + } + for symbol, fn := range functions { + if _, duplicate := source.functions[symbol]; duplicate { + return nil, goAMD64FrameContext("duplicate original TEXT symbol") + } + declaration, exists := source.declarations[symbol] + if !exists || !reflect.DeepEqual(fn.signature, declaration.signature) { + return nil, goAMD64FrameContext("object lacks its independently checked declaration/layout") + } + fn.transport, fn.languageType = declaration.transport, declaration.languageType + source.functions[symbol] = fn + } + } + if !found { + return nil, goAMD64FrameContext("requested assembly is absent from actual complete selection") + } + source.ready = true + source.numericalCapturing = false + if err := source.Verify(ctx); err != nil { + return nil, err + } + return source, nil +} + +// Verify rechecks actual routing/selection and all original/tool/export/artifact +// bytes. Failure permanently revokes this observation; it grants no memory. +func (source *GoAMD64FrameSource) Verify(ctx context.Context) (verifyErr error) { + if source == nil || source.identity != source { + return goAMD64FrameContext("missing sealed actual-object observation") + } + if err := source.verifyNumericalFrameInputs(ctx); err != nil { + source.revokeNumericalFrame() + return err + } + defer func() { + if err := source.verifyNumericalFrameInputs(ctx); err != nil { + source.revokeNumericalFrame() + // The inner observer already revokes on failure. Its own closed + // generation must not relabel an actual command/cancellation + // failure as a source exclusion during the outer postcheck. + if verifyErr == nil || !errors.Is(err, ErrProbeNeedsContext) { + verifyErr = errors.Join(verifyErr, err) + } + } + }() + return source.verifyFrameOnly(ctx) +} + +func (source *GoAMD64FrameSource) verifyFrameOnly(ctx context.Context) (verifyErr error) { + if source == nil { + return goAMD64FrameContext("missing sealed actual-object observation") + } + source.mu.Lock() + defer source.mu.Unlock() + defer func() { + if verifyErr != nil { + source.ready = false + } + }() + if ctx != nil && ctx.Err() != nil { + if source.numericalOrigin != nil { + return fmt.Errorf("verify numerical frame observer context: %w", ctx.Err()) + } + return errors.Join(goAMD64FrameContext("expired actual-object observation"), ctx.Err()) + } + if !source.ready || ctx == nil { + return goAMD64FrameContext("closed, uninitialized or expired actual-object observation") + } + if err := source.verifyDownloadedOrigin(); err != nil { + return err + } + before := source.env + if err := source.observe(ctx); err != nil { + return err + } + if !reflect.DeepEqual(before, source.env) { + return goAMD64FrameContext("actual Go environment changed") + } + if source.downloadedOrigin == nil { + if err := source.verifyOwnedSourceDirectory(); err != nil { + return err + } + } + if err := source.verifyInputs(); err != nil { + return err + } + selected, err := source.load(ctx) + if err != nil { + return err + } + if !reflect.DeepEqual(source.selection, selected) { + return goAMD64FrameContext("actual complete package selection/export changed") + } + imports, err := source.loadOriginalDeclarationImports(ctx) + if err != nil && source.numericalOrigin != nil { + return err + } + if err != nil || !reflect.DeepEqual(source.imports, imports) { + return goAMD64FrameContext("actual complete declaration import graph changed: %v", err) + } + if err := ctx.Err(); err != nil { + if source.numericalOrigin != nil { + return fmt.Errorf("numerical frame verification context expired after actual commands: %w", err) + } + return errors.Join(goAMD64FrameContext("verification context expired after actual commands"), err) + } + return errors.Join(source.verifyInputs(), source.verifyDownloadedOrigin()) +} + +func (source *GoAMD64FrameSource) verifyOwnedSourceDirectory() error { + if err := gostaticstorage.ValidateOwnedSourceDirectory(source.request.Directory, source.env["GOROOT"], source.env["GOMODCACHE"]); err != nil { + return errors.Join(goAMD64FrameContext("owned source needs a distinct physical SDK/cache role"), err) + } + return nil +} + +func (source *GoAMD64FrameSource) verifyInputs() error { + if err := source.verifyOwned(); err != nil { + return err + } + entries, directoryInfo, err := gooriginalsrc.ReadDirectory(source.request.Directory, 256) + if err != nil || source.directoryInfo == nil || !os.SameFile(source.directoryInfo, directoryInfo) { + return goAMD64FrameContext("original package directory inventory or identity changed: %v", err) + } + var names []string + for _, entry := range entries { + names = append(names, entry.Name()) + } + if !reflect.DeepEqual(names, source.entries) { + return goAMD64FrameContext("original package directory inventory changed") + } + for name, expected := range source.inputs { + data, err := goAMD64FrameRead(name) + if err != nil || sha256.Sum256(data) != expected { + return goAMD64FrameContext("original/tool/export/object input changed: %s: %v", filepath.Base(name), err) + } + } + return nil +} + +func (source *GoAMD64FrameSource) verifyOwned() error { + info, err := os.Lstat(source.owned) + if err != nil || source.ownedInfo == nil || !info.IsDir() || !os.SameFile(source.ownedInfo, info) { + return goAMD64FrameContext("factory artifact directory identity changed or is inaccessible: %v", err) + } + return nil +} + +// Close invalidates the token before removing only its factory-owned artifacts. +// Calling it again is harmless; no source, tool or cache-owned file is removed. +func (source *GoAMD64FrameSource) Close() error { + if source == nil { + return nil + } + if source.identity != source { + return goAMD64FrameContext("copied or uninitialized observation cannot dispose original artifacts") + } + source.mu.Lock() + defer source.mu.Unlock() + source.ready = false + if source.numericalOrigin != nil && source.numericalOrigin != source { + return goAMD64FrameContext("copied numerical frame cannot close the original artifact owner") + } + if source.owned == "" { + return nil + } + if err := source.verifyOwned(); err != nil { + return err + } + err := os.RemoveAll(source.owned) + if err == nil { + source.owned = "" + source.ownedInfo = nil + } + return err +} + +func goAMD64FrameContext(format string, args ...any) error { + return fmt.Errorf("%w: actual Go frame source: %s", ErrProbeNeedsContext, fmt.Sprintf(format, args...)) +} + +func (source *GoAMD64FrameSource) run(ctx context.Context, args ...string) ([]byte, error) { + if ctx == nil { + return nil, goAMD64FrameContext("actual command requires live context") + } + if err := ctx.Err(); err != nil { + if source.numericalOrigin != nil { + return nil, fmt.Errorf("numerical frame actual command context expired: %w", err) + } + return nil, errors.Join(goAMD64FrameContext("actual command context expired"), err) + } + if err := source.verifyDownloadedOrigin(); err != nil { + return nil, err + } + // Capture has not published the source and holds no Frame lock. Verify's + // existing locked command path is guarded outside the entire round instead. + if source.numericalCapturing { + if err := source.numericalBinding.verify(); err != nil { + return nil, err + } + } + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + var output []byte + var err error + if source.legacyDownloadedModule() && len(args) > 0 && args[0] == "list" { + output, err = source.runDownloadedList(ctx, args) + } else { + output, err = goasmcmd.Run(ctx, source.request.Directory, source.request.Environment, source.request.GoBinary, args...) + } + if source.numericalCapturing { + source.numericalCommands = append(source.numericalCommands, numericalSBCommand{ + Directory: source.request.Directory, Binary: source.request.GoBinary, + Arguments: append([]string(nil), args...), Stdout: append([]byte(nil), output...), + }) + err = errors.Join(err, source.numericalBinding.verify()) + } + // Preserve both independent origins and cancellation when commands race + // revocation. Neither observation grants the other's source permission. + return output, errors.Join(err, ctx.Err(), source.verifyDownloadedOrigin()) +} + +func goAMD64FrameRead(name string) (_ []byte, readErr error) { + resolved, err := filepath.EvalSymlinks(name) + if err != nil || resolved != name { + return nil, goAMD64FrameContext("input path is redirected or inaccessible: %s", filepath.Base(name)) + } + before, err := os.Lstat(name) + if err != nil || !before.Mode().IsRegular() { + return nil, goAMD64FrameContext("input role changed before opening") + } + file, err := os.Open(name) + if err != nil { + return nil, err + } + defer func() { readErr = errors.Join(readErr, file.Close()) }() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() || !os.SameFile(before, info) || info.Size() > 64<<20 { + return nil, goAMD64FrameContext("input is not a bounded regular file") + } + data, err := io.ReadAll(io.LimitReader(file, 64<<20+1)) + if err != nil || len(data) > 64<<20 { + return nil, goAMD64FrameContext("input byte bound/read failure: %v", err) + } + after, err := os.Lstat(name) + if err != nil || !os.SameFile(info, after) || !after.Mode().IsRegular() || int64(len(data)) != info.Size() { + return nil, goAMD64FrameContext("input role/extent changed while reading") + } + return data, nil +} + +func (source *GoAMD64FrameSource) read(name string) ([]byte, error) { + data, err := goAMD64FrameRead(name) + if err != nil { + return nil, err + } + digest := sha256.Sum256(data) + if before, exists := source.inputs[name]; exists && before != digest { + return nil, goAMD64FrameContext("input changed during capture: %s", filepath.Base(name)) + } + source.inputs[name] = digest + return data, nil +} + +func (source *GoAMD64FrameSource) snapshotDirectory() error { + resolved, err := filepath.EvalSymlinks(source.request.Directory) + if err != nil || resolved != source.request.Directory { + return goAMD64FrameContext("requires canonical actual package directory") + } + entries, directoryInfo, err := gooriginalsrc.ReadDirectory(resolved, 256) + if err != nil || len(entries) == 0 || len(entries) > 256 { + return goAMD64FrameContext("package directory is unavailable or exceeds its bound") + } + source.directoryInfo = directoryInfo + var total int + for _, entry := range entries { + if entry.IsDir() { + return goAMD64FrameContext("initial source root requires a flat complete inventory") + } + source.entries = append(source.entries, entry.Name()) + data, err := source.read(filepath.Join(resolved, entry.Name())) + if err != nil { + return err + } + total += len(data) + if total > 16<<20 { + return goAMD64FrameContext("original package byte inventory exceeds its bound") + } + } + return nil +} + +func (source *GoAMD64FrameSource) observe(ctx context.Context) error { + if _, err := source.read(source.request.GoBinary); err != nil { + return err + } + data, err := source.run(ctx, "env", "-json", "GOROOT", "GOTOOLDIR", "GOCACHE", "GOMODCACHE", "GOPATH", "GOTMPDIR", "GOVERSION", "GOOS", "GOARCH", "GOFLAGS", "GOEXPERIMENT", "GOAMD64") + if err != nil { + return err + } + var env map[string]string + if err := json.Unmarshal(data, &env); err != nil { + return err + } + if env["GOOS"]+"/"+env["GOARCH"] != source.request.Target || env["GOFLAGS"] != "" || env["GOEXPERIMENT"] != "" || + filepath.Join(env["GOROOT"], "bin", gotoolchain.HostExecutable("go")) != source.request.GoBinary { + return goAMD64FrameContext("actual driver/target/CPU/source routing needs a separate audited contract") + } + switch env["GOAMD64"] { + case "v1", "v2", "v3", "v4": + // Each actual profile retains its own selected original Go sources, + // exports and object bytes. Verify compares this environment exactly; + // no capture is relabeled or reused across different CPU profiles. + default: + return goAMD64FrameContext("unregistered actual GOAMD64 profile %q", env["GOAMD64"]) + } + version, err := source.read(filepath.Join(env["GOROOT"], "VERSION")) + if err != nil || strings.Split(string(version), "\n")[0] != env["GOVERSION"] { + return goAMD64FrameContext("actual driver differs from source VERSION") + } + for _, tool := range []string{"asm", "compile", "objdump"} { + executable := gotoolchain.HostExecutable(tool) + data, err := source.run(ctx, "tool", "-n", tool) + if err != nil { + return err + } + name := strings.TrimSpace(string(data)) + if !filepath.IsAbs(name) || strings.ContainsAny(name, "\r\n") || tool != "objdump" && name != filepath.Join(env["GOTOOLDIR"], executable) { + return goAMD64FrameContext("unrecognized actual %s route", tool) + } + resolved, err := filepath.EvalSymlinks(name) + if err != nil { + return err + } + if tool == "objdump" && name != filepath.Join(env["GOTOOLDIR"], executable) { + cache, err := filepath.EvalSymlinks(env["GOCACHE"]) + if err != nil || !goAssemblyPathWithinRoot(cache, resolved) || filepath.Base(resolved) != executable { + return goAMD64FrameContext("unknown builtin object observer origin") + } + } + name = resolved + env["route/"+tool] = name + if _, err := source.read(name); err != nil { + return err + } + } + for _, name := range []string{"src/cmd/internal/obj/x86/obj6.go", "src/cmd/internal/obj/x86/asm6.go", "src/cmd/go/internal/work/gc.go", "src/cmd/asm/main.go", "src/cmd/go/internal/tool/tool.go", "src/cmd/internal/objabi/stack.go"} { + if _, err := source.read(filepath.Join(env["GOROOT"], filepath.FromSlash(name))); err != nil { + return err + } + } + source.env = env + return nil +} + +func (source *GoAMD64FrameSource) load(ctx context.Context) (goAMD64FrameSelection, error) { + selected, _, err := source.loadWithRaw(ctx) + return selected, err +} + +func (source *GoAMD64FrameSource) loadWithRaw(ctx context.Context) (goAMD64FrameSelection, []byte, error) { + var selected goAMD64FrameSelection + data, err := source.run(ctx, "list", "-json", "-compiled", "-export", "-mod=readonly", ".") + if err != nil { + return selected, data, err + } + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&selected); err != nil { + return selected, data, err + } + var extra any + if decoder.Decode(&extra) != io.EOF || len(selected.Error) != 0 || len(selected.DepsErrors) != 0 { + return selected, data, goAMD64FrameContext("incomplete or ambiguous actual package export") + } + if selected.Export == "" { + return selected, data, goAMD64FrameContext("actual package has no compiled export") + } + selected.Export, err = filepath.EvalSymlinks(selected.Export) + cache, cacheErr := filepath.EvalSymlinks(source.env["GOCACHE"]) + if err != nil || cacheErr != nil || !goAssemblyPathWithinRoot(cache, selected.Export) { + return selected, data, goAMD64FrameContext("compiled export is outside the actual build cache") + } + if export, err := source.read(selected.Export); err != nil || len(export) == 0 { + if err == nil { + err = goAMD64FrameContext("actual package export is empty") + } + return selected, data, err + } + return selected, data, nil +} + +func (source *GoAMD64FrameSource) numericDeclarations(pkg GoPackage) error { + _, err := source.typedDeclarations(pkg, true) + return err +} + +// The numeric-only query remains strict. The fixed-local binder separately +// checks the closed transport role and replays each actual typed component; +// observation itself never supplies local or incoming memory permission. +func (source *GoAMD64FrameSource) typedDeclarations(pkg GoPackage, numericOnly bool) (map[string]goAMD64FrameDeclaration, error) { + return source.typedDeclarationsForArch(pkg, numericOnly, "amd64") +} + +// The numerical SB binding also observes 386 declarations. This helper only +// rechecks language/ABI0 identity; it does not extend the AMD64 frame factory. +func (source *GoAMD64FrameSource) typedDeclarationsForArch(pkg GoPackage, numericOnly bool, arch string) (map[string]goAMD64FrameDeclaration, error) { + fset := token.NewFileSet() + var syntax []*ast.File + var compiled []string + bodyless := make(map[string]bool) + for _, name := range source.selection.GoFiles { + filename := filepath.Join(source.request.Directory, name) + data, err := source.read(filename) + if err != nil { + return nil, err + } + file, err := parser.ParseFile(fset, filename, data, parser.ParseComments) + if err != nil { + return nil, err + } + for _, declaration := range file.Decls { + switch declaration := declaration.(type) { + case *ast.FuncDecl: + if declaration.Body == nil && declaration.Recv == nil { + bodyless[declaration.Name.Name] = true + } + case *ast.GenDecl: + if declaration.Tok != token.TYPE && declaration.Tok != token.IMPORT && + declaration.Tok != token.CONST && declaration.Tok != token.VAR { + return nil, goAMD64FrameContext("unknown original declaration role") + } + default: + return nil, goAMD64FrameContext("unknown original declaration role") + } + } + syntax = append(syntax, file) + compiled = append(compiled, filename) + } + if len(pkg.Syntax) != len(syntax) { + return nil, goAMD64FrameContext("supplied syntax omitted complete ordinary source files") + } + if len(bodyless) == 0 { + return nil, goAMD64FrameContext("original source has no bodyless assembly declaration") + } + sort.Strings(compiled) + actual := append([]string(nil), source.selection.CompiledGoFiles...) + for i, name := range actual { + if !filepath.IsAbs(name) { + actual[i] = filepath.Join(source.request.Directory, name) + } + } + sort.Strings(actual) + if !reflect.DeepEqual(compiled, actual) { + return nil, goAMD64FrameContext("effective/generated compiled source mapping is unbound") + } + sizes := types.SizesFor("gc", arch) + actualImporter := importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) { + if target, mapped := source.selection.ImportMap[path]; mapped { + path = target + } + dependency, present := source.imports[path] + if !present || dependency.Export == "" { + return nil, goAMD64FrameContext("missing captured actual compiler export: %s", path) + } + data, err := source.read(dependency.Export) + if err != nil { + return nil, err + } + return io.NopCloser(bytes.NewReader(data)), nil + }) + config := &types.Config{Sizes: sizes, Importer: actualImporter} + if source.legacyDownloadedModule() { + config.GoVersion = "go1.16" + } + typed, err := config.Check(pkg.Path, fset, syntax, nil) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(typed.Scope().Names(), pkg.Types.Scope().Names()) { + return nil, goAMD64FrameContext("actual selected declarations differ from supplied types") + } + declarations := make(map[string]goAMD64FrameDeclaration) + for _, name := range typed.Scope().Names() { + if named, ok := typed.Scope().Lookup(name).(*types.TypeName); ok { + previous, supplied := pkg.Types.Scope().Lookup(name).(*types.TypeName) + if !supplied || named.IsAlias() != previous.IsAlias() { + return nil, goAMD64FrameContext("actual local type declaration differs: %s", name) + } + // The complete original Go program has already been typechecked. + // Unrelated interfaces, callbacks, maps or recursive/generic types + // do not cross an assembly ABI and must not enter its restricted + // transport grammar. Every type reachable from an actual assembly + // parameter/result is independently compared in full below. + continue + } + // Bodies and values are independently typechecked, but only original + // bodyless assembly declarations enter the frame observation table. + if !bodyless[name] { + continue + } + fn, ok := typed.Scope().Lookup(name).(*types.Func) + previous, supplied := pkg.Types.Scope().Lookup(name).(*types.Func) + if !ok || !supplied { + return nil, goAMD64FrameContext("missing actual typed declaration") + } + signature := fn.Type().(*types.Signature) + priorSignature, valid := previous.Type().(*types.Signature) + if !valid || priorSignature == nil || signature.Recv() != nil || signature.Variadic() || signature.TypeParams().Len() != 0 || + priorSignature.Recv() != nil || priorSignature.Variadic() || priorSignature.TypeParams().Len() != 0 { + return nil, goAMD64FrameContext("custom, generic or variadic declaration transport") + } + role := goAMD64FrameNumericTransport + var language strings.Builder + for _, tuple := range []*types.Tuple{signature.Params(), signature.Results()} { + if tuple.Len() > 4096 { + return nil, goAMD64FrameContext("declaration tuple bound") + } + language.WriteString("[") + for i := 0; i < tuple.Len(); i++ { + typ := tuple.At(i).Type() + switch underlying := typ.Underlying().(type) { + case *types.Basic: + if underlying.Kind() == types.Bool { + role = goAMD64FrameCompositeTransport + } else if underlying.Info()&types.IsInteger == 0 { + return nil, goAMD64FrameContext("observation excludes noninteger scalar transport") + } + case *types.Pointer, *types.Slice: + role = goAMD64FrameCompositeTransport + default: + return nil, goAMD64FrameContext("by-value aggregate transport needs a separate observation contract") + } + identity, err := goAMD64FrameLanguageType(typ, sizes) + if err != nil { + return nil, err + } + fmt.Fprintf(&language, "%q:%s;", tuple.At(i).Name(), identity) + if language.Len() > 64<<10 { + return nil, goAMD64FrameContext("declaration language byte bound") + } + } + language.WriteString("]") + } + if numericOnly && role != goAMD64FrameNumericTransport { + return nil, goAMD64FrameContext("fixed-local consumer requires numeric transport, not pointer/slice observation") + } + // Named types from two independent typechecks have different object + // identities. Compare their complete language/layout trees, not Ptr or + // qualified spelling alone; underlying and pointee changes must fail. + priorLanguage := strings.Builder{} + for _, tuple := range []*types.Tuple{priorSignature.Params(), priorSignature.Results()} { + if tuple.Len() > 4096 { + return nil, goAMD64FrameContext("supplied declaration tuple bound") + } + priorLanguage.WriteString("[") + for i := 0; i < tuple.Len(); i++ { + identity, err := goAMD64FrameLanguageType(tuple.At(i).Type(), sizes) + if err != nil { + return nil, err + } + fmt.Fprintf(&priorLanguage, "%q:%s;", tuple.At(i).Name(), identity) + if priorLanguage.Len() > 64<<10 { + return nil, goAMD64FrameContext("supplied declaration language byte bound") + } + } + priorLanguage.WriteString("]") + } + if language.String() != priorLanguage.String() { + return nil, goAMD64FrameContext("actual language type/pointee/field identity mismatch: %s", name) + } + actual, err := DeriveGoFuncABI0Signature(pkg.Path+"."+name, fn, arch, sizes) + prior, priorErr := DeriveGoFuncABI0Signature(pkg.Path+"."+name, previous, arch, sizes) + if err != nil || priorErr != nil || !reflect.DeepEqual(actual, prior) { + return nil, goAMD64FrameContext("actual typed declaration/layout mismatch: %s", name) + } + declarations[actual.Name] = goAMD64FrameDeclaration{signature: actual, transport: role, languageType: language.String()} + } + return declarations, nil +} + +// A bounded semantic language tree distinguishes equal-sized LLVM pointers, +// named-underlying changes and aggregate element/field models. Sizes belongs +// to the independent go/types model, not an observation of native pointee +// padding (Go1.20 StdSizes differs from its compiler). Only pointer/slice ABI0 +// slots are bound to the actual object. No pointee extent, alias or GC grant. +func goAMD64FrameLanguageType(typ types.Type, sizes types.Sizes) (string, error) { + if sizes == nil { + return "", goAMD64FrameContext("missing target language layout") + } + var output strings.Builder + active := make(map[types.Type]bool) + nodes := 0 + var appendType func(types.Type, int) error + appendType = func(typ types.Type, depth int) error { + nodes++ + if typ == nil || depth > 32 || nodes > 4096 || active[typ] || output.Len() > 64<<10 { + return goAMD64FrameContext("recursive or unbounded language type") + } + active[typ] = true + defer delete(active, typ) + // Check the language role before invoking Sizes: gc Sizes may panic + // for type parameters. An unknown or generic role is not a layout. + switch concrete := typ.(type) { + case *types.Basic: + if concrete.Info()&(types.IsInteger|types.IsBoolean) == 0 || concrete.Info()&types.IsUntyped != 0 { + return goAMD64FrameContext("unsupported noninteger or untyped language type") + } + case *types.Named: + if concrete.TypeParams().Len() != 0 { + return goAMD64FrameContext("generic language type") + } + case *types.Pointer, *types.Slice, *types.Array, *types.Struct: + default: + if !goAMD64FrameIsAlias(typ) { + return goAMD64FrameContext("unknown language type role") + } + if method := reflect.ValueOf(typ).MethodByName("TypeParams"); method.IsValid() { + parameters, ok := method.Call(nil)[0].Interface().(*types.TypeParamList) + if !ok || parameters.Len() != 0 { + return goAMD64FrameContext("generic alias language type") + } + } + } + // Composite identity is written below one node at a time. Asking + // TypeString to expand an anonymous DAG defeats the node/byte bound. + spelling := "" + switch typ.(type) { + case *types.Basic, *types.Named: + spelling = types.TypeString(typ, func(pkg *types.Package) string { return pkg.Path() }) + default: + if goAMD64FrameIsAlias(typ) { + spelling = types.TypeString(typ, func(pkg *types.Package) string { return pkg.Path() }) + } + } + fmt.Fprintf(&output, "(%q:", spelling) + switch typ := typ.(type) { + case *types.Basic: + if typ.Kind() == types.Bool { + output.WriteString("bool") + } else if typ.Info()&types.IsInteger != 0 { + fmt.Fprintf(&output, "integer%d", typ.Kind()) + } else { + return goAMD64FrameContext("unsupported noninteger pointee/element") + } + case *types.Named: + if typ.TypeParams().Len() != 0 { + return goAMD64FrameContext("generic language type") + } + output.WriteString("named:") + if err := appendType(typ.Underlying(), depth+1); err != nil { + return err + } + case *types.Pointer: + output.WriteString("pointer:") + if err := appendType(typ.Elem(), depth+1); err != nil { + return err + } + case *types.Slice: + output.WriteString("slice:") + if err := appendType(typ.Elem(), depth+1); err != nil { + return err + } + case *types.Array: + fmt.Fprintf(&output, "array%d:", typ.Len()) + if err := appendType(typ.Elem(), depth+1); err != nil { + return err + } + // Go1.20's Sizes can wrap a large array to zero. Its recorded + // result is not evidence of a representable concrete layout. + element, alignment := sizes.Sizeof(typ.Elem()), sizes.Alignof(typ.Elem()) + const maxLayout = int64(1<<63 - 1) + if typ.Len() < 0 || element < 0 || alignment <= 0 || element > maxLayout-(alignment-1) { + return goAMD64FrameContext("array element layout is unrepresentable") + } + stride := (element + alignment - 1) / alignment * alignment + if typ.Len() > 0 && stride != 0 && typ.Len()-1 > (maxLayout-element)/stride { + return goAMD64FrameContext("array language layout overflow") + } + case *types.Struct: + if typ.NumFields() > 4096-nodes { + return goAMD64FrameContext("language field bound") + } + fields := make([]*types.Var, typ.NumFields()) + for i := range fields { + fields[i] = typ.Field(i) + } + // Validate every descendant before Sizes traverses the aggregate: + // generic/incomplete elements are not a concrete layout. + for _, field := range fields { + if err := appendType(field.Type(), depth+1); err != nil { + return err + } + } + offsets := sizes.Offsetsof(fields) + output.WriteString("struct:") + for i, field := range fields { + pkg := "" + if field.Pkg() != nil { + pkg = field.Pkg().Path() + } + width := sizes.Sizeof(field.Type()) + if offsets[i] < 0 || width < 0 || offsets[i] > (1<<63-1)-width { + return goAMD64FrameContext("struct language layout overflow") + } + fmt.Fprintf(&output, "%q/%q/%q/%t/%d:", pkg, field.Name(), typ.Tag(i), field.Embedded(), offsets[i]) + if output.Len() > 64<<10 { + return goAMD64FrameContext("language field metadata byte bound") + } + } + default: + // Alias is newer than Go1.20. Its actual spelling and the local + // TypeName.IsAlias role are retained alongside the complete RHS. + // Underlying alone would lose a named RHS's language identity. + if !goAMD64FrameIsAlias(typ) { + return goAMD64FrameContext("unknown language type role") + } + method := reflect.ValueOf(typ).MethodByName("Rhs") + if !method.IsValid() || method.Type().NumIn() != 0 || method.Type().NumOut() != 1 { + return goAMD64FrameContext("unknown alias RHS contract") + } + rhs, ok := method.Call(nil)[0].Interface().(types.Type) + if !ok { + return goAMD64FrameContext("missing alias RHS language type") + } + output.WriteString("alias:") + if err := appendType(rhs, depth+1); err != nil { + return err + } + } + size, align := sizes.Sizeof(typ), sizes.Alignof(typ) + if size < 0 || align <= 0 { + return goAMD64FrameContext("language type lacks a concrete target layout") + } + fmt.Fprintf(&output, ";model_size=%d,model_align=%d)", size, align) + return nil + } + if err := appendType(typ, 0); err != nil { + return "", err + } + if output.Len() > 64<<10 { + return "", goAMD64FrameContext("language type byte bound") + } + return output.String(), nil +} + +func goAMD64FrameIsAlias(typ types.Type) bool { + kind := reflect.TypeOf(typ) + return kind != nil && kind.Kind() == reflect.Pointer && kind.Elem().PkgPath() == "go/types" && kind.Elem().Name() == "Alias" +} + +func (source *GoAMD64FrameSource) preprocess(name string, original []byte) (string, error) { + if len(source.assemblerDefines) == 0 { + return "", goAMD64FrameContext("CPP lacks its actual assembler macro registration") + } + source.origins[filepath.Join(source.request.Directory, name)] = bytes.Count(original, []byte("\n")) + 1 + // The owned asm command passes this exact basename from the canonical + // package directory; stock Go can retain that relative spelling in -S. + source.origins[name] = source.origins[filepath.Join(source.request.Directory, name)] + var expand func(string, []byte) (string, error) + expand = func(name string, data []byte) (string, error) { + return PreprocessAssemblySource(string(data), AssemblyPreprocessOptions{FileName: name, Defines: source.assemblerDefines, ReadInclude: func(parent, include string) (string, []byte, error) { + if filepath.Base(include) != include || include == "go_asm.h" { + return "", nil, goAMD64FrameContext("generated/relative CPP origin requires qualified metadata") + } + file := filepath.Join(source.request.Directory, include) + if _, err := os.Stat(file); os.IsNotExist(err) { + file = filepath.Join(source.env["GOROOT"], "pkg", "include", include) + } + data, err := source.read(file) + if err != nil { + return "", nil, err + } + source.origins[file] = bytes.Count(data, []byte("\n")) + 1 + return file, data, nil + }}) + } + return expand(name, original) +} + +var goAMD64FrameHeader = regexp.MustCompile(`^(\S+) STEXT nosplit(?: asm)? size=([0-9]+) (?:align=0x[0-9a-f]+ )?args=0x([0-9a-f]+) locals=0x([0-9a-f]+)(?: funcid=0x[0-9a-f]+)?(?: align=0x[0-9a-f]+)?$`) +var goAMD64FrameRow = regexp.MustCompile(`^\s*0x([0-9a-f]+) [0-9]+ \((.+)\)\s+(.+)$`) + +func goAMD64FrameListing(listing []byte, file *File, pkg GoPackage, origins map[string]int) (map[string]goAMD64FrameFunction, error) { + return goAMD64FrameListingWithObservations(listing, file, pkg, origins, nil, nil) +} + +func goAMD64FrameListingWithNumericalObject(listing []byte, file *File, pkg GoPackage, origins map[string]int, texts []gostaticstorage.TextRecord) (map[string]goAMD64FrameFunction, error) { + return goAMD64FrameListingWithObservations(listing, file, pkg, origins, texts, nil) +} + +func goAMD64FrameListingWithWindowsUnwind(listing []byte, file *File, pkg GoPackage, origins map[string]int, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind) (map[string]goAMD64FrameFunction, error) { + return goAMD64FrameListingWithObservations(listing, file, pkg, origins, nil, unwind) +} + +// These independent, target-specific observation routes retain the strict +// default parser. A numerical relocation does not authorize Windows metadata, +// and Windows unwind metadata does not authorize a numerical memory read. +func goAMD64FrameListingWithObservations(listing []byte, file *File, pkg GoPackage, origins map[string]int, texts []gostaticstorage.TextRecord, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind) (map[string]goAMD64FrameFunction, error) { + if texts != nil && unwind != nil { + return nil, goAMD64FrameContext("mixed target-specific object observation roles") + } + return goAMD64FrameListingRecords(listing, file, pkg, origins, texts, unwind, nil) +} + +func goAMD64FrameListingWithDataObject(listing []byte, file *File, pkg GoPackage, origins map[string]int, data *gostaticstorage.AMD64AssemblyData, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind) (map[string]goAMD64FrameFunction, error) { + if data == nil || unwind != nil && !reflect.DeepEqual(data.Member.Texts, unwind.Texts) { + return nil, goAMD64FrameContext("mixed DATA/Windows complete object TEXT observations differ") + } + return goAMD64FrameListingRecords(listing, file, pkg, origins, data.Member.Texts, unwind, data) +} + +func goAMD64FrameListingRecords(listing []byte, file *File, pkg GoPackage, origins map[string]int, texts []gostaticstorage.TextRecord, unwind *gostaticstorage.WindowsAMD64AssemblyUnwind, data *gostaticstorage.AMD64AssemblyData) (map[string]goAMD64FrameFunction, error) { + functions := make(map[string]goAMD64FrameFunction) + metadata := make(map[string][]byte) + metadataHeaders := make(map[string]string) + seenMetadata := make(map[string]bool) + if unwind != nil { + for _, record := range unwind.Unwind { + if _, duplicate := metadata[record.Name]; duplicate { + return nil, goAMD64FrameContext("duplicate actual Windows metadata definition") + } + metadata[record.Name] = record.Data + header := record.Name + " SSEHUNWINDINFO dupok size=12" + if unwind.ListingAlignment { + header += " align=0x4" + } + metadataHeaders[header] = record.Name + } + } + scanner := bufio.NewScanner(bytes.NewReader(listing)) + scanner.Buffer(nil, 4<<20) + name := "" + metadataName := "" + var metadataBytes []byte + var expected int64 + textIndex, relocationIndex := -1, 0 + finishRelocations := func() error { + if texts != nil && textIndex >= 0 && relocationIndex != len(texts[textIndex].Relocations) { + return goAMD64FrameContext("missing complete original object/listing relocations") + } + return nil + } + for scanner.Scan() { + line := scanner.Text() + if header := goAMD64FrameHeader.FindStringSubmatch(line); header != nil { + if metadataName != "" { + return nil, goAMD64FrameContext("Windows listing TEXT follows its metadata inventory") + } + if err := finishRelocations(); err != nil { + return nil, err + } + if texts != nil { + textIndex++ + relocationIndex = 0 + if textIndex >= len(texts) || texts[textIndex].Name != header[1] { + return nil, goAMD64FrameContext("reordered or foreign complete object/listing TEXT") + } + } + if name != "" && int64(len(functions[name].code)) != expected { + return nil, goAMD64FrameContext("partial native symbol bytes") + } + name = header[1] + if _, duplicate := functions[name]; duplicate { + return nil, goAMD64FrameContext("duplicate native TEXT") + } + expected, _ = strconv.ParseInt(header[2], 10, 64) + args, _ := strconv.ParseInt(header[3], 16, 64) + locals, _ := strconv.ParseInt(header[4], 16, 64) + functions[name] = goAMD64FrameFunction{locals: locals, args: args, positions: make(map[int]string), sourcePositions: make(map[int][]goAMD64FrameSourcePosition)} + continue + } + if header, exists := metadataHeaders[line]; exists { + if seenMetadata[header] || metadata[header] == nil || + name != "" && int64(len(functions[name].code)) != expected || + metadataName != "" && !bytes.Equal(metadataBytes, metadata[metadataName]) { + return nil, goAMD64FrameContext("unbound, duplicate or incomplete Windows metadata listing") + } + metadataName, name, metadataBytes = header, "", nil + seenMetadata[metadataName] = true + continue + } + if metadataName != "" { + row, err := goAMD64WindowsUnwindByteRow(line, len(metadataBytes), metadata[metadataName]) + if err != nil { + return nil, err + } + metadataBytes = append(metadataBytes, row...) + continue + } + if name == "" { + return nil, goAMD64FrameContext("native listing contains unbound/unknown symbol") + } + fn := functions[name] + if strings.HasPrefix(strings.TrimSpace(line), "rel ") && texts != nil { + relocations := texts[textIndex].Relocations + if relocationIndex >= len(relocations) { + return nil, goAMD64FrameContext("extra actual object/listing relocation") + } + expectedLine := goAMD64FrameNumericalRelocationLine(relocations[relocationIndex]) + if data != nil { + expectedLine = goAMD64FrameDataRelocationLine(relocations[relocationIndex], data) + } + if expectedLine == "" || strings.Join(strings.Fields(line), " ") != expectedLine { + return nil, goAMD64FrameContext("foreign, duplicate or reordered actual object/listing relocation") + } + relocationIndex++ + continue + } + if row := goAMD64FrameRow.FindStringSubmatch(line); row != nil { + pc, err := strconv.ParseInt(row[1], 16, 32) + if err != nil || pc < 0 || pc >= expected { + return nil, goAMD64FrameContext("invalid original instruction/source PC") + } + separator := strings.LastIndexByte(row[2], ':') + if separator < 0 { + return nil, goAMD64FrameContext("unbound original instruction/source path") + } + line, err := strconv.Atoi(row[2][separator+1:]) + if err != nil || line <= 0 || line > origins[row[2][:separator]] { + return nil, goAMD64FrameContext("unbound original instruction/source line: %s", row[2]) + } + fn.positions[int(pc)] += row[2] + "\t" + row[3] + "\n" + fn.sourcePositions[int(pc)] = append(fn.sourcePositions[int(pc)], + goAMD64FrameSourcePosition{row[2][:separator], line}) + } else { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") { + return nil, goAMD64FrameContext("unknown native listing/relocation record: %s", line) + } + pc, err := strconv.ParseInt(strings.TrimPrefix(fields[0], "0x"), 16, 32) + if err != nil || pc != int64(len(fn.code)) { + return nil, goAMD64FrameContext("native byte gap/overlap") + } + count := 0 + for _, word := range fields[1:] { + if len(word) != 2 { + break + } + value, err := hex.DecodeString(word) + if err != nil { + break + } + fn.code = append(fn.code, value[0]) + count++ + } + if count == 0 || count > 16 || int64(len(fn.code)) > expected { + return nil, goAMD64FrameContext("invalid native byte record") + } + } + functions[name] = fn + } + if err := scanner.Err(); err != nil { + return nil, err + } + if err := finishRelocations(); err != nil { + return nil, err + } + if texts != nil && textIndex+1 != len(texts) { + return nil, goAMD64FrameContext("missing complete original object/listing TEXT") + } + if len(functions) != len(file.Funcs) || + metadataName == "" && (name == "" || int64(len(functions[name].code)) != expected) || + metadataName != "" && !bytes.Equal(metadataBytes, metadata[metadataName]) || + len(seenMetadata) != len(metadata) { + return nil, goAMD64FrameContext("missing/incomplete original TEXT inventory") + } + bound := make(map[string]goAMD64FrameFunction) + for _, original := range file.Funcs { + symbol := pkg.Path + "." + strings.TrimPrefix(original.Sym, "·") + observed := symbol + fn, exists := functions[observed] + if !exists { + // Older actual Go objects retain PkgIdxSelf's empty-package + // spelling. Keep those observed bytes/symbols distinct from the + // package identity independently selected and type-checked above. + observed = `"".` + strings.TrimPrefix(original.Sym, "·") + fn, exists = functions[observed] + } + declaration, declared := pkg.Types.Scope().Lookup(strings.TrimPrefix(original.Sym, "·")).(*types.Func) + if !exists || !declared || original.FrameSize <= 0 || original.FrameSize%8 != 0 || fn.locals != original.FrameSize+8 || fn.args != original.ArgSize || strings.Contains(original.Sym, "<") { + return nil, goAMD64FrameContext("original ABI0 TEXT/declaration/frame identity mismatch") + } + var err error + fn.signature, err = DeriveGoFuncABI0Signature(symbol, declaration, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + return nil, err + } + var end int64 + for _, slot := range append(append([]FrameSlot(nil), fn.signature.Frame.Params...), fn.signature.Frame.Results...) { + if next := slot.Offset + frameTypeSize(slot.Type, 8); next > end { + end = next + } + } + if !goabi.MatchesABI0TextSize("amd64", fn.args, end) { + return nil, goAMD64FrameContext("original TEXT args disagree with actual declaration") + } + for _, ins := range original.Instrs { + switch ins.Op { + case "BYTE", "WORD", "LONG", "QUAD", OpCALL, "JMP", "PCALIGN", "PCALIGNMAX": + return nil, goAMD64FrameContext("raw, calling or address-sensitive original frame requires a later source contract") + case OpRET: + if len(ins.Args) != 0 { + return nil, goAMD64FrameContext("nonordinary original RET transport") + } + } + } + if err := goAMD64FrameMachineShape(fn, original.FrameSize); err != nil { + return nil, err + } + fn.listingSymbol, fn.objectSymbol = observed, symbol + bound[symbol] = fn + } + return bound, nil +} + +func goAMD64FrameMachineShape(fn goAMD64FrameFunction, frame int64) error { + var instructions []x86asm.Inst + starts := make(map[int]bool) + for pc := 0; pc < len(fn.code); { + var ins x86asm.Inst + if x86NumericalVectorPrefix(fn.code[pc:], 64) { + length, err := x86NumericalVectorBoundary(fn.code[pc:], 64) + if err != nil || length <= 0 { + return goAMD64FrameContext("unbound original vector machine instruction: %v", err) + } + // Exact boundaries only: these cannot impersonate the scalar + // automatic prologue, epilogue, native CALL or return below. + ins.Len = length + } else { + var err error + ins, err = x86asm.Decode(fn.code[pc:], 64) + if err != nil || ins.Len == 0 || ins.Op == 0 || ins.Op == x86asm.CALL || ins.Op == x86asm.LCALL { + return goAMD64FrameContext("unknown or calling original machine instruction") + } + } + if len(fn.positions[pc]) == 0 { + return goAMD64FrameContext("machine instruction lacks its original source position") + } + starts[pc] = true + instructions = append(instructions, ins) + pc += ins.Len + } + for pc := range fn.positions { + if !starts[pc] { + return goAMD64FrameContext("original source position is not a machine instruction boundary") + } + } + if len(instructions) < 6 { + return goAMD64FrameContext("partial automatic BP/SP frame") + } + first, second, third := instructions[0], instructions[1], instructions[2] + push := first.Op == x86asm.PUSH && first.Args[0] == x86asm.RBP && + second.Op == x86asm.MOV && second.Args[0] == x86asm.RBP && second.Args[1] == x86asm.RSP && + third.Op == x86asm.SUB && third.Args[0] == x86asm.RSP && third.Args[1] == x86asm.Imm(frame) + slot := x86asm.Mem{Base: x86asm.RSP, Scale: 1, Disp: frame} + stored := first.Op == x86asm.SUB && first.Args[0] == x86asm.RSP && first.Args[1] == x86asm.Imm(frame+8) && + second.Op == x86asm.MOV && second.Args[0] == slot && second.Args[1] == x86asm.RBP && second.DataSize == 64 && + third.Op == x86asm.LEA && third.Args[0] == x86asm.RBP && third.Args[1] == slot && third.DataSize == 64 + if !push && !stored { + return goAMD64FrameContext("unrecognized automatic BP/SP prologue") + } + returns := 0 + for i, ins := range instructions { + if ins.Op == x86asm.RET { + returns++ + if i < 2 { + return goAMD64FrameContext("partial original RET epilogue") + } + penultimate, previous := instructions[i-2], instructions[i-1] + pushReturn := push && previous.Op == x86asm.POP && previous.Args[0] == x86asm.RBP && + penultimate.Op == x86asm.ADD && penultimate.Args[0] == x86asm.RSP && penultimate.Args[1] == x86asm.Imm(frame) + storedReturn := stored && penultimate.Op == x86asm.MOV && penultimate.Args[0] == x86asm.RBP && penultimate.Args[1] == slot && penultimate.DataSize == 64 && + previous.Op == x86asm.ADD && previous.Args[0] == x86asm.RSP && previous.Args[1] == x86asm.Imm(frame+8) + if !pushReturn && !storedReturn { + return goAMD64FrameContext("unrecognized original RET epilogue") + } + } + } + if returns == 0 || instructions[len(instructions)-1].Op != x86asm.RET { + return goAMD64FrameContext("original frame has no conventional RET") + } + return nil +} + +func (source *GoAMD64FrameSource) verifyObjectListing(ctx context.Context, object string, wanted map[string]goAMD64FrameFunction) error { + data, err := source.run(ctx, "tool", "objdump", object) + if err != nil { + return err + } + return goAMD64FrameObjectBytes(data, wanted) +} + +func goAMD64FrameObjectBytes(data []byte, wanted map[string]goAMD64FrameFunction) error { + actual := make(map[string][]byte) + addresses := make(map[string]uint64) + name := "" + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + if len(fields) == 3 && fields[0] == "TEXT" && strings.HasSuffix(fields[1], "(SB)") { + name = strings.TrimSuffix(fields[1], "(SB)") + if _, duplicate := actual[name]; duplicate { + return goAMD64FrameContext("duplicate actual object symbol") + } + actual[name] = nil + continue + } + if len(fields) < 4 || name == "" || !strings.HasPrefix(fields[1], "0x") { + return goAMD64FrameContext("unknown actual object disassembly record") + } + value, err := hex.DecodeString(fields[2]) + if err != nil || len(value) == 0 { + return goAMD64FrameContext("invalid actual object bytes") + } + address, err := strconv.ParseUint(strings.TrimPrefix(fields[1], "0x"), 16, 64) + if len(actual[name]) == 0 { + addresses[name] = address + } + if err != nil || address != addresses[name]+uint64(len(actual[name])) { + return goAMD64FrameContext("actual object instruction bytes have a gap/overlap") + } + actual[name] = append(actual[name], value...) + } + if len(actual) != len(wanted) { + return goAMD64FrameContext("object/listing symbol inventories differ") + } + for _, fn := range wanted { + if !bytes.Equal(actual[fn.objectSymbol], fn.code) { + return goAMD64FrameContext("object/listing physical bytes differ for %s", fn.objectSymbol) + } + } + return nil +} diff --git a/amd64_go_frame_source_ownership_test.go b/amd64_go_frame_source_ownership_test.go new file mode 100644 index 000000000..b234cacef --- /dev/null +++ b/amd64_go_frame_source_ownership_test.go @@ -0,0 +1,246 @@ +package plan9asm + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGoAMD64FrameSourceCloseRejectsReplacedOwnedDirectory(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + temporary := t.TempDir() + t.Setenv("TMPDIR", temporary) + request.Environment = append(request.Environment, "TMPDIR="+temporary, "GOTMPDIR="+temporary) + source := requireGoAMD64FrameCapture(t, pkg, request) + owned := source.owned + original, err := os.Stat(owned) + if err != nil { + t.Fatal(err) + } + // Every path here belongs to t.TempDir. The replacement models a different + // owner at the same spelling; Close must not recursively remove that owner. + moved := filepath.Join(t.TempDir(), "original-artifacts") + if err := os.Rename(owned, moved); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(owned, 0700); err != nil { + t.Fatal(err) + } + replacement, err := os.Stat(owned) + if err != nil || os.SameFile(original, replacement) { + t.Fatalf("fixture did not replace directory identity: %v", err) + } + canary := filepath.Join(owned, "replacement-owner.txt") + if err := os.WriteFile(canary, []byte("not a factory artifact\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.Close(); !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("Close did not reject changed directory identity: %v", err) + } + if _, err := os.Stat(canary); err != nil { + t.Errorf("Close removed replacement owner's file: %v", err) + } + if source.ready { + t.Error("failed cleanup retained ready observation") + } + // Restore only our known original directory for the fixture's registered + // cleanup. RemoveAll is limited to this exact t.TempDir-owned replacement. + if err := os.RemoveAll(owned); err != nil { + t.Fatal(err) + } + if err := os.Rename(moved, owned); err != nil { + t.Fatal(err) + } +} + +func TestGoAMD64FrameSourceRequestTemporaryOwnership(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + parentTemporary := t.TempDir() + requestTemporary := t.TempDir() + canonicalTemporary, err := filepath.EvalSymlinks(requestTemporary) + if err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", parentTemporary) + // Like exec.Cmd, the supplied environment's last occurrence wins. Neither + // the inherited parent TMPDIR nor the earlier supplied value is ownership. + request.Environment = append(request.Environment, + "TMPDIR="+parentTemporary, "TMPDIR="+requestTemporary, + "GOTMPDIR="+requestTemporary) + source := requireGoAMD64FrameCapture(t, pkg, request) + if filepath.Dir(source.owned) != canonicalTemporary { + t.Fatalf("factory artifacts are outside request TMPDIR: got parent %q, want %q", filepath.Dir(source.owned), canonicalTemporary) + } + if err := source.Verify(context.Background()); err != nil { + t.Fatal(err) + } + owned := source.owned + if err := source.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(owned); !os.IsNotExist(err) { + t.Fatalf("Close retained factory artifacts: %v", err) + } + for _, name := range []string{parentTemporary, requestTemporary, request.Directory} { + if _, err := os.Stat(name); err != nil { + t.Fatalf("Close removed a caller-owned directory: %v", err) + } + } +} + +func TestGoAMD64FrameSourceRepeatedEnvironmentActualValues(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "GOTMPDIR"} { + var expected string + for _, item := range request.Environment { + name, value, _ := strings.Cut(item, "=") + if name == key { + expected = value + } + } + if !filepath.IsAbs(expected) { + t.Fatalf("fixture lacks owned %s", key) + } + request.Environment = append(request.Environment, key+"="+t.TempDir(), key+"="+expected) + } + // Target and routing settings are deliberately overridden, not accepted + // from a different validation view of a repeated key. + request.Environment = append(request.Environment, "GOOS=windows", "GOOS=darwin", "GOARCH=386", "GOARCH=arm64") + source := requireGoAMD64FrameCapture(t, pkg, request) + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "GOTMPDIR"} { + var expected string + for _, item := range source.request.Environment { + name, value, _ := strings.Cut(item, "=") + if name == key { + expected = value + } + } + if source.env[key] != expected { + t.Fatalf("actual %s differs from effective supplied value: %q != %q", key, source.env[key], expected) + } + } + if source.env["GOOS"] != "linux" || source.env["GOARCH"] != "amd64" { + t.Fatalf("repeated target keys changed actual target: %v", source.env) + } +} + +func TestGoAMD64FrameSourceOwnedIdentityRejectsRedirection(t *testing.T) { + for _, kind := range []string{"equal-byte directory", "symlink", "missing"} { + t.Run(kind, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + source := requireGoAMD64FrameCapture(t, pkg, request) + owned := source.owned + moved := filepath.Join(t.TempDir(), "original-artifacts") + if err := os.Rename(owned, moved); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(owned); err != nil { + t.Error(err) + } + if err := os.Rename(moved, owned); err != nil { + t.Error(err) + } + }) + switch kind { + case "equal-byte directory": + if err := os.Mkdir(owned, 0700); err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(moved) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + data, err := os.ReadFile(filepath.Join(moved, entry.Name())) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(owned, entry.Name()), data, 0600); err != nil { + t.Fatal(err) + } + } + case "symlink": + if err := os.Symlink(moved, owned); err != nil { + t.Fatal(err) + } + } + if err := source.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) || source.ready { + t.Fatalf("changed owned directory retained observation: %v", err) + } + if err := source.Close(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Close did not reject changed ownership: %v", err) + } + if kind != "missing" { + if _, err := os.Lstat(owned); err != nil { + t.Fatalf("Close removed a replacement directory or link: %v", err) + } + } + if _, err := os.Stat(filepath.Join(moved, request.AssemblyFile+".o")); err != nil { + t.Fatalf("Close searched for or removed moved original artifacts: %v", err) + } + }) + } +} + +func TestGoAMD64FrameSourceCanonicalRootAndExplicitToolPaths(t *testing.T) { + t.Run("resolved package root is fixed", func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + original, err := filepath.EvalSymlinks(request.Directory) + if err != nil { + t.Fatal(err) + } + alias := filepath.Join(t.TempDir(), "package-alias") + if err := os.Symlink(original, alias); err != nil { + t.Fatal(err) + } + request.Directory = alias + source := requireGoAMD64FrameCapture(t, pkg, request) + if source.request.Directory != original { + t.Fatalf("capture did not bind actual canonical root: %q", source.request.Directory) + } + if err := os.Remove(alias); err != nil { + t.Fatal(err) + } + if err := os.Symlink(t.TempDir(), alias); err != nil { + t.Fatal(err) + } + if err := source.Verify(context.Background()); err != nil { + t.Fatalf("unconsumed caller alias changed fixed canonical source: %v", err) + } + }) + t.Run("Go binary symlink is not a source identity", func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + alias := filepath.Join(t.TempDir(), "go") + if err := os.Symlink(request.GoBinary, alias); err != nil { + t.Fatal(err) + } + request.GoBinary = alias + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("indirected driver produced an observation: %v", err) + } + }) +} + +func TestGoAMD64FrameSourceRepeatedForbiddenEnvironmentStaysContext(t *testing.T) { + for _, key := range []string{"GOFLAGS", "GOEXPERIMENT", "GOPACKAGESDRIVER"} { + t.Run(key, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + // Capture conservatively rejects any requested unsupported mapping, + // including one shadowed by a later empty occurrence. + request.Environment = append(request.Environment, key+"=unbound", key+"=") + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unsupported duplicate %s mapping was accepted: %v", key, err) + } + }) + } +} diff --git a/amd64_go_frame_source_test.go b/amd64_go_frame_source_test.go new file mode 100644 index 000000000..665263b9a --- /dev/null +++ b/amd64_go_frame_source_test.go @@ -0,0 +1,522 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +func requireGoAMD64FrameCapture(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest) *GoAMD64FrameSource { + t.Helper() + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + return source +} + +func goAMD64FrameSourceFixture(t *testing.T) (GoPackage, GoAMD64FrameCaptureRequest) { + t.Helper() + // Keep AST positions, the caller's request and strict original-file reads + // bound to the same physical fixture, including aliased temporary roots. + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal("cannot resolve original frame fixture directory", err) + } + source := []byte("package framefixture\nfunc Value(a uint64) uint64\n") + files := map[string][]byte{ + "go.mod": []byte("module example.invalid/framefixture\ngo 1.20\n"), + "value.go": source, + "value_amd64.s": []byte("#include \"textflag.h\"\nTEXT ·Value(SB),NOSPLIT,$160-16\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n"), + } + for name, data := range files { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, filepath.Join(dir, "value.go"), source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + pkg, err := (&types.Config{Sizes: sizes}).Check("example.invalid/framefixture", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + binary, err := exec.LookPath("go") + if err != nil { + t.Fatal(err) + } + binary, err = filepath.Abs(binary) + if err != nil { + t.Fatal(err) + } + binary, err = filepath.EvalSymlinks(binary) + if err != nil { + t.Fatal(err) + } + // This independent pure-Go module does not inherit the translator test + // binary's build tags (notably Windows' LLVM byollvm tag). Remove rather + // than shadow them: production capture rejects repeated forbidden flags. + var env []string + for _, entry := range os.Environ() { + if !strings.HasPrefix(entry, "GOFLAGS=") { + env = append(env, entry) + } + } + env = append(env, "GOFLAGS=") + for _, key := range []string{"GOPATH", "GOMODCACHE", "GOCACHE", "TMPDIR", "GOTMPDIR"} { + if !filepath.IsAbs(os.Getenv(key)) { + env = append(env, key+"="+t.TempDir()) + } + } + return GoPackage{Path: pkg.Path(), Types: pkg, Syntax: []*ast.File{syntax}}, GoAMD64FrameCaptureRequest{ + GoBinary: binary, Directory: dir, AssemblyFile: "value_amd64.s", Target: "linux/amd64", Environment: env, + } +} + +func TestGoAMD64FrameSourceActualCaptureAndSealedLifecycle(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + // Independent original-Go acceptance precedes the proposed token. This + // object observation is not itself a native-memory authorization. + raw, err := os.ReadFile(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + listing := x86InsertExtractGoListing(t, "amd64", strings.Replace(string(raw), "#include \"textflag.h\"", "#define NOSPLIT 4", 1)) + t.Logf("actual stock-Go original nonzero TEXT listing:\n%s", listing) + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } + if len(source.functions) != 1 || source.functions["example.invalid/framefixture.Value"].locals != 168 { + t.Fatalf("missing actual Go frame witness: %+v", source.functions) + } + encoded, err := json.Marshal(source) + if err != nil { + t.Fatal(err) + } + var restored GoAMD64FrameSource + if err := json.Unmarshal(encoded, &restored); err != nil { + t.Fatal(err) + } + if !errors.Is(restored.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("JSON recreated private frame authority") + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + if !errors.Is(source.Verify(ctx), ErrProbeNeedsContext) { + t.Fatal("closed witness retained authority") + } +} + +func TestGoAMD64FrameSourceRefusesSelfReportedSelection(t *testing.T) { + for _, mutation := range []string{"wrong types", "missing sibling", "dependency", "generated header", "source changed", "same width signedness", "target", "custom flags", "conditional CPP", "raw RET", "zero frame", "pointer", "unknown tool"} { + t.Run(mutation, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + switch mutation { + case "wrong types": + pkg.Types = types.NewPackage(pkg.Path, "forged") + case "missing sibling": + if err := os.WriteFile(filepath.Join(request.Directory, "sibling_amd64.s"), []byte("TEXT ·Other(SB),4,$8-0\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + case "dependency": + if err := os.WriteFile(filepath.Join(request.Directory, "dep.go"), []byte("package framefixture\nimport _ \"unsafe\"\n"), 0600); err != nil { + t.Fatal(err) + } + case "generated header": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("#include \"go_asm.h\"\nTEXT ·Value(SB),4,$160-16\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + case "source changed": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a uint32) uint32\n"), 0600); err != nil { + t.Fatal(err) + } + case "target": + request.Target = "linux/386" + case "same width signedness": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a int64) int64\n"), 0600); err != nil { + t.Fatal(err) + } + case "custom flags": + request.Environment = append(request.Environment, "GOFLAGS=-tags=custom") + case "conditional CPP": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("#ifdef GOARCH_amd64\n#include \"go_asm.h\"\nTEXT ·Value(SB),4,$160-16\nRET\n#endif\n"), 0600); err != nil { + t.Fatal(err) + } + case "raw RET": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("TEXT ·Value(SB),4,$160-16\nBYTE $0xc3\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + case "zero frame": + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte("TEXT ·Value(SB),4,$0-16\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + case "pointer": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a *uint64) *uint64\n"), 0600); err != nil { + t.Fatal(err) + } + case "unknown tool": + request.GoBinary = filepath.Join(request.Directory, "not-a-go-driver") + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + proof, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if proof != nil { + proof.Close() + t.Fatal("unsupported/incomplete source produced token") + } + if err == nil { + t.Fatal("self-reported typed selection was accepted") + } + }) + } +} + +func TestGoAMD64FrameSourceActualTargetAndFullSiblingInventory(t *testing.T) { + for _, target := range []string{"linux/amd64", "darwin/amd64", "windows/amd64"} { + t.Run(target, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = target + data, err := os.ReadFile(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(request.Directory, "sibling_amd64.s"), bytes.ReplaceAll(data, []byte("Value"), []byte("Other")), 0600); err != nil { + t.Fatal(err) + } + declarations := []byte("package framefixture\nfunc Value(a uint64) uint64\nfunc Other(a uint64) uint64\n") + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), declarations, 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "value.go", declarations, 0) + if err != nil { + t.Fatal(err) + } + pkg.Types, err = (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check(pkg.Path, fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if target == "windows/amd64" && errors.Is(err, ErrProbeNeedsContext) && strings.Contains(err.Error(), "SSEHUNWINDINFO") { + if source != nil { + t.Fatal("unsupported generated unwind role returned a token") + } + t.Logf("actual Windows auxiliary object role remains unbound: %v", err) + return + } + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + if len(source.selection.SFiles) != 2 || len(source.functions) != 2 || source.env["GOOS"]+"/"+source.env["GOARCH"] != target { + t.Fatalf("incomplete actual target/sibling observation: %+v", source.selection) + } + }) + } +} + +func TestGoAMD64FrameSourceInputMutationPermanentlyRevokes(t *testing.T) { + for _, kind := range []string{"Go", "ASM", "unused header", "include", "inventory", "object", "listing", "cancel"} { + t.Run(kind, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + header := filepath.Join(request.Directory, "fixture.h") + if err := os.WriteFile(header, []byte("#define FRAME 160\n"), 0600); err != nil { + t.Fatal(err) + } + if kind == "include" { + assembly := filepath.Join(request.Directory, request.AssemblyFile) + original, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + original = append([]byte("#include \"fixture.h\"\n"), bytes.Replace(original, []byte("$160-16"), []byte("$FRAME-16"), 1)...) + if err := os.WriteFile(assembly, original, 0600); err != nil { + t.Fatal(err) + } + } + source := requireGoAMD64FrameCapture(t, pkg, request) + if kind == "include" && source.origins[filepath.Join(source.request.Directory, "fixture.h")] == 0 { + t.Fatal("active original CPP header was not captured") + } + filename := filepath.Join(source.request.Directory, "value.go") + switch kind { + case "ASM": + filename = filepath.Join(source.request.Directory, request.AssemblyFile) + case "unused header", "include": + filename = filepath.Join(source.request.Directory, "fixture.h") + case "inventory": + filename = filepath.Join(source.request.Directory, "new.h") + case "object": + filename = filepath.Join(source.owned, request.AssemblyFile+".o") + case "listing": + filename = filepath.Join(source.owned, request.AssemblyFile+".o.listing") + } + ctx := context.Background() + if kind == "cancel" { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } else if err := os.WriteFile(filename, []byte("changed\n"), 0600); err != nil { + t.Fatal(err) + } + if err := source.Verify(ctx); !errors.Is(err, ErrProbeNeedsContext) || source.ready { + t.Fatalf("changed input retained observation: %v", err) + } + if !errors.Is(source.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatal("revoked observation was reopened") + } + }) + } +} + +func TestGoAMD64FrameSourceObjectListingMutations(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + source := requireGoAMD64FrameCapture(t, pkg, request) + listing, err := os.ReadFile(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(source.request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(request.AssemblyFile, raw) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + fn := source.functions[pkg.Path+".Value"] + var pcs []int + for pc := range fn.positions { + if pc > 0 { + pcs = append(pcs, pc) + } + } + sort.Ints(pcs) + for _, change := range []string{"source path", "source line", "interior PC", "partial bytes", "frame", "symbol", "duplicate", "relocation"} { + t.Run(change, func(t *testing.T) { + mutated := append([]byte(nil), listing...) + switch change { + case "source path": + mutated = bytes.ReplaceAll(mutated, []byte(request.AssemblyFile), []byte("unbound.s")) + case "source line": + mutated = bytes.ReplaceAll(mutated, []byte(".s:2)"), []byte(".s:999999)")) + case "interior PC": + before := fmt.Sprintf("0x%04x %05d", pcs[0], pcs[0]) + after := fmt.Sprintf("0x%04x %05d", pcs[0]+1, pcs[0]+1) + mutated = bytes.Replace(mutated, []byte(before), []byte(after), 1) + case "partial bytes": + mutated = mutated[:bytes.LastIndex(mutated, []byte("\n\t0x"))+1] + case "frame": + mutated = bytes.Replace(mutated, []byte("locals=0xa8"), []byte("locals=0xa0"), 1) + case "symbol": + mutated = bytes.ReplaceAll(mutated, []byte(fn.listingSymbol), []byte("forged.Value")) + case "duplicate": + mutated = append(mutated, listing...) + case "relocation": + mutated = append(mutated, []byte("rel 0+4 t=R_CALL forged\n")...) + } + if _, err := goAMD64FrameListing(mutated, file, pkg, source.origins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mutated %s listing accepted: %v", change, err) + } + }) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + object := filepath.Join(source.owned, request.AssemblyFile+".o") + disassembly, err := source.run(ctx, "tool", "objdump", object) + if err != nil { + t.Fatal(err) + } + if err := goAMD64FrameObjectBytes(disassembly, source.functions); err != nil { + t.Fatal(err) + } + for _, change := range []string{"bytes", "symbol", "missing", "gap", "extra"} { + t.Run("object "+change, func(t *testing.T) { + mutated := append([]byte(nil), disassembly...) + switch change { + case "bytes": + rows := strings.Split(string(mutated), "\n") + fields := strings.Fields(rows[1]) + rows[1] = strings.Replace(rows[1], fields[2], "90", 1) + mutated = []byte(strings.Join(rows, "\n")) + case "symbol": + mutated = bytes.ReplaceAll(mutated, []byte(fn.objectSymbol), []byte("forged.Value")) + case "missing": + mutated = mutated[:bytes.LastIndex(mutated, []byte("\n "))+1] + case "gap": + rows := strings.Split(string(mutated), "\n") + fields := strings.Fields(rows[2]) + rows[2] = strings.Replace(rows[2], fields[1], "0x0", 1) + mutated = []byte(strings.Join(rows, "\n")) + case "extra": + mutated = append(mutated, disassembly...) + } + if err := goAMD64FrameObjectBytes(mutated, source.functions); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mutated %s object accepted: %v", change, err) + } + }) + } +} + +func TestGoAMD64FrameSourceConcurrentVerifyClose(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + source := requireGoAMD64FrameCapture(t, pkg, request) + owned := source.owned + var workers sync.WaitGroup + for i := 0; i < 3; i++ { + workers.Add(1) + go func() { + defer workers.Done() + if err := source.Verify(context.Background()); err != nil && !errors.Is(err, ErrProbeNeedsContext) { + t.Error(err) + } + }() + } + if err := source.Close(); err != nil { + t.Fatal(err) + } + workers.Wait() + if _, err := os.Stat(owned); !os.IsNotExist(err) { + t.Fatalf("factory-owned artifacts survived Close: %v", err) + } + if _, err := os.Stat(filepath.Join(request.Directory, request.AssemblyFile)); err != nil { + t.Fatalf("Close removed original source: %v", err) + } +} + +func TestGoAMD64FrameSourceFailedSiblingCleansOnlyOwnedObjects(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + temporary := t.TempDir() + t.Setenv("TMPDIR", t.TempDir()) + request.Environment = append(request.Environment, "TMPDIR="+temporary, "GOTMPDIR="+temporary) + // The first object succeeds. The later selected sibling cannot be bound + // to any actual declaration; its failure must not leak the earlier object. + sibling := filepath.Join(request.Directory, "z_sibling_amd64.s") + if err := os.WriteFile(sibling, []byte("TEXT ·Other(SB),4,$160-0\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("partial complete-file observation accepted: %v", err) + } + entries, err := os.ReadDir(temporary) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "plan9asm-go-frame-object-") { + t.Fatalf("partial capture leaked %s", entry.Name()) + } + } + if _, err := os.Stat(sibling); err != nil { + t.Fatalf("cleanup removed an original selected source: %v", err) + } +} + +func TestGoAMD64FrameSourceRejectsIndirectedOriginalAndCacheFallback(t *testing.T) { + for _, change := range []string{"source symlink", "package subtree", "implicit cache"} { + t.Run(change, func(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + switch change { + case "source symlink": + target := filepath.Join(t.TempDir(), "source.go") + if err := os.WriteFile(target, []byte("package framefixture\nfunc Value(a uint64) uint64\n"), 0600); err != nil { + t.Fatal(err) + } + original := filepath.Join(request.Directory, "value.go") + if err := os.Remove(original); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, original); err != nil { + t.Fatal(err) + } + case "package subtree": + if err := os.Mkdir(filepath.Join(request.Directory, "nested"), 0700); err != nil { + t.Fatal(err) + } + case "implicit cache": + request.Environment = nil + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unsupported original source/cache origin captured: %v", err) + } + }) + } +} + +func TestGoAMD64FrameSourceRejectsSelectedSystemObjects(t *testing.T) { + pkg, request := goAMD64FrameSourceFixture(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + // A genuine Go object renamed .syso is still selected by the actual driver + // and packed into its export. It must not be silently treated as an ASM/H + // source role merely because the directory and file bytes were hashed. + env := append(append([]string(nil), request.Environment...), "GOOS=linux", "GOARCH=amd64", "GOTOOLCHAIN=local") + object := filepath.Join(request.Directory, "system_amd64.syso") + include := filepath.Join(filepath.Dir(filepath.Dir(request.GoBinary)), "pkg", "include") + if _, err := goasmcmd.Run(ctx, request.Directory, env, request.GoBinary, "tool", "asm", "-p", pkg.Path, "-I", include, "-o", object, request.AssemblyFile); err != nil { + t.Fatal(err) + } + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil { + if closeErr := source.Close(); closeErr != nil { + t.Fatal(closeErr) + } + t.Fatal("actual selected .syso role was omitted from complete observation") + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown system-object transport did not retain Context: %v", err) + } +} diff --git a/amd64_go_frame_standard_imports.go b/amd64_go_frame_standard_imports.go new file mode 100644 index 000000000..2736f1d56 --- /dev/null +++ b/amd64_go_frame_standard_imports.go @@ -0,0 +1,149 @@ +package plan9asm + +import ( + "bytes" + "context" + "encoding/json" + "io" + "path/filepath" + "reflect" +) + +// SDK imports supply actual compiler types, never caller-memory permissions. +// Exact downloaded modules have their own independent archive handoff; +// nonordinary compiler roles remain separate. +func (source *GoAMD64FrameSource) loadOriginalDeclarationImports(ctx context.Context) (map[string]goAMD64FrameSelection, error) { + if len(source.selection.Imports) == 0 { + return nil, nil + } + output, err := source.run(ctx, "list", "-deps", "-compiled", "-export", "-json", "-mod=readonly", ".") + if err != nil { + return nil, err + } + if err := source.captureDownloadedDeclarationOrigins(ctx, output); err != nil { + return nil, err + } + return source.bindOriginalDeclarationImports(output) +} + +// A caller retaining the actual command bytes may share the same complete +// graph decoder. This routine observes files/exports, not frame authority. +func (source *GoAMD64FrameSource) bindOriginalDeclarationImports(output []byte) (map[string]goAMD64FrameSelection, error) { + if len(output) > 16<<20 { + return nil, goAMD64FrameContext("declaration import graph exceeds its bound") + } + root, err := filepath.EvalSymlinks(source.env["GOROOT"]) + if err != nil { + return nil, err + } + cache, err := filepath.EvalSymlinks(source.env["GOCACHE"]) + if err != nil { + return nil, err + } + graph := make(map[string]goAMD64FrameSelection) + decoder := json.NewDecoder(bytes.NewReader(output)) + var originalFound bool + var inputBytes, inputFiles int + for { + var selected goAMD64FrameSelection + if err := decoder.Decode(&selected); err == io.EOF { + break + } else if err != nil { + return nil, err + } + if selected.ImportPath == "" || len(graph) >= 4096 || + len(selected.Error)+len(selected.DepsErrors) != 0 || selected.ForTest != "" || selected.nonordinaryRole() { + return nil, goAMD64FrameContext("incomplete or nonordinary declaration import graph") + } + if selected.ImportPath == source.selection.ImportPath { + selected.Export, err = filepath.EvalSymlinks(selected.Export) + if err != nil || originalFound || !reflect.DeepEqual(selected, source.selection) { + return nil, goAMD64FrameContext("declaration graph lost the actual original package") + } + originalFound = true + continue + } + if _, duplicate := graph[selected.ImportPath]; duplicate || + !filepath.IsLocal(filepath.FromSlash(selected.ImportPath)) { + return nil, goAMD64FrameContext("foreign or duplicate declaration dependency needs authenticated handoff") + } + dir, err := filepath.EvalSymlinks(selected.Dir) + var wanted string + var wantedErr error + if selected.Standard && selected.Module == nil { + wanted, wantedErr = filepath.EvalSymlinks(filepath.Join(root, "src", filepath.FromSlash(selected.ImportPath))) + } else { + wanted, wantedErr = source.downloadedDeclarationDirectory(selected) + } + if err != nil || wantedErr != nil || dir != wanted { + return nil, goAMD64FrameContext("actual declaration dependency escaped its original source origin: %v", wantedErr) + } + compiled := append([]string(nil), selected.CompiledGoFiles...) + for index, file := range compiled { + if filepath.IsAbs(file) { + relative, err := filepath.Rel(dir, file) + if err != nil || filepath.Base(relative) != relative { + return nil, goAMD64FrameContext("generated dependency compiler sources are unbound") + } + compiled[index] = relative + } + } + if selected.ImportPath == "unsafe" { + compiled = selected.GoFiles // This builtin package has no compiled export. + } + if len(selected.GoFiles) == 0 || !reflect.DeepEqual(compiled, selected.GoFiles) { + return nil, goAMD64FrameContext("declaration dependency omitted ordinary compiler sources") + } + for _, files := range [][]string{selected.GoFiles, selected.SFiles, selected.HFiles} { + for _, file := range files { + if filepath.Base(file) != file { + return nil, goAMD64FrameContext("declaration dependency source is not a package-local file") + } + data, err := source.read(filepath.Join(dir, file)) + if err != nil { + return nil, err + } + inputBytes += len(data) + inputFiles++ + if inputBytes > 64<<20 || inputFiles > 32768 { + return nil, goAMD64FrameContext("complete declaration dependency inputs exceed their bound") + } + } + } + if selected.ImportPath != "unsafe" { + selected.Export, err = filepath.EvalSymlinks(selected.Export) + if err != nil || !goAssemblyPathWithinRoot(cache, selected.Export) { + return nil, goAMD64FrameContext("declaration compiler export is outside its actual cache") + } + data, err := source.read(selected.Export) + if err != nil || len(data) == 0 { + return nil, goAMD64FrameContext("declaration compiler export is empty or changed: %v", err) + } + } + graph[selected.ImportPath] = selected + } + if !originalFound { + return nil, goAMD64FrameContext("declaration graph omitted its original package") + } + for _, selected := range append([]goAMD64FrameSelection{source.selection}, frameImportGraphValues(graph)...) { + for _, path := range selected.Imports { + if _, present := graph[path]; !present { + return nil, goAMD64FrameContext("declaration graph omitted an actual import") + } + } + for _, target := range selected.ImportMap { + if _, present := graph[target]; !present { + return nil, goAMD64FrameContext("declaration graph omitted an actual import mapping") + } + } + } + return graph, nil +} + +func frameImportGraphValues(graph map[string]goAMD64FrameSelection) []goAMD64FrameSelection { + values := make([]goAMD64FrameSelection, 0, len(graph)) + for _, selected := range graph { + values = append(values, selected) + } + return values +} diff --git a/amd64_go_frame_typed_source_test.go b/amd64_go_frame_typed_source_test.go new file mode 100644 index 000000000..5509ff271 --- /dev/null +++ b/amd64_go_frame_typed_source_test.go @@ -0,0 +1,490 @@ +package plan9asm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func goAMD64FrameTypedFixture(t *testing.T, declaration, assembly string) (GoPackage, GoAMD64FrameCaptureRequest) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + source := []byte("package framefixture\n" + declaration + "\n") + filename := filepath.Join(request.Directory, "value.go") + if err := os.WriteFile(filename, source, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(request.Directory, request.AssemblyFile), []byte(assembly), 0600); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, filename, source, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check("example.invalid/framefixture", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: typed.Path(), Types: typed, Syntax: []*ast.File{file}}, request +} + +func TestGoAMD64FrameTypedSourceSliceObservation(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte, n uint64) uint64", "#include \"textflag.h\"\nTEXT ·Value(SB),NOSPLIT,$160-40\nMOVQ n+24(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+32(FP)\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + fn := source.functions[pkg.Path+".Value"] + if fn.args != 40 || fn.locals != 168 || len(fn.signature.Frame.Params) != 4 || fn.signature.Frame.Results[0].Offset != 32 { + t.Fatalf("actual slice observation lost complete original ABI0 frame: %+v", fn) + } + if fn.transport != goAMD64FrameCompositeTransport || fn.languageType == "" { + t.Fatal("slice observation lost its nonnumeric transport role") + } + for i, slot := range fn.signature.Frame.Params[:3] { + if slot.Offset != int64(i*8) || slot.Index != 0 || slot.Field != i { + t.Fatalf("slice field %d has an invented offset/identity: %+v", i, slot) + } + } + if !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("slice observation entered the numeric-only fixed-local contract") + } +} + +func TestGoAMD64FrameTypedSourceActualLayoutMatrix(t *testing.T) { + for _, test := range []struct { + name, declaration string + args int64 + params, results []int64 + role goAMD64FrameTransport + }{ + {"integer widths", "func Value(a int8, b uint8, c int16, d uint16, e int32, f uint32, g int64, h uint64, i int, j uint, k uintptr) (r uint64)", 64, []int64{0, 1, 2, 4, 8, 12, 16, 24, 32, 40, 48}, []int64{56}, goAMD64FrameNumericTransport}, + {"named integer", "type Count uint64\nfunc Value(a Count) Count", 16, []int64{0}, []int64{8}, goAMD64FrameNumericTransport}, + {"integer alias", "type Count = uint64\nfunc Value(a Count) Count", 16, []int64{0}, []int64{8}, goAMD64FrameNumericTransport}, + {"pointer", "func Value(a *uint64, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"pointer to array", "func Value(a *[64]byte, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"nested fields", "type Packet struct { Tag byte; Samples [3]int16; Tail uint64 }\nfunc Value(a *Packet, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"nested pointer fields", "type Packet struct { P *[4]uint16; S []uint32 }\nfunc Value(a *Packet, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"named pointer", "type Pointer *uint64\nfunc Value(a Pointer, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"pointer alias", "type Pointer = *uint64\nfunc Value(a Pointer, n uint64) uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"slice integer", "func Value(a []int32, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"slice pointer", "func Value(a []*uint64, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"named slice", "type Bytes []byte\nfunc Value(a Bytes, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"slice alias", "type Bytes = []byte\nfunc Value(a Bytes, n uint64) uint64", 40, []int64{0, 8, 16, 24}, []int64{32}, goAMD64FrameCompositeTransport}, + {"pointer result", "func Value(a *uint64, n uint64) *uint64", 24, []int64{0, 8}, []int64{16}, goAMD64FrameCompositeTransport}, + {"slice result", "func Value(a []byte, n uint64) []byte", 56, []int64{0, 8, 16, 24}, []int64{32, 40, 48}, goAMD64FrameCompositeTransport}, + {"mixed results", "func Value(a *uint64, n int32) (p *uint64, count uint32)", 32, []int64{0, 8}, []int64{16, 24}, goAMD64FrameCompositeTransport}, + } { + t.Run(test.name, func(t *testing.T) { + // Object capture does not execute the body or claim initialized + // results/GC safety. The independent ABI0 offsets are the oracle. + assembly := fmt.Sprintf("TEXT ·Value(SB),4,$160-%d\nRET\n", test.args) + pkg, request := goAMD64FrameTypedFixture(t, test.declaration, assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + fn := source.functions[pkg.Path+".Value"] + if fn.args != test.args || fn.locals != 168 || fn.transport != test.role || fn.languageType == "" { + t.Fatalf("actual object/transport mismatch: %+v", fn) + } + check := func(slots []FrameSlot, offsets []int64) { + t.Helper() + if len(slots) != len(offsets) { + t.Fatalf("complete ABI0 slots differ: %+v; want %v", slots, offsets) + } + for i, slot := range slots { + if slot.Offset != offsets[i] { + t.Fatalf("ABI0 slot %d is %+v; want offset %d", i, slot, offsets[i]) + } + } + } + check(fn.signature.Frame.Params, test.params) + check(fn.signature.Frame.Results, test.results) + object := filepath.Join(source.owned, request.AssemblyFile+".o") + t.Logf("actual stock-Go observation: target=%s object_sha=%x listing_sha=%x code_bytes=%d args=%d locals=%d transport=%d", request.Target, source.inputs[object], source.inputs[object+".listing"], len(fn.code), fn.args, fn.locals, fn.transport) + err := source.numericDeclarations(pkg) + if test.role == goAMD64FrameNumericTransport && err != nil || test.role != goAMD64FrameNumericTransport && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("numeric-only consumer role mismatch: %v", err) + } + }) + } +} + +func goAMD64FrameTypedSelection(t *testing.T, pkg GoPackage, request GoAMD64FrameCaptureRequest) *GoAMD64FrameSource { + t.Helper() + // The production capture entry resolves the source root before binding + // selection paths. This internal fixture must exercise that same contract, + // including temporary-directory aliases supplied by the host OS. + canonical, err := filepath.EvalSymlinks(request.Directory) + if err != nil { + t.Fatal(err) + } + request.Directory = canonical + return &GoAMD64FrameSource{ + request: request, inputs: make(map[string][32]byte), + selection: goAMD64FrameSelection{GoFiles: []string{"value.go"}, CompiledGoFiles: []string{filepath.Join(request.Directory, "value.go")}}, + } +} + +func TestGoAMD64FrameTypedSelectionCanonicalSourceRoot(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a *uint64) uint64", "TEXT ·Value(SB),4,$160-16\nRET\n") + canonical, err := filepath.EvalSymlinks(request.Directory) + if err != nil { + t.Fatal(err) + } + source := goAMD64FrameTypedSelection(t, pkg, request) + if source.request.Directory != canonical || len(source.selection.CompiledGoFiles) != 1 || + source.selection.CompiledGoFiles[0] != filepath.Join(canonical, "value.go") { + t.Fatal("typed fixture did not bind the production entry's canonical source root") + } + if _, err := source.typedDeclarations(pkg, false); err != nil { + t.Fatalf("canonical source must reach the independent language check: %v", err) + } +} + +func TestGoAMD64FrameTypedSourceLanguageIdentityMutations(t *testing.T) { + for _, test := range []struct{ name, original, mutation string }{ + {"same pointer width", "func Value(a *uint64) uint64", "func Value(a *int64) uint64"}, + {"slice element", "func Value(a []byte) uint64", "func Value(a []uint16) uint64"}, + {"slice versus pointer", "func Value(a []byte) uint64", "func Value(a *byte) uint64"}, + {"array length", "func Value(a *[16]byte) uint64", "func Value(a *[32]byte) uint64"}, + {"named underlying", "type Count uint64\nfunc Value(a *Count) uint64", "type Count int64\nfunc Value(a *Count) uint64"}, + {"alias versus named", "type Count = uint64\nfunc Value(a *Count) uint64", "type Count uint64\nfunc Value(a *Count) uint64"}, + {"alias RHS identity", "type A uint64\ntype B uint64\ntype Choice = A\nfunc Value(a *Choice) uint64", "type A uint64\ntype B uint64\ntype Choice = B\nfunc Value(a *Choice) uint64"}, + {"field signedness", "type Packet struct { X uint64 }\nfunc Value(a *Packet) uint64", "type Packet struct { X int64 }\nfunc Value(a *Packet) uint64"}, + {"field identity", "type Packet struct { X uint64 }\nfunc Value(a *Packet) uint64", "type Packet struct { Y uint64 }\nfunc Value(a *Packet) uint64"}, + {"field order", "type Packet struct { X byte; Y uint64 }\nfunc Value(a *Packet) uint64", "type Packet struct { Y uint64; X byte }\nfunc Value(a *Packet) uint64"}, + {"field tag", "type Packet struct { X uint64 `a` }\nfunc Value(a *Packet) uint64", "type Packet struct { X uint64 `b` }\nfunc Value(a *Packet) uint64"}, + {"embedded identity", "type Count uint64\ntype Packet struct { Count }\nfunc Value(a *Packet) uint64", "type Count uint64\ntype Packet struct { Count Count }\nfunc Value(a *Packet) uint64"}, + {"parameter name", "func Value(a *uint64) uint64", "func Value(b *uint64) uint64"}, + {"result pointee", "func Value(a *uint64) *uint64", "func Value(a *uint64) *int64"}, + { + "unrelated interface does not hide signedness", + "type XOF interface { Clone() XOF }\nfunc Value(a *uint64) uint64", + "type XOF interface { Clone() XOF }\nfunc Value(a *int64) uint64", + }, + { + "unrelated recursive type does not hide extent", + "type Node struct { Next *Node }\nfunc Value(a *[16]byte) uint64", + "type Node struct { Next *Node }\nfunc Value(a *[32]byte) uint64", + }, + { + "unrelated callback does not hide alias RHS", + "type Callback func()\ntype A uint64\ntype B uint64\ntype Choice = A\nfunc Value(a *Choice) uint64", + "type Callback func()\ntype A uint64\ntype B uint64\ntype Choice = B\nfunc Value(a *Choice) uint64", + }, + } { + t.Run(test.name, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, test.original, "TEXT ·Value(SB),4,$160-16\nRET\n") + guard := goAMD64FrameTypedSelection(t, pkg, request) + if _, err := guard.typedDeclarations(pkg, false); err != nil { + t.Fatalf("unmutated independent typecheck failed: %v", err) + } + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\n"+test.mutation+"\n"), 0600); err != nil { + t.Fatal(err) + } + guard.inputs = make(map[string][32]byte) + if _, err := guard.typedDeclarations(pkg, false); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("stale supplied type tree accepted actual source mutation: %v", err) + } + }) + } +} + +func TestGoAMD64FrameTypedSourceUnsupportedRolesRemainContext(t *testing.T) { + for _, declaration := range []string{ + "func Value(a float32) uint64", "func Value(a complex128) uint64", + "func Value(a string) uint64", "func Value(a *float64) uint64", + "func Value(a map[uint64]uint64) uint64", "func Value(a chan uint64) uint64", + "func Value(a func(uint64) uint64) uint64", "func Value(a interface{}) uint64", + "func Value(a [2]uint64) uint64", "func Value(a struct { X uint64 }) uint64", + "type Node struct { Next *Node }\nfunc Value(a *Node) uint64", + "type Node[T any] struct { Value T }\nfunc Value(a *Node[uint64]) uint64", + "func Value[T ~uint64](a T) uint64 { return uint64(a) }", "func Value(a ...uint64) uint64", + "func Value(a *uint64) uint64 { return *a }", + } { + t.Run(declaration, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, declaration, "TEXT ·Value(SB),4,$160-16\nRET\n") + guard := goAMD64FrameTypedSelection(t, pkg, request) + _, err := guard.typedDeclarations(pkg, false) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown/GC/aggregate role did not retain Context: %v", err) + } + if strings.Contains(err.Error(), "input path is redirected or inaccessible") { + t.Fatalf("path rejection cannot prove an unsupported language role: %v", err) + } + }) + } +} + +func TestGoAMD64FrameTypedSourceOrdinaryPointerValueIsObservationOnly(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, + "var P *uint64\nfunc Value(a *uint64) uint64", + "TEXT ·Value(SB),4,$160-16\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + if len(source.functions) != 1 || len(source.declarations) != 1 { + t.Fatal("ordinary pointer value created an additional frame capability") + } + function := source.functions[pkg.Path+".Value"] + if function.transport != goAMD64FrameCompositeTransport || function.args != 16 { + t.Fatalf("original pointer declaration lost its separate observation role: %+v", function) + } + if !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("ordinary pointer value or declaration entered numeric-only fixed-local admission") + } +} + +func TestGoAMD64FrameTypedSourceActualCaptureRejectsStalePointee(t *testing.T) { + for _, test := range []struct{ original, mutation string }{ + {"func Value(a *uint64) uint64", "func Value(a *int64) uint64"}, + {"type Packet struct { X [4]byte }\nfunc Value(a *Packet) uint64", "type Packet struct { X [8]byte }\nfunc Value(a *Packet) uint64"}, + {"type A uint64\ntype B uint64\ntype Choice = A\nfunc Value(a *Choice) uint64", "type A uint64\ntype B uint64\ntype Choice = B\nfunc Value(a *Choice) uint64"}, + } { + t.Run(test.mutation, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, test.original, "TEXT ·Value(SB),4,$160-16\nRET\n") + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\n"+test.mutation+"\n"), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if source != nil || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual compiler export accepted stale supplied language identity: %v", err) + } + }) + } +} + +func TestGoAMD64FrameTypedSourceLanguageBounds(t *testing.T) { + sizes := types.SizesFor("gc", "amd64") + constraint := types.NewInterfaceType(nil, nil).Complete() + parameter := types.NewTypeParam(types.NewTypeName(token.NoPos, nil, "T", nil), constraint) + for _, typ := range []types.Type{nil, parameter, types.Typ[types.UntypedInt], types.Typ[types.Float64], types.NewMap(types.Typ[types.Uint64], types.Typ[types.Uint64])} { + if _, err := goAMD64FrameLanguageType(typ, sizes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown language role did not fail closed: %v", err) + } + } + deep := types.Type(types.Typ[types.Uint64]) + for i := 0; i < 34; i++ { + deep = types.NewPointer(deep) + } + fields := make([]*types.Var, 4097) + for i := range fields { + fields[i] = types.NewVar(token.NoPos, nil, fmt.Sprintf("F%d", i), types.Typ[types.Uint64]) + } + for _, typ := range []types.Type{deep, types.NewStruct(fields, nil), types.NewArray(types.Typ[types.Uint64], 1<<62)} { + if _, err := goAMD64FrameLanguageType(typ, sizes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unbounded or overflowing type obtained an observation: %v", err) + } + } + large := types.NewArray(types.Typ[types.Uint8], 1<<62) + fields = []*types.Var{ + types.NewVar(token.NoPos, nil, "A", large), + types.NewVar(token.NoPos, nil, "B", large), + } + if _, err := goAMD64FrameLanguageType(types.NewStruct(fields, nil), sizes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("aggregate size overflow was recorded as a layout: %v", err) + } + if _, err := goAMD64FrameLanguageType(types.Typ[types.Uint64], nil); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("missing target sizes were treated as a layout") + } +} + +func TestGoAMD64FrameTypedSourceMissingSignatureRemainsContext(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte) uint64", "TEXT ·Value(SB),4,$160-32\nRET\n") + pkg.Types = types.NewPackage(pkg.Path, "framefixture") + pkg.Types.Scope().Insert(types.NewFunc(token.NoPos, pkg.Types, "Value", nil)) + guard := goAMD64FrameTypedSelection(t, pkg, request) + if _, err := guard.typedDeclarations(pkg, false); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("missing supplied typed signature did not retain Context: %v", err) + } +} + +func TestGoAMD64FrameTypedSourceMutationAndCloseRevocation(t *testing.T) { + for _, change := range []string{"source", "object", "header inventory", "cancel", "close"} { + t.Run(change, func(t *testing.T) { + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte, n uint64) uint64", "TEXT ·Value(SB),4,$160-40\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + switch change { + case "source": + if err := os.WriteFile(filepath.Join(request.Directory, "value.go"), []byte("package framefixture\nfunc Value(a []uint16, n uint64) uint64\n"), 0600); err != nil { + t.Fatal(err) + } + case "object": + if err := os.WriteFile(filepath.Join(source.owned, request.AssemblyFile+".o"), []byte("not the original object"), 0600); err != nil { + t.Fatal(err) + } + case "header inventory": + if err := os.WriteFile(filepath.Join(request.Directory, "late.h"), []byte("#define NEW 1\n"), 0600); err != nil { + t.Fatal(err) + } + case "cancel": + cancel() + case "close": + if err := source.Close(); err != nil { + t.Fatal(err) + } + } + if err := source.Verify(ctx); err == nil { + t.Fatal("changed/closed composite source retained its observation") + } + if !errors.Is(source.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatal("failed composite Verify did not permanently revoke the token") + } + }) + } +} + +func TestGoAMD64FrameTypedSourceNoMemoryPermissionInThreeAPIs(t *testing.T) { + assembly := "TEXT ·Value(SB),4,$160-40\nPXOR X0,X0\nMOVOU X0,16(SP)\nPADDQ 16(SP),X0\nRET\n" + pkg, request := goAMD64FrameTypedFixture(t, "func Value(a []byte, n uint64) uint64", assembly) + source := requireGoAMD64FrameCapture(t, pkg, request) + sig := source.functions[pkg.Path+".Value"].signature + if !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("composite observation entered fixed-local numeric transport") + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) } + opt := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: map[string]FuncSig{sig.Name: sig}} + for _, api := range []string{"Translate", "TranslateModuleInContext", "TranslateGoModule"} { + t.Run(api, func(t *testing.T) { + var err error + switch api { + case "Translate": + _, err = Translate(file, opt) + case "TranslateModuleInContext": + ctx := llvm.NewContext() + defer ctx.Dispose() + module, failure := TranslateModuleInContext(ctx, file, opt) + err = failure + if !module.IsNil() { + defer module.Dispose() + } + case "TranslateGoModule": + translation, failure := TranslateGoModule(pkg, []byte(assembly), GoModuleOptions{GOOS: "linux", GOARCH: "amd64", TargetTriple: opt.TargetTriple, ResolveSym: resolve}) + err = failure + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("sealed observation alone granted native local memory: %v", err) + } + }) + } + encoded, err := json.Marshal(source) + if err != nil { + t.Fatal(err) + } + var restored GoAMD64FrameSource + if err := json.Unmarshal(encoded, &restored); err != nil || !errors.Is(restored.Verify(context.Background()), ErrProbeNeedsContext) { + t.Fatalf("public JSON recreated a composite observation: %v", err) + } + if !reflect.DeepEqual(source.declarations[sig.Name].signature, sig) { + t.Fatal("object signature diverged from independently typed declaration") + } +} + +type goAMD64FrameLayoutCounter struct { + types.Sizes + structCalls int +} + +func (counter *goAMD64FrameLayoutCounter) Sizeof(typ types.Type) int64 { + if _, ok := typ.(*types.Struct); ok { + counter.structCalls++ + } + return counter.Sizes.Sizeof(typ) +} + +func TestGoAMD64FrameTypedSourceMetadataByteBoundBeforeLayout(t *testing.T) { + for _, field := range []string{"name", "tag", "package"} { + t.Run(field, func(t *testing.T) { + name, tag := "Value", "" + pkg := types.NewPackage("example.invalid/model", "model") + switch field { + case "name": + name = strings.Repeat("x", 64<<10+1) + case "tag": + tag = strings.Repeat("x", 64<<10+1) + case "package": + pkg = types.NewPackage("example.invalid/"+strings.Repeat("x", 64<<10+1), "model") + } + field := types.NewVar(token.NoPos, pkg, name, types.Typ[types.Uint64]) + structure := types.NewStruct([]*types.Var{field}, []string{tag}) + counter := &goAMD64FrameLayoutCounter{Sizes: types.SizesFor("gc", "amd64")} + if _, err := goAMD64FrameLanguageType(structure, counter); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("oversized field metadata was recorded: %v", err) + } + if counter.structCalls != 0 { + t.Fatalf("aggregate layout continued after metadata exceeded its bound: %d calls", counter.structCalls) + } + }) + } +} + +func TestGoAMD64FrameTypedSourcePointeeModelIsNotMachineExtent(t *testing.T) { + declaration := `type Packet struct { X uint64; Y byte } +type EmptyTail struct { X uint64; Y struct{} } +func Value(a *Packet, b *EmptyTail, n uint64) uint64` + pkg, request := goAMD64FrameTypedFixture(t, declaration, "TEXT ·Value(SB),4,$160-32\nRET\n") + source := requireGoAMD64FrameCapture(t, pkg, request) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + header := filepath.Join(source.owned, "pointee-layout.h") + archive := filepath.Join(source.owned, "pointee-layout.a") + if _, err := source.run(ctx, "tool", "compile", "-p", pkg.Path, "-asmhdr", header, "-o", archive, "value.go"); err != nil { + t.Fatal(err) + } + data, err := source.read(header) + if err != nil { + t.Fatal(err) + } + if data, err := source.read(archive); err != nil || len(data) == 0 { + t.Fatalf("actual original compiler archive missing: %v", err) + } + definitions := make(map[string]string) + for _, line := range strings.Split(string(data), "\n") { + fields := strings.Fields(line) + if len(fields) == 3 && fields[0] == "#define" { + if _, duplicate := definitions[fields[1]]; duplicate { + t.Fatalf("duplicate original machine definition: %s", line) + } + definitions[fields[1]] = fields[2] + } + } + fn := source.functions[pkg.Path+".Value"] + for _, name := range []string{"Packet", "EmptyTail"} { + // This is the independently observed machine definition. The + // capture descriptor is deliberately only the go/types model. + if definitions[name+"__size"] != "16" || definitions[name+"_X"] != "0" || definitions[name+"_Y"] != "8" { + t.Fatalf("unexpected actual compiler padded layout for %s: %v", name, definitions) + } + typ := pkg.Types.Scope().Lookup(name).Type() + model := types.SizesFor("gc", "amd64").Sizeof(typ) + identity, err := goAMD64FrameLanguageType(typ, types.SizesFor("gc", "amd64")) + if err != nil || !strings.Contains(fn.languageType, identity) { + t.Fatalf("missing independent language model: %v", err) + } + t.Logf("%s: recorded go/types model=%d; independently observed stock-Go machine size=16", name, model) + } + if fn.transport != goAMD64FrameCompositeTransport || !errors.Is(source.numericDeclarations(pkg), ErrProbeNeedsContext) { + t.Fatal("language model was used as a numeric backing/pointee permission") + } + if err := source.Verify(ctx); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_go_legacy_metadata_roles_test.go b/amd64_go_legacy_metadata_roles_test.go new file mode 100644 index 000000000..84bda8116 --- /dev/null +++ b/amd64_go_legacy_metadata_roles_test.go @@ -0,0 +1,68 @@ +package plan9asm + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/xgo-dev/plan9asm/internal/golegacymod" + "github.com/xgo-dev/plan9asm/internal/gooriginalsrc" +) + +func TestGoAMD64LegacyDeclarationMetadataRoles(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + metadata := filepath.Join(t.TempDir(), "legacy.mod") + member := filepath.Join(root, "go.mod") + for _, name := range []string{metadata, member} { + if err := os.WriteFile(name, []byte("module example.invalid/legacy\n"), 0600); err != nil { + t.Fatal(err) + } + } + metadata, err = filepath.EvalSymlinks(metadata) + if err != nil { + t.Fatal(err) + } + original := gooriginalsrc.Module{Root: root, ModulePath: "example.invalid/legacy", + SourceModule: "example.invalid/legacy", SourceVersion: "v1.0.0", GoMod: metadata, + GoModOrigin: golegacymod.ProxyGoModProtocol} + for _, role := range []string{"dependency", "invented member", "main"} { + t.Run(role, func(t *testing.T) { + data, err := json.Marshal(map[string]any{ + "Dir": root, "ImportPath": original.ModulePath, + "Module": map[string]any{"Path": original.ModulePath, "Version": original.SourceVersion, + "Dir": root, "GoMod": metadata}, + }) + if err != nil { + t.Fatal(err) + } + var selected goAMD64FrameSelection + if err := json.Unmarshal(data, &selected); err != nil { + t.Fatal(err) + } + // Predicate data only, deliberately not an actual whole-ZIP origin + // or frame token. The duplicate member tests source-role confusion. + source := &GoAMD64FrameSource{downloadedOrigin: &gooriginalsrc.OriginSet{}, downloadedModule: &original, + downloadedImports: map[string]gooriginalsrc.Module{original.ModulePath + "@" + original.SourceVersion: original}} + if role == "invented member" { + selected.Module.GoMod = member + } + if role == "main" { + selected.Module.Main = true + selected.Module.Version, selected.Module.GoVersion = "", "1.16" + } + _, err = source.downloadedDeclarationDirectory(selected) + if role == "dependency" && err != nil || role != "dependency" && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("legacy metadata lost its exact dependency-only origin", err) + } + if err := source.Verify(context.Background()); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("module predicate data manufactured frame authority", err) + } + }) + } +} diff --git a/amd64_go_windows_unwind.go b/amd64_go_windows_unwind.go new file mode 100644 index 000000000..1f3276858 --- /dev/null +++ b/amd64_go_windows_unwind.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "bytes" + "path/filepath" + "strconv" + "strings" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" + "golang.org/x/arch/x86/x86asm" +) + +func (source *GoAMD64FrameSource) observeWindowsUnwind(object []byte) (gostaticstorage.WindowsAMD64AssemblyUnwind, error) { + var empty gostaticstorage.WindowsAMD64AssemblyUnwind + for _, declaration := range source.declarations { + if !fixedLocalTransportRole(declaration.transport) { + return empty, goAMD64FrameContext("Windows unwind join requires the closed typed ABI0 transport scope") + } + } + builtin, err := gostaticstorage.WindowsAMD64UnwindSources(source.env["GOVERSION"], func(name string) ([]byte, error) { + return source.read(filepath.Join(source.env["GOROOT"], filepath.FromSlash(name))) + }) + if err != nil { + return empty, goAMD64FrameContext("Windows unwind actual SDK sources: %v", err) + } + observed, err := gostaticstorage.ObserveWindowsAMD64AssemblyUnwind(object, builtin) + if err != nil { + return empty, goAMD64FrameContext("Windows unwind complete object metadata: %v", err) + } + if observed.Version != source.env["GOVERSION"] { + return empty, goAMD64FrameContext("Windows unwind object differs from its actual producer") + } + return observed, nil +} + +func goAMD64WindowsUnwindByteRow(line string, offset int, expected []byte) ([]byte, error) { + fields := strings.Fields(line) + if len(expected) != 12 || offset != 0 || len(fields) != 14 || fields[0] != "0x0000" { + return nil, goAMD64FrameContext("unknown Windows unwind listing byte/relocation/extent record") + } + actual := make([]byte, 12) + var ascii strings.Builder + for index := range actual { + if len(fields[index+1]) != 2 { + return nil, goAMD64FrameContext("Windows unwind listing has an incomplete byte record") + } + value, err := strconv.ParseUint(fields[index+1], 16, 8) + if err != nil { + return nil, goAMD64FrameContext("Windows unwind listing byte is invalid") + } + actual[index] = byte(value) + printed := byte('.') + if actual[index] >= ' ' && actual[index] <= 0x7e { + printed = actual[index] + } + ascii.WriteByte(printed) + } + if !bytes.Equal(actual, expected) || fields[13] != ascii.String() { + return nil, goAMD64FrameContext("Windows unwind listing differs from complete original binary bytes") + } + return actual, nil +} + +// This private join consumes the factory's independently read whole object. +// A public offline observation is never itself an authorization input. +func goAMD64FrameWindowsUnwindListing(listing []byte, file *File, pkg GoPackage, origins map[string]int, directory string, observed gostaticstorage.WindowsAMD64AssemblyUnwind) (map[string]goAMD64FrameFunction, error) { + functions, err := goAMD64FrameListingWithWindowsUnwind(listing, file, pkg, origins, &observed) + if err != nil { + return nil, err + } + for _, text := range observed.Texts { + if len(text.Relocations) != 0 { + return nil, goAMD64FrameContext("Windows object/listing TEXT relocation requires its separate complete observer") + } + } + if err := goAMD64FrameWindowsUnwindFunctions(functions, origins, directory, observed); err != nil { + return nil, err + } + return functions, nil +} + +func goAMD64FrameWindowsUnwindFunctions(functions map[string]goAMD64FrameFunction, origins map[string]int, directory string, observed gostaticstorage.WindowsAMD64AssemblyUnwind) error { + if len(observed.Texts) != len(functions) { + return goAMD64FrameContext("Windows object/listing complete TEXT inventories differ") + } + unwind := make(map[string]gostaticstorage.WindowsAMD64UnwindRecord) + for _, record := range observed.Unwind { + unwind[record.Name] = record + } + seen := make(map[string]bool) + for _, text := range observed.Texts { + fn, exists := functions[text.Name] + if !exists || seen[text.Name] || text.ABI != 0 || text.Kind != 1 || text.Flags != 16 || text.Flags2 != observed.TextFlags2 || + !bytes.Equal(text.Code, fn.code) { + return goAMD64FrameContext("Windows object/listing TEXT name/ABI/code role differs") + } + seen[text.Name] = true + var metadata gostaticstorage.WindowsAMD64UnwindRecord + for _, aux := range text.Auxiliary { + if aux.Type == observed.AuxType { + metadata = unwind[aux.Target.Name] + } + } + first, err := x86asm.Decode(fn.code, 64) + if err != nil || first.Op != x86asm.PUSH || first.Args[0] != x86asm.RBP || first.Len != int(metadata.PushEnd) { + return goAMD64FrameContext("Windows unwind push end differs from actual automatic prologue") + } + second, err := x86asm.Decode(fn.code[first.Len:], 64) + if err != nil || second.Op != x86asm.MOV || second.Args[0] != x86asm.RBP || second.Args[1] != x86asm.RSP || + first.Len+second.Len != int(metadata.PrologueEnd) { + return goAMD64FrameContext("Windows unwind frame end differs from actual automatic prologue") + } + if err := goAMD64WindowsUnwindSourcePCs(text, fn, directory, origins); err != nil { + return err + } + } + return nil +} + +func goAMD64WindowsUnwindSourcePCs(text gostaticstorage.TextRecord, fn goAMD64FrameFunction, directory string, origins map[string]int) error { + normalize := func(name string) string { + if !filepath.IsAbs(name) { + name = filepath.Join(directory, filepath.FromSlash(name)) + } + return filepath.Clean(name) + } + // Go object file tables use slash paths on Windows, while captured + // origins retain host paths and the command's relative source spelling. + // Compare their already-bound paths consistently with the listing join; + // aliases must still agree on the complete original source extent. + linesByPath := make(map[string]int, len(origins)) + for name, lines := range origins { + path := normalize(name) + if previous, found := linesByPath[path]; found && previous != lines { + return goAMD64FrameContext("Windows source origin aliases disagree on their original extent") + } + linesByPath[path] = lines + } + var previous int64 + for _, span := range text.Positions { + if span.File == "" || span.Line <= 0 || int(span.Line) > linesByPath[normalize(span.File)] || span.Start < 0 || + span.Start != previous || span.End <= span.Start || span.End > int64(len(fn.code)) { + return goAMD64FrameContext("Windows object source-PC interval lacks its original file/line/extent") + } + previous = span.End + } + if previous != int64(len(fn.code)) { + return goAMD64FrameContext("Windows source-PC intervals do not cover the complete physical TEXT") + } + for pc, positions := range fn.sourcePositions { + matched := false + for _, span := range text.Positions { + if span.Start > int64(pc) || span.End <= int64(pc) { + continue + } + for _, position := range positions { + if normalize(position.file) == normalize(span.File) && position.line == int(span.Line) { + matched = true + } + } + } + if !matched { + return goAMD64FrameContext("Windows object/listing source-PC file/line witness differs") + } + } + return nil +} diff --git a/amd64_go_windows_unwind_join_test.go b/amd64_go_windows_unwind_join_test.go new file mode 100644 index 000000000..3439782c0 --- /dev/null +++ b/amd64_go_windows_unwind_join_test.go @@ -0,0 +1,239 @@ +//go:build go1.23 + +package plan9asm + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/gostaticstorage" +) + +func TestGoAMD64WindowsUnwindCompleteObjectListingJoinGuards(t *testing.T) { + goAMD64WindowsUnwindCompleteObjectListingJoinGuards(t) +} + +func TestGoAMD64WindowsUnwindCompleteJoinTemporaryParentAlias(t *testing.T) { + goAMD64WindowsUnwindTemporaryParentAlias(t) + // A new child owns a fresh testing.TempDir root under the explicit alias. + // The parent's root was already created while constructing that alias. + t.Run("AliasedTemporaryParent", goAMD64WindowsUnwindCompleteObjectListingJoinGuards) +} + +func goAMD64WindowsUnwindCompleteObjectListingJoinGuards(t *testing.T) { + t.Helper() + pkg, request := goAMD64FrameSourceFixture(t) + request.Target = "windows/amd64" + source := requireGoAMD64FrameCapture(t, pkg, request) + t.Logf("ACTUAL_WINDOWS_SOURCE_CAPTURE request=%q observed=%q functions=%d", + request.Directory, source.request.Directory, len(source.functions)) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + object, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o")) + if err != nil { + t.Fatal(err) + } + listing, err := source.read(filepath.Join(source.owned, request.AssemblyFile+".o.listing")) + if err != nil { + t.Fatal(err) + } + original, err := source.read(filepath.Join(request.Directory, request.AssemblyFile)) + if err != nil { + t.Fatal(err) + } + expanded, err := source.preprocess(request.AssemblyFile, original) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, expanded) + if err != nil { + t.Fatal(err) + } + observed, err := source.observeWindowsUnwind(object) + if err != nil { + t.Fatal(err) + } + for _, version := range []string{"go1.27.1", "go1.27.2", "go1.99.1"} { + if version == observed.Version { + continue + } + changed := bytes.Replace(object, []byte(observed.Version+" "), []byte(version+" "), 1) + if bytes.Equal(changed, object) { + t.Fatal("object version mutation did not replace its producer") + } + if _, err := source.observeWindowsUnwind(changed); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("object substituted another actual producer", version, err) + } + } + functions, err := goAMD64FrameWindowsUnwindListing(listing, file, pkg, source.origins, request.Directory, observed) + if err != nil { + t.Fatal("independent actual original metadata join control failed", err) + } + for _, spelling := range []string{"absolute", "slash absolute", "relative", "conflicting extent", "foreign directory"} { + t.Run("source origins/"+spelling, func(t *testing.T) { + origins := make(map[string]int) + for name, lines := range source.origins { + if !filepath.IsAbs(name) { + name = filepath.Join(request.Directory, name) + } + if spelling == "slash absolute" { + name = filepath.ToSlash(name) + } + if spelling == "relative" { + var err error + name, err = filepath.Rel(request.Directory, name) + if err != nil { + t.Fatal(err) + } + } + if spelling == "foreign directory" { + name = filepath.Join(request.Directory, "foreign", filepath.Base(name)) + } + origins[name] = lines + } + if spelling == "conflicting extent" { + origins[request.AssemblyFile] = source.origins[request.AssemblyFile] + 1 + } + for _, text := range observed.Texts { + err := goAMD64WindowsUnwindSourcePCs(text, functions[text.Name], request.Directory, origins) + if spelling == "conflicting extent" || spelling == "foreign directory" { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound or contradictory source origin acquired a PC witness", err) + } + } else if err != nil { + t.Fatal("actual source origin was lost through its host path spelling", err) + } + } + }) + } + serialized, err := json.Marshal(observed) + if err != nil { + t.Fatal(err) + } + // These are ordinary read-only observations, deliberately copied only for + // decoder negatives. None are accepted as a public Source/factory input. + type mutation func([]byte, *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte + metadataStart := bytes.Index(listing, []byte(observed.Unwind[0].Name+" SSEHUNWINDINFO")) + if metadataStart < 0 { + t.Fatal("original listing omitted its actual complete metadata section") + } + mutations := map[string]mutation{ + "wrong metadata name": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("go:sehuw.12."), []byte("go:sehuw.13."), 1) + }, + "wrong metadata kind": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("SSEHUNWINDINFO"), []byte("SRODATA"), 1) + }, + "wrong metadata flags": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("SSEHUNWINDINFO dupok"), []byte("SSEHUNWINDINFO"), 1) + }, + "wrong metadata alignment": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("SSEHUNWINDINFO dupok size=12"), []byte("SSEHUNWINDINFO dupok size=12 align=0x8"), 1) + }, + "other SDK alignment spelling": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + if observed.ListingAlignment { + return bytes.Replace(v, []byte("size=12 align=0x4"), []byte("size=12"), 1) + } + return bytes.Replace(v, []byte("SSEHUNWINDINFO dupok size=12"), []byte("SSEHUNWINDINFO dupok size=12 align=0x4"), 1) + }, + "wrong metadata byte": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte("01 04 02 05"), []byte("02 04 02 05"), 1) + }, + "wrong metadata ASCII": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return bytes.Replace(v, []byte(".......P...."), []byte("............"), 1) + }, + "missing metadata section": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { return v[:metadataStart] }, + "duplicate metadata section": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return append(v, v[metadataStart:]...) + }, + "unknown trailing record": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + return append(v, []byte("UNKNOWN_NATIVE_RECORD\n")...) + }, + "truncated metadata EOF": func(v []byte, _ *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { return v[:len(v)-5] }, + "different actual TEXT bytes": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Code[0] ^= 1 + return v + }, + "different actual TEXT ABI": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { o.Texts[0].ABI = 1; return v }, + "extra actual TEXT": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts = append(o.Texts, o.Texts[0]) + return v + }, + "missing actual TEXT": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { o.Texts = nil; return v }, + "wrong object source line": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions[0].Line++ + return v + }, + "wrong object source file": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions[0].File = "other.s" + return v + }, + "object source-PC gap": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions[0].Start = 1 + return v + }, + "object source-PC overlap": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions = append(o.Texts[0].Positions, o.Texts[0].Positions[0]) + return v + }, + "object source-PC missing tail": func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + o.Texts[0].Positions = o.Texts[0].Positions[:len(o.Texts[0].Positions)-1] + return v + }, + } + for _, role := range []string{"wrong push end", "wrong prologue end"} { + mutations[role] = func(v []byte, o *gostaticstorage.WindowsAMD64AssemblyUnwind) []byte { + metadata := &o.Unwind[0] + oldName := metadata.Name + if role == "wrong push end" { + metadata.PushEnd, metadata.Data[6] = 2, 2 + } else { + metadata.PrologueEnd, metadata.Data[1], metadata.Data[4] = 5, 5, 5 + } + metadata.Name = "go:sehuw.12." + base64.StdEncoding.EncodeToString(metadata.Data) + for index := range o.Texts[0].Auxiliary { + if o.Texts[0].Auxiliary[index].Type == o.AuxType { + o.Texts[0].Auxiliary[index].Target.Name = metadata.Name + } + } + v = bytes.ReplaceAll(v, []byte(oldName), []byte(metadata.Name)) + rowStart := bytes.Index(v[metadataStart:], []byte("\n")) + metadataStart + 1 + return append(append(v[:rowStart:rowStart], goAMD64WindowsUnwindTestByteRow(metadata.Data)...), '\n') + } + } + for name, change := range mutations { + t.Run(name, func(t *testing.T) { + var independent gostaticstorage.WindowsAMD64AssemblyUnwind + if err := json.Unmarshal(serialized, &independent); err != nil { + t.Fatal(err) + } + changed := change(append([]byte(nil), listing...), &independent) + if _, err := goAMD64FrameWindowsUnwindListing(changed, file, pkg, source.origins, request.Directory, independent); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed original Windows binary/listing join was accepted", err) + } + }) + } + if err := source.Verify(ctx); err != nil { + t.Fatal("read-only metadata negative controls changed original source/artifacts", err) + } +} + +func goAMD64WindowsUnwindTestByteRow(data []byte) []byte { + var result, ascii strings.Builder + result.WriteString("\t0x0000") + for _, value := range data { + fmt.Fprintf(&result, " %02x", value) + if value < ' ' || value > 0x7e { + value = '.' + } + ascii.WriteByte(value) + } + fmt.Fprintf(&result, " %s", ascii.String()) + return []byte(result.String()) +} diff --git a/amd64_go_windows_unwind_source_test.go b/amd64_go_windows_unwind_source_test.go new file mode 100644 index 000000000..c55a92afc --- /dev/null +++ b/amd64_go_windows_unwind_source_test.go @@ -0,0 +1,141 @@ +//go:build go1.23 + +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +func TestGoAMD64WindowsUnwindActualSourceFamily(t *testing.T) { + goAMD64WindowsUnwindActualSourceFamily(t) +} + +// A temporary-directory alias is an ordinary host configuration, not permission +// to relax the original object reader's canonical physical-path contract. +func TestGoAMD64WindowsUnwindTemporaryParentAlias(t *testing.T) { + goAMD64WindowsUnwindTemporaryParentAlias(t) + goAMD64WindowsUnwindActualSourceFamily(t) +} + +func goAMD64WindowsUnwindTemporaryParentAlias(t *testing.T) { + t.Helper() + parent, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + physical := filepath.Join(parent, "physical") + alias := filepath.Join(parent, "alias") + if err := os.Mkdir(physical, 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(physical, alias); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", alias) + t.Setenv("GOTMPDIR", alias) +} + +func goAMD64WindowsUnwindActualSourceFamily(t *testing.T) { + t.Helper() + for _, role := range []string{"numeric", "void", "shared numeric and void"} { + t.Run(role, func(t *testing.T) { + declaration := "func Value(a uint64) uint64" + assembly := "TEXT ·Value(SB),4,$160-16\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n" + if role == "void" { + declaration = "func Value(a uint64)" + assembly = "TEXT ·Value(SB),4,$160-8\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nRET\n" + } else if role == "shared numeric and void" { + declaration += "\nfunc Other(a uint64)" + assembly += "TEXT ·Other(SB),4,$160-8\nMOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nRET\n" + } + pkg, request := goAMD64FrameTypedFixture(t, declaration, assembly) + request.Target = "windows/amd64" + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + var environment []string + for _, value := range request.Environment { + key, _, _ := strings.Cut(value, "=") + if key != "GOOS" && key != "GOARCH" && key != "CGO_ENABLED" { + environment = append(environment, value) + } + } + environment = append(environment, "GOOS=windows", "GOARCH=amd64", "CGO_ENABLED=0") + request.Environment = environment + objectDirectory, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal("cannot resolve original Windows output directory", err) + } + object := filepath.Join(objectDirectory, "original-windows.o") + listing, err := goasmcmd.Run(ctx, request.Directory, environment, request.GoBinary, + "tool", "asm", "-S", "-p", pkg.Path, "-o", object, request.AssemblyFile) + if err != nil { + t.Fatal("actual original Windows assembler control failed", err) + } + info, err := os.Stat(object) + if err != nil || !info.Mode().IsRegular() || info.Size() <= 0 { + t.Fatal("actual original Windows object was not produced", err) + } + original, err := os.ReadFile(object) + if err != nil || int64(len(original)) != info.Size() || + !bytes.Contains(listing, []byte("SSEHUNWINDINFO")) || + !bytes.Contains(original, []byte("go:sehuw.12.")) { + t.Fatal("independent original Windows object precondition failed", err) + } + t.Logf("ACTUAL_WINDOWS_ORIGINAL_OUTPUT_EXISTS bytes=%d sha256=%x hex=%x\nACTUAL_WINDOWS_ORIGINAL_LISTING\n%s", + len(original), sha256.Sum256(original), original, listing) + data, err := goAMD64FrameRead(object) + if err != nil || !bytes.Equal(data, original) || !bytes.Contains(listing, []byte("SSEHUNWINDINFO")) || + !bytes.Contains(data, []byte("go:sehuw.12.")) { + t.Fatal("actual original Windows object/listing lacks its unwind metadata", err) + } + t.Logf("ACTUAL_WINDOWS_ORIGINAL_OBJECT bytes=%d sha256=%x hex=%x\nACTUAL_WINDOWS_ORIGINAL_LISTING\n%s", + len(data), sha256.Sum256(data), data, listing) + if !t.Run("PlainParserRefusesUnboundMetadata", func(t *testing.T) { + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + origins := map[string]int{request.AssemblyFile: strings.Count(assembly, "\n") + 1} + if _, err := goAMD64FrameListing(listing, file, pkg, origins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("plain listing parser borrowed Windows binary metadata authority", err) + } + }) { + return + } + source, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if err != nil { + t.Fatal("actual ordinary Windows numeric/void source requires its complete SEH join", err) + } + t.Cleanup(func() { + if err := source.Close(); err != nil { + t.Error(err) + } + }) + want := 1 + if role == "shared numeric and void" { + want = 2 + } + if len(source.functions) != want || source.Verify(ctx) != nil { + t.Fatal("complete Windows numeric/void original source did not verify") + } + copied := new(GoAMD64FrameSource) + reflect.ValueOf(copied).Elem().Set(reflect.ValueOf(source).Elem()) + if !errors.Is(copied.Verify(ctx), ErrProbeNeedsContext) || + !errors.Is(copied.Close(), ErrProbeNeedsContext) || source.Verify(ctx) != nil { + t.Fatal("Windows metadata route bypassed exact source owner identity") + } + if _, err := os.Stat(request.Directory); err != nil { + t.Fatal("source owner identity control changed the original tree", err) + } + }) + } +} diff --git a/amd64_gp_simd_native_result_test.go b/amd64_gp_simd_native_result_test.go new file mode 100644 index 000000000..222979d13 --- /dev/null +++ b/amd64_gp_simd_native_result_test.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +var x86GPSIMDNativeOracle = []struct { + op, original string + width int +}{ + {"MOVL", "MOVL", 4}, + {"MOVQ", "MOVQ", 8}, + {"MOVD", "MOVD", 8}, // Go's MOVD is the qword alias. + {"VMOVD", "MOVL", 4}, + {"VMOVQ", "MOVQ", 8}, +} + +func x86GPSIMDNativeFixture(name, arch, op, address, destination string, lane int) (string, string) { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + extract := "PEXTRD" + if strings.HasPrefix(op, "V") { + extract = "VPEXTRD" + } + source := fmt.Sprintf("TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),DX\nMOVL $1,BX\n%s %s,%s\n%s $%d,%s,AX\nMOVL AX,ret+%d(FP)\nRET\n", + name, word+4, move, op, address, destination, extract, lane, destination, word) + return source, fmt.Sprintf("func %s(p *[8]byte) uint32\n", name) +} + +func TestX86GPSIMDNativeResultsThreeAPIFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, row := range x86GPSIMDNativeOracle { + for _, address := range []string{"(DX)", "-1(DX)(BX*1)"} { + for _, destination := range []string{"X0", "X17"} { + if destination == "X17" && (target.arch == "386" || !strings.HasPrefix(row.op, "V")) { + continue // High banks retain their independent grammar/physical-mode checks. + } + for lane := 0; lane < 4; lane++ { + asm, decl := x86GPSIMDNativeFixture(fmt.Sprintf("F%d", count), target.arch, row.op, address, destination, lane) + source.WriteString(asm) + declarations.WriteString(decl) + count++ + } + } + } + } + want := 56 + if target.arch == "386" { + want = 40 + } + if count != want { + t.Fatal("incomplete native GP/X source and zero-lane matrix", count, want) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("GP/X memory read lost its numerical/zero result bytes:", err) + } + if strings.Count(ir, "load volatile i") != count { + t.Fatal("numerical continuation lost a paired native read") + } + compileLLVMToObject(t, llc, target.triple, "gp-x-native.ll", "gp-x-native.o", ir) + }) + } + }) + } +} + +func TestX86GPSIMDNativeResultsRetainSourceBoundaries(t *testing.T) { + for _, row := range x86GPSIMDNativeOracle { + t.Run(row.op, func(t *testing.T) { + source, declaration := x86GPSIMDNativeFixture("Value", "amd64", row.op, "(DX)", "X0", 0) + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", strings.TrimSpace(declaration), + strings.ReplaceAll(strings.SplitN(source, "\n", 2)[1], "ret+8(FP)", "ret0+8(FP)")) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("standalone query acquired native memory", err) + } + } + load := row.op + " (DX),X0" + missing := strings.Replace(source, load, "CMPL BX,$0\nJE after\n"+load+"\nafter:", 1) + for name, bad := range map[string]string{ + "unknown base": strings.Replace(source, "(DX)", "(SI)", 1), + "unknown index": strings.Replace(source, "(DX)", "(DX)(SI*1)", 1), + "pointer index": strings.Replace(source, "(DX)", "(DX)(DX*1)", 1), + "virtual frame": strings.Replace(source, "(DX)", "(SP)", 1), + "unknown static": strings.Replace(source, "(DX)", "·external(SB)", 1), + "numeric base": strings.Replace(source, load, "MOVL $42,DX\n"+load, 1), + "native store": strings.Replace(source, load, load+"\n"+row.op+" X0,(DX)", 1), + "missing path": missing, + } { + t.Run(name, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", bad, "", true) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, bad, declaration, "amd64", "x86_64-unknown-linux-gnu", api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unproved native transfer boundary accepted", api, err) + } + } + }) + } + }) + } +} + +func TestX86GPSIMDNativeResultBytesAndOrigins(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86GPSIMDNativeOracle { + t.Run(arch+"/"+row.op, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, row.op+" -1(DX)(BX*1),X1") + form, known, err := x86GPSIMDTransferProperties(ins, arch) + if !known || err != nil || form.bits != row.width*8 { + t.Fatal("native memory width changed", form, err) + } + state := x86ResultState{flags: x86ByteFlagsAll} + base, _, _ := x86ResultGP(DX, 4, arch) + index, _, _ := x86ResultGP(BX, 4, arch) + state.pointerGP[base], state.gp[index] = true, 0xff + state.writeVector(1, 64, 0x0123456789ab0080, false) + before := state.clone() + want := uint64(0x0123456789abffff) + if strings.HasPrefix(row.op, "V") { + want = ^uint64(0) + } + if !x86ResultGPSIMDTransferWithRead(ins, form, &state, nil, 0, arch, true) || + state.vectorBytes(1) != want || state.flags != before.flags || + state.gp != before.gp || state.pointerGP != before.pointerGP { + t.Fatal("native transfer lost loaded/zero bytes or changed unrelated state") + } + for _, mutation := range []string{"decoded", "raw", "raw return", "address", "unbound address", "segment"} { + changed, rejected := ins, before.clone() + switch mutation { + case "decoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "raw return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + case "address": + changed.x86AddressBits = 32 + case "unbound address": + changed.x86UnboundAddressOverride = true + case "segment": + changed.x86SegmentPrefix = 0x64 + } + if x86ResultGPSIMDTransferWithRead(changed, form, &rejected, nil, 0, arch, true) || !reflect.DeepEqual(rejected, before) { + t.Fatal("unbound instruction origin acquired native result bytes", mutation) + } + } + }) + } + } +} diff --git a/amd64_gp_simd_native_runtime_unix_test.go b/amd64_gp_simd_native_runtime_unix_test.go new file mode 100644 index 000000000..fcc8c9d54 --- /dev/null +++ b/amd64_gp_simd_native_runtime_unix_test.go @@ -0,0 +1,67 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86GPSIMDNativeResultsAndFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + for _, row := range x86GPSIMDNativeOracle { + for discarded := 0; discarded < 2; discarded++ { + name, original := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count) + body := row.op + " -1(DX)(BX*1),X0\nPEXTRQ $0,X0,AX\n" + native := row.original + " (DI),X0\nMOVQ X0,AX\n" + if discarded == 1 { + body += "MOVQ $0x87654321fedcba98,AX\n" + native += "MOVQ $0x87654321fedcba98,AX\n" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DX\nMOVL $1,BX\n%sMOVQ AX,ret+8(FP)\nRET\n", name, body) + fmt.Fprintf(&declarations, "func %s(p *[8]byte) uint64\n", name) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+native+"RET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\nextern uint64_t %s(const uint8_t*);\n", name, original) + fmt.Fprintf(&rows, "{%s,%s,%d,%d},\n", name, original, row.width, discarded) + count++ + } + } + if count != 10 { + t.Fatal("incomplete live/dead GP/X family", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86NativeScalarReadRuntimeC, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "gp-x-native", text, driver, level) + if string(output) != "10 native scalar kernels: numerical 1280, faults 120, tripwires 2; failures 0\n" { + t.Fatal("original/LLVM GP/X numerical/zero bytes or read widths differ:", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} diff --git a/amd64_gp_simd_transfer_boundaries_test.go b/amd64_gp_simd_transfer_boundaries_test.go new file mode 100644 index 000000000..7f28d9cc9 --- /dev/null +++ b/amd64_gp_simd_transfer_boundaries_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86GPSIMDTransferRejectsCompleteSiblingInvalidFormats(t *testing.T) { + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + rows := []string{op + ".Z AX,X0", op + ".BCST (AX),X0", op + ".SAE X0,AX"} + for _, args := range []string{"Y0,X1", "Z0,X1", "K1,X1", "X0,Y1", "X0,Z1", "X0,K1", "$7,X1", "$1.25,X1", + "X32,X0", "X0,X32", "X0,K1,X1", "X0,$1", "X0,$1.25", "(AX)(SP*1),X1"} { + rows = append(rows, op+" "+args) + } + if op == "MOVL" || op == "VMOVD" { + rows = append(rows, op+" X0,X1") + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + ins := x86YMMReadSource(t, row) + if _, known, err := x86GPSIMDTransferProperties(ins, arch); !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("invalid syntax became an owned valid/context form", row, arch, known, err) + } + if x86CallerVectorMoveKnown(ins) { + t.Fatal("invalid sibling acquired caller effect") + } + x86RequireZeroMaskAPIOutcome(t, arch, source, false) + } + }) + } + } +} + +func TestX86GPSIMDTransferActualCallerKeepsSafetyBoundaries(t *testing.T) { + for name, body := range map[string]string{ + "unknown GP": "VMOVD BX,X1\nVMOVSS X1,(DX)", + "partial GP": "MOVB $1,BX\nVMOVD BX,X1\nVMOVSS X1,(DX)", + "pointer bits": "VMOVQ DI,X1\nVMOVSS X1,(DX)", + "stack identity": "MOVQ SP,X1\nVMOVSS X1,(DX)", + "frame identity": "MOVQ BP,X1\nVMOVSS X1,(DX)", + "undefined X": "VMOVD X0,AX\nMOVL AX,(DX)", + "partial X for qword": "MOVSS (DI),X2\nMOVSS X2,X0\nVMOVQ X0,AX\nMOVL AX,(DX)", + "unknown memory": "VMOVD (BX),X1\nVMOVSS X1,(DX)", + "length overread": "VMOVQ 4(CX),X1\nVMOVSS X1,(DX)", + "result overread": "MOVQ (DX),X1\nVMOVSS X1,(DX)", + "input write": "VMOVUPS (DI),X0\nVMOVD X0,(DI)", + "length write": "VMOVUPS (DI),X0\nVMOVQ X0,(CX)", + "wide result": "VMOVUPS (DI),X0\nVMOVQ X0,(DX)", + "result overlap": "VMOVUPS (DI),X0\nVMOVD X0,1(DX)", + "pointer lost as GP": "VMOVUPS (DI),X0\nVMOVQ X0,DI\nVMOVD (DI),X1\nVMOVSS X1,(DX)", + "g identity overwritten": "VMOVUPS (DI),X0\nVMOVQ X0,R14\nVMOVSS X0,(DX)", + "unknown join": "CMPQ AX,$0\nJE joined\nVMOVD AX,X1\njoined:\nVMOVSS X1,(DX)", + "legacy upper unknown": "MOVL AX,X1\nVEXTRACTF128 $1,Y1,X0\nVMOVSS X0,(DX)", + "MMX native state": "MOVQ X0,M0\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound state acquired caller permission", err) + } + }) + } +} + +func TestX86GPSIMDTransferParserRejectsInvalidScale(t *testing.T) { + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + source := "TEXT f(SB),4,$0-0\n" + op + " X0,(AX)(CX*3)\nRET\n" + t.Run(op, func(t *testing.T) { + if _, err := Parse(ArchAMD64, source); err == nil || + !strings.Contains(err.Error(), `invalid x86 symbolic address register/scale group: "(CX*3)"`) { + t.Fatal("parser repaired an invalid scale token", err) + } + for _, arch := range []string{"amd64", "386"} { + goAssemblerOracleOutput(t, arch, source, "", false) + } + for _, target := range x86RawReturnTargets { + if _, err := x86ScalarFPBitViewAPI(t, strings.Replace(source, "f(SB)", "·view(SB)", 1), + "func view()", target.arch, target.triple, "Go"); err == nil || + !strings.Contains(err.Error(), `invalid x86 symbolic address register/scale group: "(CX*3)"`) { + t.Fatal("Go API repaired an invalid original scale", target.triple, err) + } + } + }) + } +} + +func TestX86GPSIMDTransferFPCompleteFieldsThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + bits := 32 + if op == "MOVQ" || op == "MOVD" || op == "VMOVQ" { + bits = 64 + } + for _, input := range []string{"uint64", fmt.Sprintf("[%d]byte", bits/8)} { + fixture := x86ScalarFPBitFixture{name: op + "/" + input, + declaration: fmt.Sprintf("func view(x %s) uint%d\n", input, bits), + body: op + " x+0(FP),X1\n" + op + " X1,ret+8(FP)"} + for _, target := range x86RawReturnTargets { + t.Run(fixture.name+"/"+target.triple, func(t *testing.T) { + source := x86ScalarFPBitAssembly(t, fixture, target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, fixture.declaration, target.arch, target.triple, api) + if err != nil || ir == "" { + t.Fatal("complete independent field lost its data", api, err) + } + compileLLVMToObject(t, llc, target.triple, "transfer-fp-"+api+".ll", "transfer-fp-"+api+".o", ir) + } + }) + } + } + } +} diff --git a/amd64_gp_simd_transfer_family_test.go b/amd64_gp_simd_transfer_family_test.go new file mode 100644 index 000000000..ac75768a9 --- /dev/null +++ b/amd64_gp_simd_transfer_family_test.go @@ -0,0 +1,169 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// This table is an independent projection of all twelve original AVX rows, +// including their zcase/advances, W bits, transfer opcodes and memory tuples. +func x86OfficialGPSIMDTransferRows(t *testing.T) []string { + t.Helper() + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + var rows []string + counts := map[string]int{} + for _, rule := range rules { + if rule.Opcode != "VMOVD" && rule.Opcode != "VMOVQ" { + continue + } + classes := [][]string{{"Yxr", "Yml"}, {"Yml", "Yxr"}, {"YxrEvex", "Yml"}, {"Yml", "YxrEvex"}} + zCases := []string{"Zvex_r_v_rm", "Zvex_rm_v_r", "Zevex_r_v_rm", "Zevex_rm_v_r"} + offsets := []int{0, 2, 4, 7} + advances := []int{2, 2, 3, 3} + opcodes := []string{"0x7E", "0x6E", "0x7E", "0x6E"} + ws := []string{"W0", "W0", "W0", "W0"} + pps := []string{"66", "66", "66", "66"} + sources, destinations := []string{"X0", "AX", "X17", "AX"}, []string{"AX", "X1", "AX", "X17"} + table, count, tuple := "_yvmovd", 10, "evexN4" + if rule.Opcode == "VMOVQ" { + classes = [][]string{{"Yxr", "Yml"}, {"Yxr", "Yxm"}, {"Yml", "Yxr"}, {"Yxm", "Yxr"}, + {"YxrEvex", "Yml"}, {"YxrEvex", "YxmEvex"}, {"Yml", "YxrEvex"}, {"YxmEvex", "YxrEvex"}} + zCases = []string{"Zvex_r_v_rm", "Zvex_r_v_rm", "Zvex_rm_v_r", "Zvex_rm_v_r", "Zevex_r_v_rm", "Zevex_r_v_rm", "Zevex_rm_v_r", "Zevex_rm_v_r"} + offsets, advances = []int{0, 2, 4, 6, 8, 11, 14, 17}, []int{2, 2, 2, 2, 3, 3, 3, 3} + opcodes = []string{"0x7E", "0xD6", "0x6E", "0x7E", "0x7E", "0xD6", "0x6E", "0x7E"} + ws, pps = []string{"W1", "W0", "W1", "W0", "W1", "W1", "W1", "W1"}, []string{"66", "66", "66", "F3", "66", "66", "66", "F3"} + sources, destinations = []string{"X0", "X0", "AX", "X0", "X17", "X17", "AX", "X0"}, []string{"AX", "X1", "X1", "X1", "AX", "X0", "X17", "X17"} + table, count, tuple = "_yvmovq", 20, "evexN8" + } + index := rule.RowIndex + if index < 0 || index >= len(classes) || rule.Ytab != table || rule.Prefix != "Pavx" || + !reflect.DeepEqual(rule.Classes, classes[index]) || rule.ZCase != zCases[index] || + rule.EncodingOffset != offsets[index] || rule.ZOffset != advances[index] || len(rule.OpBytes) != count { + t.Fatalf("unmodeled complete GP/SIMD transfer row: %+v", rule) + } + generation := "vex" + if strings.Contains(zCases[index], "evex") { + generation = "evex" + } + terms := []string{"avxEscape", generation + "128", generation + pps[index], generation + "0F", generation + ws[index]} + if !reflect.DeepEqual(strings.Split(rule.OpBytes[offsets[index]], " | "), terms) { + t.Fatalf("unmodeled GP/SIMD encoding generation/W/prefix: %+v", rule) + } + opcodeIndex := offsets[index] + 1 + if generation == "evex" { + if rule.OpBytes[opcodeIndex] != tuple { + t.Fatalf("unmodeled transfer memory width/mask attributes: %+v", rule) + } + opcodeIndex++ + } + if rule.OpBytes[opcodeIndex] != opcodes[index] { + t.Fatalf("unmodeled complete transfer opcode: %+v", rule) + } + rows = append(rows, rule.Opcode+" "+sources[index]+","+destinations[index]) + counts[rule.Opcode]++ + } + if counts["VMOVD"] != 4 || counts["VMOVQ"] != 8 || len(rows) != 12 { + t.Fatalf("complete original transfer denominator changed: %v", counts) + } + return rows +} + +func TestX86GPSIMDTransferCompleteCallerRegistration(t *testing.T) { + rows := x86OfficialGPSIMDTransferRows(t) + for _, op := range []string{"MOVL", "MOVQ", "MOVD"} { + rows = append(rows, op+" AX,X0", op+" X0,AX", op+" (DI),X0", op+" X0,(DX)") + if op != "MOVL" { + rows = append(rows, op+" X0,X1") + } + } + for _, op := range []string{"VMOVD", "VMOVQ"} { + rows = append(rows, op+" (DI),X0", op+" X0,(DX)") + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + ins := x86YMMReadSource(t, row) + if !x86CallerVectorMoveKnown(ins) { + t.Fatal("Go-accepted whole-family transfer has no caller effect", row) + } + }) + } +} + +func TestX86GPSIMDTransferActualSelectedCallerThreeAPIs(t *testing.T) { + count := 0 + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + for _, direction := range []string{"gp-to-x", "x-to-gp", "memory-load", "memory-store", "x-to-x"} { + if direction == "x-to-x" && (op == "MOVL" || op == "VMOVD") { + continue + } + count++ + t.Run(op+"/"+direction, func(t *testing.T) { + body := "MOVQ $7,AX\n" + op + " AX,X1\nVMOVSS X1,(DX)" + switch direction { + case "x-to-gp": + body = "VMOVUPS (DI),X0\n" + op + " X0,AX\nMOVL AX,(DX)" + case "memory-load": + body = op + " (DI),X1\nVMOVSS X1,(DX)" + case "memory-store": + body = "VMOVUPS (DI),X0\n" + op + " X0,(DX)" + case "x-to-x": + body = "VMOVUPS (DI),X0\n" + op + " X0,X1\nVMOVSS X1,(DX)" + } + source := x86CallerFixtureSource + if direction == "memory-store" && op != "MOVL" && op != "VMOVD" { + source = strings.NewReplacer("var res float32", "var res uint64", ") float32 {", ") uint64 {").Replace(source) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, source, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Error("text caller consumer", err) + } else { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + compileLLVMToObject(t, llc, options.TargetTriple, "transfer-text.ll", "transfer-text.o", ir) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Error("module caller consumer", err) + } else { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + compileLLVMToObject(t, llc, options.TargetTriple, "transfer-module.ll", "transfer-module.o", module.String()) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, options.TargetTriple); err != nil { + t.Error("actual selected typed caller", err) + } + }) + } + } + // Scope denominator is independent of the typed implementation table. + if count != 23 { + t.Fatalf("caller direction denominator changed: %d", count) + } +} diff --git a/amd64_gp_simd_transfer_fault_runtime_test.go b/amd64_gp_simd_transfer_fault_runtime_test.go new file mode 100644 index 000000000..9c2882a4b --- /dev/null +++ b/amd64_gp_simd_transfer_fault_runtime_test.go @@ -0,0 +1,111 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The reference executes the actual Go object's three legacy opcodes. VEX +// translation is compared only for its equivalent unmasked read width, not +// claimed as original AVX/EVEX execution on Rosetta. +func TestCrossLinuxRuntimeMatrixX86GPSIMDTransferOriginalDeadReadFaults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, references, pairs strings.Builder + sigs := map[string]FuncSig{} + for index, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + name := fmt.Sprint("transferfault", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ p+0(FP),AX\n%s (AX),X1\nRET\n", name, op) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + original := op + if op == "VMOVD" { + original = "MOVL" + } else if op == "VMOVQ" { + original = "MOVQ" + } + listing := x86InsertExtractGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+original+" (DI),X1\nRET\n") + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if len(code) < 4 || code[len(code)-1] != 0xc3 { + t.Fatal("original Go memory opcode did not retain its exact ordinary RET", op, code) + } + var operands []string + for _, value := range code[:len(code)-1] { + operands = append(operands, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&references, "extern void %s(const void*);\nstatic void original%d(const void *p) {\n", name, index) + fmt.Fprintf(&references, " __asm__ volatile(\".byte %s\"::\"D\"(p):\"xmm1\",\"memory\");\n}\n", strings.Join(operands, ",")) + if index > 0 { + pairs.WriteString(",") + } + width := 8 + if op == "MOVL" || op == "VMOVD" { + width = 4 + } + fmt.Fprintf(&pairs, "{original%d,%s,%d}", index, name, width) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +` + references.String() + ` +typedef void (*operation)(const void*); +struct pair {operation original,translated;unsigned width;}; +static const struct pair pairs[]={ ` + pairs.String() + ` }; +static int outcome(operation fn,const void *p) { + pid_t pid=fork(); if(pid<0) return -1; + if(!pid) { + struct rlimit limit={0,0}; if(setrlimit(RLIMIT_CORE,&limit)) _exit(120); + fn(p); _exit(0); + } + int status; if(waitpid(pid,&status,0)!=pid) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) return WTERMSIG(status); + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); if(page<128) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op<5;op++) for(unsigned extent=0;extent<=8;extent++) { + const void *p=region+page-extent; + int original=outcome(pairs[op].original,p),translated=outcome(pairs[op].translated,p); + if(original!=translated || (extent>=pairs[op].width ? original!=0 : original!=SIGSEGV && original!=SIGBUS)) { + fprintf(stderr,"read op=%u extent=%u original=%d translated=%d\n",op,extent,original,translated); + return 3; + } + } + for(unsigned op=0;op<5;op++) { + int original=outcome(pairs[op].original,0),translated=outcome(pairs[op].translated,0); + if(original!=translated || (original!=SIGSEGV && original!=SIGBUS)) return 4; + } + return munmap(region,2*page)!=0; +} +` + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "gp-simd-original-dead-faults", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_gp_simd_transfer_forms.go b/amd64_gp_simd_transfer_forms.go new file mode 100644 index 000000000..1caf4b37b --- /dev/null +++ b/amd64_gp_simd_transfer_forms.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86GPSIMDTransferSpec struct { + bits int + legacy, rawOnly bool +} + +// MOVD is Go's MOVQ alias, not Intel's 32-bit MOVD. Ordinary GP-only +// MOVL/Q/D and the separate MMX rows retain their existing scalar grammar. +var x86GPSIMDTransferSpecs = map[Op]x86GPSIMDTransferSpec{ + "MOVL": {bits: 32, legacy: true}, + "MOVQ": {bits: 64, legacy: true}, + "MOVD": {bits: 64, legacy: true}, + "VMOVD": {bits: 32}, + "VMOVQ": {bits: 64}, + "VMOVW": {bits: 16, rawOnly: true}, +} + +type x86GPSIMDTransferForm struct { + spec x86GPSIMDTransferSpec + bits, memory int + sourceX, destinationX, mmx bool + memoryWrite bool +} + +// Encoding/width selection grants neither initialized data nor memory backing. +// In mode32 VEX.W is ignored for GP transfers, but Q memory/X copies stay 64-bit. +func x86GPSIMDTransferProperties(ins Instr, arch string) (x86GPSIMDTransferForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86GPSIMDTransferSpecs[Op(base)] + if !known { + return x86GPSIMDTransferForm{}, false, nil + } + f := x86GPSIMDTransferForm{spec: spec, bits: spec.bits, memory: -1} + hasX := false + for _, arg := range ins.Args { + hasX = hasX || isAMD64GoXRegister(arg) + } + if spec.legacy && !hasX { + return f, false, nil + } + fail := func() (x86GPSIMDTransferForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside complete Go GP/X transfer grammar: %q", arch, ins.Op, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 2 || spec.rawOnly && !ins.x86Encoded { + return fail() + } + src, dst := ins.Args[0], ins.Args[1] + f.sourceX, f.destinationX = isAMD64GoXRegister(src), isAMD64GoXRegister(dst) + for _, arg := range ins.Args { + if isAMD64GoXRegister(arg) { + index, _ := amd64ParseXReg(arg.Reg) + if spec.legacy && index >= 16 { + return fail() + } + } + } + if f.sourceX && f.destinationX { + if spec.bits != 64 { + return fail() + } + } else if f.sourceX || f.destinationX { + other, index := src, 0 + if f.sourceX { + other, index = dst, 1 + } + if isAMD64MemoryOperand(other) && x86PackedShiftMemoryValid(other, arch) { + f.memory, f.memoryWrite = index, index == 1 + } else if other.Kind == OpReg && isX86YrlRegisterForArch(other.Reg, arch) { + if spec.legacy && spec.bits == 64 && arch == "386" { + return fail() + } + if arch == "386" && !spec.legacy && f.bits > 32 { + f.bits = 32 + } + } else if f.sourceX && spec.legacy && spec.bits == 64 && isAMD64MOVDMRegister(other) { + f.mmx = true // Go's MOVDQ2Q row, never a physical caller-state proof. + } else if !f.sourceX && other.Kind == OpImm && ins.x86RIPLiteral && + !other.ImmIsFloat && other.ImmRaw == "" { + // Only the decoder's already-bound RIP data receipt supplies this + // representation. A named literal cannot acquire it. + } else { + return fail() + } + } else { + return fail() + } + return f, true, x86Mode32VectorRegisterContext(ins, arch) +} + +// Preserve the original operand class before normalization can rewrite an +// invalid register/address spelling. Direct and textual APIs share this guard. +func validateX86GPSIMDTransferFile(file *File, arch string) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + if _, known, err := x86GPSIMDTransferProperties(ins, arch); known && err != nil { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + } + } + return nil +} diff --git a/amd64_gp_simd_transfer_grammar_test.go b/amd64_gp_simd_transfer_grammar_test.go new file mode 100644 index 000000000..d9a94be77 --- /dev/null +++ b/amd64_gp_simd_transfer_grammar_test.go @@ -0,0 +1,242 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86GPSIMDTransferCompleteLegacyOriginalTables(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, table := range []struct{ op, name, prefix, bytes, rows string }{ + {"MOVB", "ymovb", "Pb", "0x88,0x8a,0xb0,0xc6,00", ` + {Zr_m,1,argList{Yrb,Ymb}},{Zm_r,1,argList{Ymb,Yrb}}, + {Zib_rp,1,argList{Yi32,Yrb}},{Zibo_m,2,argList{Yi32,Ymb}},`}, + {"MOVW", "ymovw", "Pe", "0x89,0x8b,0xb8,0xc7,00,0", ` + {Zr_m,1,argList{Yrl,Yml}},{Zm_r,1,argList{Yml,Yrl}}, + {Zil_rp,1,argList{Yi32,Yrl}},{Zilo_m,2,argList{Yi32,Yml}}, + {Zaut_r,2,argList{Yiauto,Yrl}},`}, + {"MOVL", "ymovl", "Px", "0x89,0x8b,0xb8,0xc7,00,0x6e,0x7e,Pe,0x6e,Pe,0x7e,0", ` + {Zr_m,1,argList{Yrl,Yml}},{Zm_r,1,argList{Yml,Yrl}}, + {Zil_rp,1,argList{Yi32,Yrl}},{Zilo_m,2,argList{Yi32,Yml}}, + {Zm_r_xm,1,argList{Yml,Ymr}},{Zr_m_xm,1,argList{Ymr,Yml}}, + {Zm_r_xm,2,argList{Yml,Yxr}},{Zr_m_xm,2,argList{Yxr,Yml}}, + {Zaut_r,2,argList{Yiauto,Yrl}},`}, + {"MOVQ", "ymovq", "Pw8", "0x6f,0x7f,Pf2,0xd6,Pf3,0x7e,Pe,0xd6,0x89,0x8b,0xb8,0xc7,00,0x6e,0x7e,Pe,0x6e,Pe,0x7e,0", ` + {Zm_r_xm_nr,1,argList{Ym,Ymr}},{Zr_m_xm_nr,1,argList{Ymr,Ym}}, + {Zm_r_xm_nr,2,argList{Yxr,Ymr}},{Zm_r_xm_nr,2,argList{Yxm,Yxr}}, + {Zr_m_xm_nr,2,argList{Yxr,Yxm}},{Zr_m,1,argList{Yrl,Yml}}, + {Zm_r,1,argList{Yml,Yrl}},{Ziq_rp,1,argList{Yi64,Yrl}}, + {Zilo_m,2,argList{Yi32,Yml}},{Zm_r_xm,1,argList{Ymm,Ymr}}, + {Zr_m_xm,1,argList{Ymr,Ymm}},{Zm_r_xm,2,argList{Yml,Yxr}}, + {Zr_m_xm,2,argList{Yxr,Yml}},{Zaut_r,1,argList{Yiauto,Yrl}},`}, + } { + t.Run(table.op, func(t *testing.T) { + definition := regexp.MustCompile(`(?s)var ` + table.name + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(string(source)) + entry := regexp.MustCompile(`\{A` + table.op + `, (\w+), (\w+), opBytes\{([^}]+)\}\}`).FindStringSubmatch(string(source)) + strip := func(text string) string { + text = regexp.MustCompile(`//[^\n]*`).ReplaceAllString(text, "") + return strings.Join(strings.Fields(text), "") + } + if len(definition) != 2 || len(entry) != 4 || entry[1] != table.name || entry[2] != table.prefix { + t.Fatal("complete legacy row/prefix/opcode inventory changed", entry, definition) + } + inventories := []struct{ name, rows, bytes string }{{"current", table.rows, table.bytes}} + if table.op == "MOVQ" { + // Go 1.20 has a separate signed-imm32 row before its imm64 + // row. Pin that complete historical inventory, rather than + // discarding scalar rows to make the GP/X subset look equal. + inventories = append(inventories, struct{ name, rows, bytes string }{ + "signed-imm32", + strings.Replace(table.rows, "{Ziq_rp,1,argList{Yi64,Yrl}}", + "{Zilo_m,2,argList{Ys32,Yrl}},{Ziq_rp,1,argList{Yi64,Yrl}}", 1), + "0x6f,0x7f,Pf2,0xd6,Pf3,0x7e,Pe,0xd6,0x89,0x8b,0xc7,00,0xb8,0xc7,00,0x6e,0x7e,Pe,0x6e,Pe,0x7e,0", + }) + } + for _, inventory := range inventories { + if strip(definition[1]) == strip(inventory.rows) && strip(entry[3]) == inventory.bytes { + t.Logf("%s complete %s inventory: %s", runtime.Version(), table.op, inventory.name) + return + } + } + t.Fatal("complete legacy row/prefix/opcode inventory changed", entry, definition) + }) + } +} + +func TestX86GPSIMDTransferAllRegisterAndAddressAxes(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var assembly strings.Builder + assembly.WriteString("TEXT f(SB),4,$0-0\n") + count := 0 + appendRow := func(op, src, dst string) { + row := op + " " + src + "," + dst + ins := x86YMMReadSource(t, row) + before := fmt.Sprintf("%#v", ins) + if _, known, err := x86GPSIMDTransferProperties(ins, arch); !known || err != nil { + t.Fatal("independent valid axis not consumed", row, err) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("grammar mutated original source") + } + fmt.Fprintln(&assembly, row) + count++ + } + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + legacy := !strings.HasPrefix(op, "V") + gps := []string{"AX", "BX", "CX", "DX", "DI", "SI", "BP", "SP"} + if arch == "amd64" { + gps = append(gps, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + } + if arch == "amd64" || !legacy || op == "MOVL" { + for _, gp := range gps { + appendRow(op, gp, "X0") + appendRow(op, "X0", gp) + } + } + limit := 8 + if arch == "amd64" { + limit = 16 + if !legacy { + limit = 32 + } + } + for index := 0; index < limit; index++ { + x := fmt.Sprint("X", index) + appendRow(op, "3(AX)", x) + appendRow(op, x, "5(BX)") + if op == "MOVQ" || op == "MOVD" || op == "VMOVQ" { + appendRow(op, x, "X0") + appendRow(op, x, x) + } + } + for _, memory := range []string{"(AX)(CX*1)", "7(AX)(CX*2)", "-9(AX)(CX*4)", "11(AX)(CX*8)"} { + appendRow(op, memory, "X1") + appendRow(op, "X1", memory) + } + } + assembly.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, assembly.String(), "", true) + want := 552 + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + want, triples = 216, []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + if count != want { + t.Fatalf("independent register/address denominator changed: %d, want %d", count, want) + } + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + pkg := x86FloatMulDivShapesPackage(t, "package shapes\nfunc f()\n", arch) + for _, triple := range triples { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "transfer-axes-text.ll", "transfer-axes-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "transfer-axes-module.ll", "transfer-axes-module.o", module.String()) + goSource := strings.Replace(assembly.String(), "TEXT f(SB)", "TEXT ·f(SB)", 1) + result, err := translateGoModuleInContext(ctx, pkg, []byte(goSource), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "transfer-axes-go.ll", "transfer-axes-go.o", result.Module.String()) + }) + } + }) + } +} + +func TestX86GPSIMDTransferPartialByteKnowledgeAndUnaffectedState(t *testing.T) { + for _, op := range []string{"MOVL", "MOVQ", "MOVD", "VMOVD", "VMOVQ"} { + width := 4 + if op == "MOVQ" || op == "MOVD" || op == "VMOVQ" { + width = 8 + } + for _, direction := range []string{"gp-to-x", "x-to-gp", "x-to-x"} { + if direction == "x-to-x" && width == 4 { + continue + } + t.Run(op+"/"+direction, func(t *testing.T) { + for seed := 0; seed < 256; seed++ { + initial := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, known: map[Reg]uint8{AX: uint8(seed), BX: 0x53}, + vectors: map[Reg]uint64{"Z0": uint64(seed), "Z1": 0xa735de42c9156ef0, "Z31": 0x1927}, + masks: map[Reg]bool{"K1": true}, + } + args := "AX,X1" + if direction == "x-to-gp" { + args = "X0,AX" + } else if direction == "x-to-x" { + args = "X0,X1" + } + ins := x86YMMReadSource(t, op+" "+args) + state, want := x86CloneCallerState(initial), x86CloneCallerState(initial) + low := uint64(seed) & (uint64(1)< +#include +#include +` + declarations.String() + ` +static int check(void (*program)(uint8_t*,const uint8_t*,uint64_t),const char *name, + unsigned width,int kind,int vector) { + uint64_t seeds[]={0,UINT64_MAX,UINT64_C(0x98af3bce72510d64),UINT64_C(0x8000000000000001)}; + for(unsigned s=0;s<4;s++) { + uint8_t input[82],original[82],output[84],before[84],want[84]; + for(unsigned i=0;i<82;i++) input[i]=(uint8_t)(i*37+149); + for(unsigned i=0;i<84;i++) output[i]=(uint8_t)(i*19+71); + memcpy(original,input,sizeof(input)); memcpy(before,output,sizeof(output)); + memcpy(want,before,sizeof(want)); + uint64_t gp=UINT64_MAX; + if(kind==1) { gp=0; memcpy(&gp,input+1,width); } + else if(kind==3) memcpy(want+4,input+1,width); + else { + for(unsigned i=0;i<64;i++) { + uint8_t value=before[1+i]; + if(i>=16) { if(vector) value=0; } + else if(i>=width) value=0; + else if(kind==0) value=(uint8_t)(seeds[s]>>(i*8)); + else if(kind==2) value=input[4+i]; + else if(kind==4) value=input[1+i]; + want[1+i]=value; + } + } + memcpy(want+65,&gp,8); want[73]=1; want[74]=0; + program(output+1,input+1,seeds[s]); + if(memcmp(input,original,sizeof(input)) || memcmp(output,want,sizeof(output))) { + for(unsigned i=0;i<84;i++) if(output[i]!=want[i]) + fprintf(stderr,"%s seed=%u byte=%u got=%x want=%x\n",name,s,i,output[i],want[i]); + return 1; + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "gp-simd-transfer-complete-bytes", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_gpsimd_overwritten_store_test.go b/amd64_gpsimd_overwritten_store_test.go new file mode 100644 index 000000000..2679f1d08 --- /dev/null +++ b/amd64_gpsimd_overwritten_store_test.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Two different native writes to the same address must remain observable even +// when no numerical consumer reads the first one. Plan 9 MOVD is the 64-bit +// MOVQ alias; it is not Intel's 32-bit MOVD. Expected widths below are independent +// of the production GP/SIMD transfer grammar and emitter. +func TestX86GPSIMDOverwrittenNativeStoresThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + forms := []struct { + op string + bits int + }{{"MOVL", 32}, {"MOVQ", 64}, {"MOVD", 64}, {"VMOVD", 32}, {"VMOVQ", 64}} + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var assembly, declarations strings.Builder + declarations.WriteString("package stores\n") + sigs := map[string]FuncSig{} + pointerBytes, pointerMove := 8, "MOVQ" + if target.arch == "386" { + pointerBytes, pointerMove = 4, "MOVL" + } + for index, form := range forms { + for _, memory := range []bool{true, false} { + name := fmt.Sprintf("Store%d_%t", index, memory) + destination := fmt.Sprintf("sink%d<>(SB)", index) + sig := FuncSig{Name: name, Ret: Void} + frame := 0 + if memory { + frame, destination = pointerBytes, "(AX)" + sig.Args = []LLVMType{Ptr} + sig.Frame = FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}} + fmt.Fprintf(&declarations, "func %s(p *[8]byte)\n", name) + } else { + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&assembly, "GLOBL %s,8,$8\n", destination) + } + sigs[name] = sig + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-%d\n", name, frame) + if memory { + fmt.Fprintf(&assembly, "%s p+0(FP),AX\n", pointerMove) + } + fmt.Fprintf(&assembly, "PXOR X0,X0\n%s X0,%s\nPCMPEQB X1,X1\n%s X1,%s\nRET\n", form.op, destination, form.op, destination) + } + } + // The original assembler independently establishes each store spelling + // and its real 386/amd64 operand acceptance before any translation. + goAssemblerOracleOutput(t, target.arch, assembly.String(), "", true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(name string) string { return strings.TrimPrefix(name, "·") } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + textIR, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "stores", declarations.String()) + goModule, err := translateGoModuleInContext(ctx, pkg, []byte(assembly.String()), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if goModule != nil && !goModule.Module.IsNil() { + defer goModule.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + for _, api := range []struct{ name, ir string }{{"text", textIR}, {"module", module.String()}, {"go-module", goModule.Module.String()}} { + t.Run(api.name, func(t *testing.T) { + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_default_O2=%t", optimized), func(t *testing.T) { + ir := api.ir + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + for index, form := range forms { + for _, memory := range []bool{true, false} { + name := fmt.Sprintf("Store%d_%t", index, memory) + t.Run(form.op+fmt.Sprintf("/OpMem=%t", memory), func(t *testing.T) { + x86GPSIMDCheckNativeStoreEffects(t, ir, name, form.bits) + }) + } + } + // Object emission is a separate backend-O0 axis, both before + // and after the explicit LLVM IR default pass pipeline. + compileLLVMToObject(t, llc, target.triple, "native-stores.ll", "native-stores.o", ir) + }) + } + }) + } + }) + } +} + +func x86GPSIMDCheckNativeStoreEffects(t *testing.T, ir, name string, bits int) { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := parseIRModuleInContext(ctx, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + function := module.NamedFunction(name) + if function.IsNil() { + t.Fatal("native store function disappeared", name) + } + stores, observable := 0, 0 + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for instruction := block.FirstInstruction(); !instruction.IsNil(); instruction = llvm.NextInstruction(instruction) { + if instruction.IsAStoreInst().IsNil() || x86GPSIMDStoreTargetsBackingAlloca(instruction.Operand(1)) { + continue // Register banks and synthesized ABI0 frames are not native writes. + } + typ := instruction.Operand(0).Type() + if typ.TypeKind() != llvm.IntegerTypeKind { + t.Fatal("native scalar write widened to a non-integer type", instruction.String()) + } + stores++ + if typ.IntTypeWidth() != bits { + t.Errorf("native store width=%d want=%d: %s", typ.IntTypeWidth(), bits, instruction.String()) + } + if instruction.IsVolatile() { + observable++ + } + } + } + t.Logf("native stores=%d observable=%d exact-width=%d", stores, observable, bits) + if stores != 2 || observable != 2 { + t.Errorf("overwritten native store effect lost: stores=%d observable=%d want=2/2\n%s", stores, observable, function.String()) + } +} + +func x86GPSIMDStoreTargetsBackingAlloca(pointer llvm.Value) bool { + for depth := 0; depth < 16; depth++ { + if !pointer.IsAAllocaInst().IsNil() { + return true + } + if pointer.IsAGetElementPtrInst().IsNil() && pointer.IsABitCastInst().IsNil() { + return false + } + pointer = pointer.Operand(0) + } + return false +} diff --git a/amd64_helper_edge_test.go b/amd64_helper_edge_test.go index 8b3a077c9..08f1f180a 100644 --- a/amd64_helper_edge_test.go +++ b/amd64_helper_edge_test.go @@ -1,6 +1,8 @@ package plan9asm import ( + "errors" + "math" "strings" "testing" ) @@ -154,14 +156,29 @@ func TestAMD64CtxHelperEdges(t *testing.T) { t.Fatalf("evalFPToI64(%d) = (%q, %v)", off, got, err) } } + if got, err := c.evalFPToF32(56); err != nil || got == "" { + t.Fatalf("evalFPToF32(float) = (%q, %v)", got, err) + } + if got, err := c.evalFPToF32(32); err != nil || got == "" { + t.Fatalf("evalFPToF32(i32 bits) = (%q, %v)", got, err) + } + if _, err := c.evalFPToF32(999); err == nil { + t.Fatal("evalFPToF32(missing slot) unexpectedly succeeded") + } c.fpParams[64] = FrameSlot{Offset: 64, Type: LLVMType("v4i32"), Index: 0} if _, err := c.evalFPToI64(64); err == nil { t.Fatalf("evalFPToI64(unsupported type) unexpectedly succeeded") } + if _, err := c.evalFPToF32(64); err == nil { + t.Fatal("evalFPToF32(unsupported type) unexpectedly succeeded") + } c.fpParams[72] = FrameSlot{Offset: 72, Type: I64, Index: 99} if _, err := c.evalFPToI64(72); err == nil { t.Fatalf("evalFPToI64(invalid index) unexpectedly succeeded") } + if _, err := c.evalFPToF32(72); err == nil { + t.Fatal("evalFPToF32(invalid index) unexpectedly succeeded") + } if err := c.storeFPResult(80, I64, "11"); err != nil { t.Fatalf("storeFPResult(i64->i32) error = %v", err) @@ -285,8 +302,8 @@ func TestAMD64CtxHelperEdges(t *testing.T) { "alloca <64 x i8>", "alloca i64", "ptrtoint ptr %arg0 to i64", - "bitcast double %arg6 to i64", - "bitcast float %arg7 to i32", + "load double, ptr %fp_arg_48", + "load float, ptr %fp_arg_56", "trunc i64 11 to i32", "inttoptr i64 12 to ptr", "bitcast i64 13 to double", @@ -356,9 +373,14 @@ func TestAMD64AtomicAndBranchEdges(t *testing.T) { c.blockBase = []int{0, 1, 2} c.blockByIdx = map[int]int{0: 0, 1: 1, 2: 2} - if ok, term, err := c.lowerAtomic("LOCK", Instr{Raw: "LOCK"}); !ok || term || err != nil { + beforeStandaloneLock := b.Len() + beforeLockTemporary, beforeLockPrefix := c.tmp, c.namedAtomicLock + if ok, term, err := c.lowerAtomic("LOCK", Instr{Raw: "LOCK"}); !ok || term || !errors.Is(err, ErrProbeNeedsContext) { t.Fatalf("lowerAtomic(LOCK) = (%v, %v, %v)", ok, term, err) } + if b.Len() != beforeStandaloneLock || c.tmp != beforeLockTemporary || c.namedAtomicLock != beforeLockPrefix { + t.Fatal("standalone LOCK changed IR or temporary state without a complete adjacent source operation") + } if ok, _, err := c.lowerAtomic("CMPXCHGL", Instr{ Raw: "CMPXCHGL CX, 8(BX)", Args: []Operand{ @@ -505,15 +527,32 @@ func TestAMD64AtomicAndBranchEdges(t *testing.T) { if err := c.tailCallAndRet(Operand{Kind: OpReg, Reg: AX}); err == nil { t.Fatalf("tailCallAndRet(non-sym) unexpectedly succeeded") } + + var b386 strings.Builder + c386 := newX86Ctx(&b386, Func{}, FuncSig{Name: "example.ptr386", Ret: Ptr}, testResolveSym("example"), map[string]FuncSig{ + "example.word386": {Name: "example.word386", Ret: I32}, + }, "386", "i386-unknown-linux-gnu", false) + if err := c386.emitEntryAllocas(); err != nil { + t.Fatal(err) + } + if err := c386.tailCallAndRet(Operand{Kind: OpSym, Sym: "word386(SB)"}); err != nil { + t.Fatalf("386 tailCallAndRet(i32 -> ptr) error = %v", err) + } + if got := b386.String(); !strings.Contains(got, "inttoptr i32") || !strings.Contains(got, "ret ptr") { + t.Fatalf("386 tailCallAndRet(i32 -> ptr) output:\n%s", got) + } if err := c.tailCallIndirectAddrAndRet("123"); err != nil { t.Fatalf("tailCallIndirectAddrAndRet() error = %v", err) } out := b.String() + if strings.Contains(out, "cmpxchg ptr") { + t.Fatal("native CMPXCHGL silently acquired software atomic alignment or LOCK semantics") + } for _, want := range []string{ - "cmpxchg ptr", - "atomicrmw add ptr", - "atomicrmw xchg ptr", + "asm sideeffect \"cmpxchgl $3, $2; sete $1\"", + "asm sideeffect \"lock; xadd", + "asm sideeffect \"xchg", "atomicrmw and ptr", "icmp eq i8", "icmp slt i8", @@ -556,13 +595,20 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { t.Fatalf("lowerArith(%s) = (%v, %v, %v)", op, ok, term, err) } } + if ok, term, err := c.lowerScalarShiftRotate("RCRQ", Instr{Raw: "RCRQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { + t.Fatalf("lowerScalarShiftRotate(RCRQ) = (%v, %v, %v)", ok, term, err) + } mustLower("PUSHQ", Instr{Raw: "PUSHQ $1", Args: []Operand{{Kind: OpImm, Imm: 1}}}) mustLower("POPQ", Instr{Raw: "POPQ CX", Args: []Operand{{Kind: OpReg, Reg: CX}}}) mustLower("PUSHFQ", Instr{Raw: "PUSHFQ"}) mustLower("POPFQ", Instr{Raw: "POPFQ"}) mustLower("LFENCE", Instr{Raw: "LFENCE"}) - mustLower("UNDEF", Instr{Raw: "UNDEF"}) + trap, trapOutput := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + if known, terminated, err := trap.lowerArith("UNDEF", Instr{Raw: "UNDEF"}); !known || !terminated || err != nil || + !strings.Contains(trapOutput.String(), "asm sideeffect \"ud2\"") { + t.Fatalf("UNDEF must terminate with its architectural fault: %t/%t/%v", known, terminated, err) + } mustLower("RDTSC", Instr{Raw: "RDTSC"}) mustLower(OpCPUID, Instr{Raw: "CPUID"}) mustLower(OpXGETBV, Instr{Raw: "XGETBV"}) @@ -572,7 +618,6 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { mustLower("STOSQ", Instr{Raw: "STOSQ"}) mustLower("NEGL", Instr{Raw: "NEGL AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}) mustLower("NEGL", Instr{Raw: "NEGL 4(BX)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 4}}}}) - mustLower("RCRQ", Instr{Raw: "RCRQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ADDQ", Instr{Raw: "ADDQ $2, AX", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: AX}}}) mustLower("SUBQ", Instr{Raw: "SUBQ CX, 8(BX)", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) mustLower("XORQ", Instr{Raw: "XORQ $3, AX", Args: []Operand{{Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: AX}}}) @@ -581,8 +626,8 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { mustLower("ADCQ", Instr{Raw: "ADCQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ADCB", Instr{Raw: "ADCB $1, AL", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AL}}}) mustLower("SBBQ", Instr{Raw: "SBBQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("ADCXQ", Instr{Raw: "ADCXQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) - mustLower("ADOXQ", Instr{Raw: "ADOXQ $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + mustLower("ADCXQ", Instr{Raw: "ADCXQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) + mustLower("ADOXQ", Instr{Raw: "ADOXQ CX, AX", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: AX}}}) mustLower("ADDL", Instr{Raw: "ADDL $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) mustLower("ADDB", Instr{Raw: "ADDB CL, DX", Args: []Operand{{Kind: OpReg, Reg: CL}, {Kind: OpReg, Reg: DX}}}) mustLower("SUBL", Instr{Raw: "SUBL CX, 12(BX)", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 12}}}}) @@ -601,7 +646,11 @@ func TestAMD64ArithmeticCoverage(t *testing.T) { mustLower("LEAQ", Instr{Raw: "LEAQ 8(BX)(CX*2), DI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Index: CX, Scale: 2, Off: 8}}, {Kind: OpReg, Reg: DI}}}) mustLower("LEAL", Instr{Raw: "LEAL 4(BX), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 4}}, {Kind: OpReg, Reg: SI}}}) mustLower("LEAQ", Instr{Raw: "LEAQ ret+16(FP), R8", Args: []Operand{{Kind: OpFP, FPOffset: 16}, {Kind: OpReg, Reg: Reg("R8")}}}) - mustLower("LEAQ", Instr{Raw: "LEAQ $ret+16(FP), R9", Args: []Operand{{Kind: OpFPAddr, FPOffset: 16}, {Kind: OpReg, Reg: Reg("R9")}}}) + // TYPE_ADDR is not Go's LEA Ym source; retain the former false-positive + // spelling as a rejection, independently checked by the full LEA oracle. + if _, _, err := c.lowerArith("LEAQ", Instr{Raw: "LEAQ $ret+16(FP), R9", Args: []Operand{{Kind: OpFPAddr, FPOffset: 16}, {Kind: OpReg, Reg: Reg("R9")}}}); err == nil { + t.Fatal("LEAQ accepted a TYPE_ADDR operand outside Go's Ym/Yrl row") + } mustLower("LEAQ", Instr{Raw: "LEAQ global<>(SB), AX", Args: []Operand{{Kind: OpSym, Sym: "global<>(SB)"}, {Kind: OpReg, Reg: AX}}}) mustLower("POPCNTL", Instr{Raw: "POPCNTL AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}) mustLower("POPCNTQ", Instr{Raw: "POPCNTQ AX, CX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: CX}}}) @@ -715,8 +764,8 @@ func TestAMD64EcosystemScalarValidationCoverage(t *testing.T) { {Raw: string(op) + " X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}, {Raw: string(op) + " AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, } { - if ok, term, err := c.lowerArith(op, ins); ok || term || err != nil { - t.Fatalf("lowerArith(%s) invalid form = (%v, %v, %v), want unsupported", op, ok, term, err) + if ok, term, err := c.lowerArith(op, ins); !ok || term || err == nil { + t.Fatalf("lowerArith(%s) invalid form = (%v, %v, %v), want recognized validation error", op, ok, term, err) } } } @@ -814,7 +863,7 @@ func TestAMD64ADCBUsesCarryFlag(t *testing.T) { "store i1 true, ptr %flags_cf", "load i1, ptr %flags_cf", "zext i1 %", - "add i8", + "add i16", "zext i8", "icmp ugt i16", "shl i64", @@ -850,11 +899,22 @@ func TestAMD64VectorCoverage(t *testing.T) { t.Fatalf("lowerVec(%s) = (%v, %v, %v)", op, ok, term, err) } } + mustLowerTransfer := func(op Op, ins Instr) { + t.Helper() + ins.Op = op + if ok, term, err := c.lowerVectorScalarIntegerMove(op, ins); !ok || term || err != nil { + t.Fatalf("lowerVectorScalarIntegerMove(%s) = (%v, %v, %v)", op, ok, term, err) + } + } - mustLower("MOVL", Instr{Raw: "MOVL $1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}}}) - mustLower("MOVD", Instr{Raw: "MOVD AX, X1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X1")}}}) - mustLower("MOVQ", Instr{Raw: "MOVQ BX, X2", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: Reg("X2")}}}) - mustLower("MOVQ", Instr{Raw: "MOVQ X2, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X2")}, {Kind: OpReg, Reg: AX}}}) + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\nMOVL $1,AX\nMOVL AX,X0\nMOVD AX,X1\nMOVQ BX,X2\nMOVQ X2,AX\nRET\n", "", true) + if err := c.storeReg(AX, "1"); err != nil { + t.Fatal(err) + } + mustLowerTransfer("MOVL", Instr{Raw: "MOVL AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}) + mustLowerTransfer("MOVD", Instr{Raw: "MOVD AX, X1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X1")}}}) + mustLowerTransfer("MOVQ", Instr{Raw: "MOVQ BX, X2", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: Reg("X2")}}}) + mustLowerTransfer("MOVQ", Instr{Raw: "MOVQ X2, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X2")}, {Kind: OpReg, Reg: AX}}}) mustLower("KXORQ", Instr{Raw: "KXORQ K1, K2, K3", Args: []Operand{{Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K2")}, {Kind: OpReg, Reg: Reg("K3")}}}) mustLower("KMOVB", Instr{Raw: "KMOVB AX, K1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("K1")}}}) mustLower("KMOVW", Instr{Raw: "KMOVW K1, 8(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("K1")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) @@ -923,7 +983,9 @@ func TestAMD64VectorCoverage(t *testing.T) { mustLower("MOVOU", Instr{Raw: "MOVOU X0, 112(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 112}}}}) mustLower("PXOR", Instr{Raw: "PXOR X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) mustLower("PAND", Instr{Raw: "PAND X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) - mustLower("PANDN", Instr{Raw: "PANDN X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) + if ok, term, err := c.lowerPackedAndNot("PANDN", Instr{Raw: "PANDN X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}); !ok || term || err != nil { + t.Fatalf("lowerPackedAndNot(PANDN) = (%v, %v, %v)", ok, term, err) + } mustLower("PADDL", Instr{Raw: "PADDL X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) mustLower("PADDQ", Instr{Raw: "PADDQ X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) mustLower("PSUBL", Instr{Raw: "PSUBL X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) @@ -966,9 +1028,6 @@ func TestAMD64VectorCoverage(t *testing.T) { if got := llvmShiftLeftBytesMask(3); !strings.Contains(got, "i32 16") { t.Fatalf("llvmShiftLeftBytesMask(3) = %q", got) } - if got := llvmAlignRightBytesMask(20); !strings.Contains(got, "i32 20") { - t.Fatalf("llvmAlignRightBytesMask(20) = %q", got) - } if got := llvmAllOnesI8Vec(4); got != "" { t.Fatalf("llvmAllOnesI8Vec(4) = %q", got) } @@ -1112,6 +1171,7 @@ func TestAMD64FPMovCoverage(t *testing.T) { }, } c, b := newAMD64CtxWithFuncForTest(t, fn, sig, nil) + c.goarch, c.targetTriple = "amd64", "x86_64-unknown-linux-gnu" check := func(kind string, ins Instr, ok bool, err error) { t.Helper() if err != nil { @@ -1152,7 +1212,7 @@ func TestAMD64FPMovCoverage(t *testing.T) { {Op: "ANDNPD", Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}, Raw: "ANDNPD X1, X0"}, {Op: "ORPD", Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}, Raw: "ORPD X1, X0"}, {Op: "XORPS", Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}, Raw: "XORPS X1, X0"}, - {Op: "MOVSD", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: "X0"}}, Raw: "MOVSD $7, X0"}, + {Op: "MOVSD", Args: []Operand{{Kind: OpImm, Imm: int64(math.Float64bits(7)), ImmIsFloat: true}, {Kind: OpReg, Reg: "X0"}}, Raw: "MOVSD $7.0, X0"}, {Op: "MOVSD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 16}}}, Raw: "MOVSD X0, 16(BX)"}, {Op: "MOVSD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpSym, Sym: "example.f64(SB)"}}, Raw: "MOVSD X0, example.f64(SB)"}, {Op: "MOVSD", Args: []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpFP, FPOffset: 40}}, Raw: "MOVSD X0, ret+40(FP)"}, @@ -1181,7 +1241,7 @@ func TestAMD64FPMovCoverage(t *testing.T) { c.setCmpFlags("1", "2") for _, ins := range []Instr{ {Op: "CMOVQLT", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpReg, Reg: DX}}, Raw: "CMOVQLT CX, DX"}, - {Op: "MOVLQSX", Args: []Operand{{Kind: OpImm, Imm: 21}, {Kind: OpReg, Reg: AX}}, Raw: "MOVLQSX $21, AX"}, + {Op: "MOVLQSX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}, Raw: "MOVLQSX AX, AX"}, {Op: "MOVLQSX", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}}, Raw: "MOVLQSX BX, CX"}, {Op: "MOVLQSX", Args: []Operand{{Kind: OpFP, FPOffset: 16}, {Kind: OpReg, Reg: DX}}, Raw: "MOVLQSX arg+16(FP), DX"}, {Op: "MOVLQSX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: SI, Off: 4}}, {Kind: OpReg, Reg: SI}}, Raw: "MOVLQSX 4(SI), SI"}, @@ -1245,18 +1305,19 @@ func TestAMD64FPMovCoverage(t *testing.T) { out := b.String() for _, want := range []string{ - "fadd double", - "fsub double", - "fmul double", - "fdiv double", + "call double @llvm.experimental.constrained.fadd.f64", + "call double @__plan9asm_add_sub_special_f64", + "call double @llvm.experimental.constrained.fsub.f64", + "call double @llvm.experimental.constrained.fmul.f64", + "call double @llvm.experimental.constrained.fdiv.f64", "fneg double", - "fcmp ueq double", + `call { i8, i8, i8 } asm sideeffect "comisd $3, $4; setb $0; sete $1; setp $2"`, "fcmp ord double", "call double @llvm.sqrt.f64", - "call double @llvm.rint.f64", - "sitofp i64", - "sitofp i32", - "fptosi double", + `call i32 asm sideeffect "cvtsd2sil $1, $0"`, + "@llvm.experimental.constrained.sitofp.f64.i64", + "@llvm.experimental.constrained.sitofp.f64.i32", + `call i64 asm sideeffect "cvttsd2siq $1, $0"`, "select i1", "bitcast i64", "bitcast double", @@ -1265,7 +1326,9 @@ func TestAMD64FPMovCoverage(t *testing.T) { "store i32", "store i16", "store i8", - `load <16 x i8>, ptr @"example.vec"`, + // Native MOVAPD must retain its aligned opcode and dead-read fault; + // an ordinary LLVM load is no longer the correct lowering contract. + `call <4 x float> asm sideeffect "movapd ($1), $0", "=x,r,~{memory}"(ptr @"example.vec")`, } { if !strings.Contains(out, want) { t.Fatalf("missing %q in output:\n%s", want, out) @@ -1307,13 +1370,13 @@ func TestAMD64CmpBtCoverage(t *testing.T) { } } - check("CMPB", Instr{Raw: "CMPB $1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}}) + check("CMPB", Instr{Raw: "CMPB AX, $1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpImm, Imm: 1}}}) check("CMPW", Instr{Raw: "CMPW 4(BX), CX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 4}}, {Kind: OpReg, Reg: CX}}}) check("CMPL", Instr{Raw: "CMPL arg+0(FP), DI", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: DI}}}) check("CMPQ", Instr{Raw: "CMPQ example.global(SB), SI", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: SI}}}) check("TESTB", Instr{Raw: "TESTB AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}) check("TESTW", Instr{Raw: "TESTW $7, arg+0(FP)", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpFP, FPOffset: 0}}}) - check("TESTL", Instr{Raw: "TESTL $const, 8(BX)", Args: []Operand{{Kind: OpSym, Sym: "$const"}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) + check("TESTL", Instr{Raw: "TESTL $7, 8(BX)", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}}) check("TESTQ", Instr{Raw: "TESTQ example.global(SB), DI", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: DI}}}) check("BTQ", Instr{Raw: "BTQ $3, AX", Args: []Operand{{Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: AX}}}) check("BTSQ", Instr{Raw: "BTSQ DX, AX", Args: []Operand{{Kind: OpReg, Reg: DX}, {Kind: OpReg, Reg: AX}}}) @@ -1329,7 +1392,7 @@ func TestAMD64CmpBtCoverage(t *testing.T) { if _, _, err := c.lowerCmpBt("TESTQ", Instr{Raw: "TESTQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { t.Fatalf("short TESTQ unexpectedly succeeded") } - if _, _, err := c.lowerCmpBt("BTQ", Instr{Raw: "BTQ AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}); err == nil { + if _, _, err := c.lowerCmpBt("BTQ", Instr{Raw: "BTQ (AX), BX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: BX}}}); err == nil { t.Fatalf("bad BTQ unexpectedly succeeded") } if _, _, err := c.lowerCmpBt("BTSQ", Instr{Raw: "BTSQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { @@ -1338,8 +1401,8 @@ func TestAMD64CmpBtCoverage(t *testing.T) { if _, _, err := c.lowerCmpBt("BTRQ", Instr{Raw: "BTRQ AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { t.Fatalf("short BTRQ unexpectedly succeeded") } - if got, err := c.evalIntSized(Operand{Kind: OpSym, Sym: "$const"}, I32); err != nil || got != "0" { - t.Fatalf("evalIntSized($const) = (%q, %v)", got, err) + if _, err := c.evalIntSized(Operand{Kind: OpSym, Sym: "$const"}, I32); err == nil { + t.Fatal("evalIntSized($const) invented a value for an unresolved constant") } if _, err := c.evalIntSized(Operand{Kind: OpSym, Sym: "bad"}, I32); err == nil { t.Fatalf("evalIntSized(bad sym) unexpectedly succeeded") @@ -1358,7 +1421,7 @@ func TestAMD64CmpBtCoverage(t *testing.T) { "store i1 false, ptr %flags_cf", "lshr i64", "shl i64 1", - "load i64, ptr @\"example.global\"", + "load volatile i64, ptr @\"example.global\"", } { if !strings.Contains(out, want) { t.Fatalf("missing %q in output:\n%s", want, out) @@ -1403,7 +1466,7 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { } } - checkMov("MOVLQSX", Instr{Raw: "MOVLQSX $7, AX", Args: []Operand{{Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: AX}}}) + checkMov("MOVLQSX", Instr{Raw: "MOVLQSX AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}) checkMov("MOVLQSX", Instr{Raw: "MOVLQSX BX, CX", Args: []Operand{{Kind: OpReg, Reg: BX}, {Kind: OpReg, Reg: CX}}}) checkMov("MOVLQSX", Instr{Raw: "MOVLQSX arg+0(FP), DX", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: DX}}}) checkMov("MOVLQSX", Instr{Raw: "MOVLQSX 8(BX), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: SI}}}) @@ -1457,8 +1520,8 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { } cBadReg, _ := newAMD64CtxWithFuncForTest(t, Func{}, sig, nil) - if ok, term, err := cBadReg.lowerMov("MOVB", Instr{Raw: "MOVB BAD, ret+24(FP)", Args: []Operand{{Kind: OpReg, Reg: Reg("BAD")}, {Kind: OpFP, FPOffset: 24}}}); !ok || term || err != nil { - t.Fatalf("MOVB missing src reg = (%v, %v, %v)", ok, term, err) + if ok, term, err := cBadReg.lowerMov("MOVB", Instr{Raw: "MOVB BAD, ret+24(FP)", Args: []Operand{{Kind: OpReg, Reg: Reg("BAD")}, {Kind: OpFP, FPOffset: 24}}}); !ok || term || err == nil { + t.Fatalf("MOVB invalid src reg = (%v, %v, %v), want handled error", ok, term, err) } if ok, term, err := cBadReg.lowerMov("CMOVQLT", Instr{Raw: "CMOVQLT BAD, BX", Args: []Operand{{Kind: OpReg, Reg: Reg("BAD")}, {Kind: OpReg, Reg: BX}}}); !ok || term || err != nil { t.Fatalf("CMOVQLT missing regs = (%v, %v, %v)", ok, term, err) @@ -1471,20 +1534,20 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { } cBadMem, _ := newAMD64CtxWithFuncForTest(t, Func{}, sig, nil) - if ok, term, err := cBadMem.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX 8(BAD), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: SI}}}); !ok || term || err != nil { + if ok, term, err := cBadMem.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX 8(BAD), SI", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: SI}}}); !ok || term || err == nil { t.Fatalf("MOVLQSX bad mem = (%v, %v, %v)", ok, term, err) } if ok, term, err := cBadMem.lowerMov("MOVW", Instr{Raw: "MOVW arg+0(FP), 8(BAD)", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}}}); !ok || term || err != nil { t.Fatalf("MOVW bad dst mem = (%v, %v, %v)", ok, term, err) } - if ok, term, err := cBadMem.lowerMov("MOVQ", Instr{Raw: "MOVQ 8(BAD), AX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { + if ok, term, err := cBadMem.lowerMov("MOVQ", Instr{Raw: "MOVQ 8(BAD), AX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 8}}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("MOVQ bad src mem = (%v, %v, %v)", ok, term, err) } if ok, term, err := cBadMem.lowerMov("MOVL", Instr{Raw: "MOVL 24(BAD), example.dest(SB)", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: Reg("BAD"), Off: 24}}, {Kind: OpSym, Sym: "example.dest(SB)"}}}); !ok || term || err != nil { t.Fatalf("MOVL bad src mem = (%v, %v, %v)", ok, term, err) } - if ok, term, err := c.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX broken, AX", Args: []Operand{{Kind: OpSym, Sym: "broken"}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { + if ok, term, err := c.lowerMov("MOVLQSX", Instr{Raw: "MOVLQSX broken, AX", Args: []Operand{{Kind: OpSym, Sym: "broken"}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("MOVLQSX bad sym = (%v, %v, %v)", ok, term, err) } if ok, term, err := c.lowerMov("MOVB", Instr{Raw: "MOVB broken, AX", Args: []Operand{{Kind: OpSym, Sym: "broken"}, {Kind: OpReg, Reg: AX}}}); !ok || term || err != nil { @@ -1543,7 +1606,7 @@ func TestAMD64MovSyscallAndCRC32Coverage(t *testing.T) { if ok, term, err := sysc.lowerSyscall("BAD", Instr{}); ok || term || err != nil { t.Fatalf("lowerSyscall(BAD) = (%v, %v, %v)", ok, term, err) } - if _, _, err := sysc.lowerCrc32("CRC32B", Instr{Raw: "CRC32B AX, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}}}); err == nil { + if _, _, err := sysc.lowerCrc32("CRC32B", Instr{Raw: "CRC32B $1, BX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: BX}}}); err == nil { t.Fatalf("bad CRC32B unexpectedly succeeded") } if ok, term, err := sysc.lowerCrc32("BAD", Instr{}); ok || term || err != nil { @@ -1654,7 +1717,7 @@ func TestAMD64BranchCoverageDeep(t *testing.T) { "or i1", "call i64 @\"example.tail\"", "ret i64", - "br label %V1", + "br label %" + amd64LLVMBlockName("V1"), } { if !strings.Contains(out, want) { t.Fatalf("missing %q in output:\n%s", want, out) @@ -1753,7 +1816,7 @@ func TestAMD64FPExtraCoverage(t *testing.T) { if _, _, err := c.lowerFP("MOVSD", Instr{Raw: "MOVSD X0, label", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}); err == nil { t.Fatalf("MOVSD bad dst unexpectedly succeeded") } - if ok, term, err := c.lowerFP("ADDSD", Instr{Raw: "ADDSD X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { + if ok, term, err := c.lowerFP("ADDSD", Instr{Raw: "ADDSD X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("ADDSD non-X dst = (%v, %v, %v)", ok, term, err) } if ok, term, err := c.lowerFP("SQRTSD", Instr{Raw: "SQRTSD X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { @@ -1768,10 +1831,10 @@ func TestAMD64FPExtraCoverage(t *testing.T) { if _, _, err := c.lowerFP("CMPSD", Instr{Raw: "CMPSD X0, X1, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}, {Kind: OpReg, Reg: AX}}}); err == nil { t.Fatalf("CMPSD reg predicate unexpectedly succeeded") } - if ok, term, err := c.lowerFP("CVTSQ2SD", Instr{Raw: "CVTSQ2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { + if ok, term, err := c.lowerFP("CVTSQ2SD", Instr{Raw: "CVTSQ2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("CVTSQ2SD non-X dst = (%v, %v, %v)", ok, term, err) } - if ok, term, err := c.lowerFP("CVTSL2SD", Instr{Raw: "CVTSL2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); ok || term || err != nil { + if ok, term, err := c.lowerFP("CVTSL2SD", Instr{Raw: "CVTSL2SD AX, AX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: AX}}}); !ok || term || err == nil { t.Fatalf("CVTSL2SD non-X dst = (%v, %v, %v)", ok, term, err) } @@ -1785,8 +1848,9 @@ func TestAMD64FPExtraCoverage(t *testing.T) { "fcmp uno double", "fcmp une double", "fcmp ugt double", - "fadd double", - "fmul double", + "call double @llvm.experimental.constrained.fadd.f64", + "call double @__plan9asm_add_sub_special_f64", + "call double @llvm.fma.f64", "fneg double", } { if !strings.Contains(out, want) { @@ -1813,7 +1877,7 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { Args: []LLVMType{I64}, Ret: Void, Frame: FrameLayout{ - Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0}}, + Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}, }, } c, b := newAMD64CtxWithFuncForTest(t, fn, sig, nil) @@ -1845,6 +1909,13 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { t.Fatalf("lowerVec(%s %q) = (%v, %v, %v)", op, ins.Raw, ok, term, err) } } + checkTransfer := func(op Op, ins Instr) { + t.Helper() + ins.Op = op + if ok, term, err := c.lowerVectorScalarIntegerMove(op, ins); !ok || term || err != nil { + t.Fatalf("lowerVectorScalarIntegerMove(%s %q) = (%v, %v, %v)", op, ins.Raw, ok, term, err) + } + } check("VZEROUPPER", Instr{Raw: "VZEROUPPER"}) check("VZEROALL", Instr{Raw: "VZEROALL"}) @@ -1859,14 +1930,14 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { check("MOVUPS", Instr{Raw: "MOVUPS 8(BX), X0", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("X0")}}}) check("MOVAPS", Instr{Raw: "MOVAPS X0, 16(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 16}}}}) check("MOVO", Instr{Raw: "MOVO X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}) - check("MOVQ", Instr{Raw: "MOVQ arg+0(FP), X0", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: Reg("X0")}}}) - check("MOVQ", Instr{Raw: "MOVQ 24(BX), X1", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 24}}, {Kind: OpReg, Reg: Reg("X1")}}}) - check("MOVQ", Instr{Raw: "MOVQ example.global(SB), X2", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: Reg("X2")}}}) - check("MOVQ", Instr{Raw: "MOVQ X0, 32(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 32}}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ arg+0(FP), X0", Args: []Operand{{Kind: OpFP, FPOffset: 0}, {Kind: OpReg, Reg: Reg("X0")}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ 24(BX), X1", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 24}}, {Kind: OpReg, Reg: Reg("X1")}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ example.global(SB), X2", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: Reg("X2")}}}) + checkTransfer("MOVQ", Instr{Raw: "MOVQ X0, 32(BX)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 32}}}}) check("MOVOU", Instr{Raw: "MOVOU example.global(SB), X0", Args: []Operand{{Kind: OpSym, Sym: "example.global(SB)"}, {Kind: OpReg, Reg: Reg("X0")}}}) check("MOVOA", Instr{Raw: "MOVOA X0, example.out(SB)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpSym, Sym: "example.out(SB)"}}}) check("KMOVB", Instr{Raw: "KMOVB 8(BX), K1", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("K1")}}}) - check("KMOVQ", Instr{Raw: "KMOVQ 8(BX), AX", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: AX}}}) + check("KMOVD", Instr{Raw: "KMOVD K1, K7", Args: []Operand{{Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K7")}}}) check("VMOVDQU64", Instr{Raw: "VMOVDQU64 example.zin(SB), Z0", Args: []Operand{{Kind: OpSym, Sym: "example.zin(SB)"}, {Kind: OpReg, Reg: Reg("Z0")}}}) check("VMOVDQU64", Instr{Raw: "VMOVDQU64 Z0, example.zout(SB)", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpSym, Sym: "example.zout(SB)"}}}) check("VMOVDQU", Instr{Raw: "VMOVDQU example.yin(SB), Y0", Args: []Operand{{Kind: OpSym, Sym: "example.yin(SB)"}, {Kind: OpReg, Reg: Reg("Y0")}}}) @@ -1875,16 +1946,16 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { check("VMOVDQU", Instr{Raw: "VMOVDQU X0, example.xout(SB)", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpSym, Sym: "example.xout(SB)"}}}) check("PEXTRB", Instr{Raw: "PEXTRB $1, X1, example.byteout(SB)", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X1")}, {Kind: OpSym, Sym: "example.byteout(SB)"}}}) - if _, _, err := c.lowerVec("MOVL", Instr{Raw: "MOVL label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { + if _, _, err := c.lowerVectorScalarIntegerMove("MOVL", Instr{Op: "MOVL", Raw: "MOVL label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { t.Fatalf("MOVL bad src unexpectedly succeeded") } - if _, _, err := c.lowerVec("MOVQ", Instr{Raw: "MOVQ label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { + if _, _, err := c.lowerVectorScalarIntegerMove("MOVQ", Instr{Op: "MOVQ", Raw: "MOVQ label, X0", Args: []Operand{{Kind: OpIdent, Ident: "label"}, {Kind: OpReg, Reg: Reg("X0")}}}); err == nil { t.Fatalf("MOVQ bad src unexpectedly succeeded") } - if _, _, err := c.lowerVec("MOVQ", Instr{Raw: "MOVQ X0, label", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}); err == nil { + if _, _, err := c.lowerVectorScalarIntegerMove("MOVQ", Instr{Op: "MOVQ", Raw: "MOVQ X0, label", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}); err == nil { t.Fatalf("MOVQ bad dst unexpectedly succeeded") } - if ok, term, err := c.lowerVec("KXORQ", Instr{Raw: "KXORQ AX, K1, K2", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K2")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("KXORQ", Instr{Raw: "KXORQ AX, K1, K2", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("K2")}}}); !ok || term || err == nil { t.Fatalf("KXORQ non-k src = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("KMOVB", Instr{Raw: "KMOVB AX", Args: []Operand{{Kind: OpReg, Reg: AX}}}); err == nil { @@ -1902,19 +1973,19 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { if _, _, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, 7", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}}}); err == nil { t.Fatalf("VPERMB non-reg dst unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); !ok || term || err == nil { t.Fatalf("VPERMB wrong dst reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, 7, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: Reg("Z2")}}}); err == nil { t.Fatalf("VPERMB non-reg mask unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMB", Instr{Raw: "VPERMB Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); !ok || term || err == nil { t.Fatalf("VPERMB wrong mask reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VGF2P8AFFINEQB", Instr{Raw: "VGF2P8AFFINEQB Z0, Z1, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Z2")}}}); err == nil { t.Fatalf("VGF2P8AFFINEQB short form unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VGF2P8AFFINEQB", Instr{Raw: "VGF2P8AFFINEQB $1, Z0, Z1, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VGF2P8AFFINEQB", Instr{Raw: "VGF2P8AFFINEQB $1, Z0, Z1, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); !ok || term || err == nil { t.Fatalf("VGF2P8AFFINEQB wrong dst reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}}}); err == nil { @@ -1923,13 +1994,13 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { if _, _, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, 7", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}}}); err == nil { t.Fatalf("VPERMI2B non-reg dst unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}); !ok || term || err == nil { t.Fatalf("VPERMI2B wrong dst reg = (%v, %v, %v)", ok, term, err) } if _, _, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, 7, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpImm, Imm: 7}, {Kind: OpReg, Reg: Reg("Z2")}}}); err == nil { t.Fatalf("VPERMI2B non-reg mask unexpectedly succeeded") } - if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); ok || term || err != nil { + if ok, term, err := c.lowerVec("VPERMI2B", Instr{Raw: "VPERMI2B Z0, Z1, AX, Z2", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z2")}}}); !ok || term || err == nil { t.Fatalf("VPERMI2B wrong mask reg = (%v, %v, %v)", ok, term, err) } for _, tc := range []struct { @@ -1938,27 +2009,27 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { want string wantHandled bool }{ - {"VPOPCNTB", Instr{Raw: "VPOPCNTB Z0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, + {"VPOPCNTB", Instr{Raw: "VPOPCNTB Z0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", true}, {"VPCMPUQ", Instr{Raw: "VPCMPUQ $3, Z0, Z1, K1", Args: []Operand{{Kind: OpImm, Imm: 3}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("K1")}}}, "bad imm", true}, {"VPCMPUQ", Instr{Raw: "VPCMPUQ $1, Z0, Z1, AX", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: AX}}}, "wrong dst reg", false}, - {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, AX, Z1", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z1")}}}, "wrong mask", false}, - {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, K1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, + {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, AX, Z1", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Z1")}}}, "wrong mask", true}, + {"VPCOMPRESSQ", Instr{Raw: "VPCOMPRESSQ Z0, K1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("K1")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", true}, {"VPXORQ", Instr{Raw: "VPXORQ Z0, Z1, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong z dst", true}, - {"VPSHUFB", Instr{Raw: "VPSHUFB Y0, Y1, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: AX}}}, "wrong dst", false}, - {"VPSHUFD", Instr{Raw: "VPSHUFD $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", false}, + {"VPSHUFB", Instr{Raw: "VPSHUFB Y0, Y1, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: AX}}}, "wrong dst", true}, + {"VPSHUFD", Instr{Raw: "VPSHUFD $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", true}, {"VPSLLD", Instr{Raw: "VPSLLD Y0, Y1", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}}}, "short form", true}, - {"VPERM2I128", Instr{Raw: "VPERM2I128 $1, Y0, Y1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst", false}, - {"VINSERTI128", Instr{Raw: "VINSERTI128 $1, X0, Y0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong dst", false}, + {"VPERM2I128", Instr{Raw: "VPERM2I128 $1, Y0, Y1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst", true}, + {"VINSERTI128", Instr{Raw: "VINSERTI128 $1, X0, Y0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong dst", true}, {"VMOVNTDQ", Instr{Raw: "VMOVNTDQ Y0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: AX}}}, "bad dst", true}, {"AESKEYGENASSIST", Instr{Raw: "AESKEYGENASSIST X0, X1", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "missing imm", true}, - {"VPTEST", Instr{Raw: "VPTEST X0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", false}, - {"PCMPESTRI", Instr{Raw: "PCMPESTRI $1, 8(BX), X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("X0")}}}, "unsupported imm", true}, + {"VPTEST", Instr{Raw: "VPTEST X0, Y0", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", true}, + {"PCMPESTRI", Instr{Raw: "PCMPESTRI $-1, 8(BX), X0", Args: []Operand{{Kind: OpImm, Imm: -1}, {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, {"PCMPESTRI", Instr{Raw: "PCMPESTRI $0x0c, AX, X0", Args: []Operand{{Kind: OpImm, Imm: 0x0c}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "bad mem operand", true}, {"VPBLENDD", Instr{Raw: "VPBLENDD $1, Y0, Y1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst", false}, {"VPBROADCASTB", Instr{Raw: "VPBROADCASTB Y0, Y1", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("Y1")}}}, "wrong src", false}, - {"VPSRLDQ", Instr{Raw: "VPSRLDQ $2, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", false}, + {"VPSRLDQ", Instr{Raw: "VPSRLDQ $2, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 2}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong src class", true}, {"PUNPCKLBW", Instr{Raw: "PUNPCKLBW Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, - {"PSHUFHW", Instr{Raw: "PSHUFHW $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, + {"PSHUFHW", Instr{Raw: "PSHUFHW $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", true}, {"SHUFPS", Instr{Raw: "SHUFPS $1, X0, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst", false}, {"MOVOU", Instr{Raw: "MOVOU AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", true}, {"MOVOU", Instr{Raw: "MOVOU X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}, "wrong dst class", false}, @@ -1966,25 +2037,32 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { {"PADDL", Instr{Raw: "PADDL X0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}}, "wrong dst class", false}, {"PSLLL", Instr{Raw: "PSLLL AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "non-imm shift", true}, {"PCMPEQL", Instr{Raw: "PCMPEQL Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, - {"VPCLMULQDQ", Instr{Raw: "VPCLMULQDQ $1, Z0, Z1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst class", false}, - {"VPTERNLOGD", Instr{Raw: "VPTERNLOGD $0x95, Z0, Z1, Z2", Args: []Operand{{Kind: OpImm, Imm: 0x95}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Z2")}}}, "unsupported imm", true}, - {"VEXTRACTF32X4", Instr{Raw: "VEXTRACTF32X4 $1, X0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong src class", false}, + {"VPCLMULQDQ", Instr{Raw: "VPCLMULQDQ $1, Z0, Z1, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong dst class", true}, + {"VPTERNLOGD", Instr{Raw: "VPTERNLOGD $0x100, Z0, Z1, Z2", Args: []Operand{{Kind: OpImm, Imm: 0x100}, {Kind: OpReg, Reg: Reg("Z0")}, {Kind: OpReg, Reg: Reg("Z1")}, {Kind: OpReg, Reg: Reg("Z2")}}}, "out-of-range imm", true}, + {"VEXTRACTF32X4", Instr{Raw: "VEXTRACTF32X4 $1, X0, X1", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "wrong src class", true}, {"PCLMULQDQ", Instr{Raw: "PCLMULQDQ AX, X0, X1", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: Reg("X1")}}}, "missing imm", true}, {"PCMPEQB", Instr{Raw: "PCMPEQB Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, {"PMOVMSKB", Instr{Raw: "PMOVMSKB Y0, AX", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: AX}}}, "wrong src class", false}, {"PSHUFB", Instr{Raw: "PSHUFB Y0, X0", Args: []Operand{{Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong mask class", true}, {"PINSRQ", Instr{Raw: "PINSRQ AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "short form", true}, - {"PINSRD", Instr{Raw: "PINSRD $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", false}, - {"PINSRW", Instr{Raw: "PINSRW $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", false}, - {"PINSRB", Instr{Raw: "PINSRB $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", false}, + {"PINSRD", Instr{Raw: "PINSRD $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", true}, + {"PINSRW", Instr{Raw: "PINSRW $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", true}, + {"PINSRB", Instr{Raw: "PINSRB $1, AX, Y0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("Y0")}}}, "wrong dst class", true}, {"PEXTRB", Instr{Raw: "PEXTRB AX, X0, BX", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: BX}}}, "missing imm", true}, - {"PALIGNR", Instr{Raw: "PALIGNR $1, Y0, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", false}, - {"PSRLDQ", Instr{Raw: "PSRLDQ $17, X0", Args: []Operand{{Kind: OpImm, Imm: 17}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, - {"PSLLDQ", Instr{Raw: "PSLLDQ $17, X0", Args: []Operand{{Kind: OpImm, Imm: 17}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, + {"PALIGNR", Instr{Raw: "PALIGNR $1, Y0, X0", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("Y0")}, {Kind: OpReg, Reg: Reg("X0")}}}, "wrong src class", true}, + {"PSRLDQ", Instr{Raw: "PSRLDQ $128, X0", Args: []Operand{{Kind: OpImm, Imm: 128}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, + {"PSLLDQ", Instr{Raw: "PSLLDQ $128, X0", Args: []Operand{{Kind: OpImm, Imm: 128}, {Kind: OpReg, Reg: Reg("X0")}}}, "invalid imm", true}, {"PSRLQ", Instr{Raw: "PSRLQ AX, X0", Args: []Operand{{Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: Reg("X0")}}}, "non-imm shift", true}, {"PEXTRD", Instr{Raw: "PEXTRD $1, X0, label", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: Reg("X0")}, {Kind: OpIdent, Ident: "label"}}}, "bad dst kind", true}, } { + before := b.String() ok, term, err := c.lowerVec(tc.op, tc.ins) + if _, typed := x86InsertExtractSpecs[tc.op]; typed { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+tc.ins.Raw+"\nRET\n", "", false) + if !ok || term || err == nil || b.String() != before { + t.Fatalf("typed insert/extract rejection emitted IR: %s: (%v, %v, %v)", tc.ins.Raw, ok, term, err) + } + } if tc.wantHandled { if !ok || err == nil { t.Fatalf("%s %s = (%v, %v, %v), want handled error", tc.op, tc.want, ok, term, err) @@ -1998,7 +2076,7 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { out := b.String() for _, want := range []string{ - "load i64, ptr @\"example.global\"", + "load volatile i64, ptr @\"example.global\"", "load i8, ptr", "and i64", "store i64", @@ -2012,6 +2090,40 @@ func TestAMD64VectorAliasAndErrorCoverage(t *testing.T) { } } +func TestAMD64RejectedLegacyVectorDestinationDoesNotMutateIR(t *testing.T) { + for _, test := range []struct { + op Op + handled bool + }{ + {op: "PSUBUSB"}, + {op: "PABSD"}, + {op: "PTEST", handled: true}, + } { + t.Run(string(test.op), func(t *testing.T) { + ins := Instr{ + Op: test.op, Raw: string(test.op) + " X0, AX", + Args: []Operand{{Kind: OpReg, Reg: Reg("X0")}, {Kind: OpReg, Reg: AX}}, + } + c, b := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + beforeIR, beforeTmp := b.String(), c.tmp + ok, term, err := c.lowerVec(test.op, ins) + if test.handled { + if !ok || term || err == nil { + t.Fatalf("lowerVec(%s) = (%v, %v, %v), want handled rejection", test.op, ok, term, err) + } + } else if ok || term || err != nil { + t.Fatalf("lowerVec(%s) = (%v, %v, %v), want unhandled rejection", test.op, ok, term, err) + } + if got := b.String(); got != beforeIR { + t.Fatalf("lowerVec(%s) emitted IR before rejecting destination:\n%s", test.op, strings.TrimPrefix(got, beforeIR)) + } + if c.tmp != beforeTmp { + t.Fatalf("lowerVec(%s) advanced temporary counter from %d to %d", test.op, beforeTmp, c.tmp) + } + }) + } +} + func TestAMD64CtxAliasAndFPFallbackCoverage(t *testing.T) { sig := FuncSig{ Name: "example.ctxfallback", @@ -2055,7 +2167,7 @@ func TestAMD64CtxAliasAndFPFallbackCoverage(t *testing.T) { t.Fatalf("storeRegSized(%s, %s) error = %v", tc.r, tc.ty, err) } } - for _, r := range []Reg{AL, AH, BL, BH, CL, CH, DL, DH} { + for _, r := range []Reg{AL, AH, BL, BH, CL, CH, DL, DH, BPB, SIB, DIB, R8B, R9B, R10B, R11B, R12B, R13B, R14B, R15B} { if got, err := c.loadReg(r); err != nil || got == "" { t.Fatalf("loadReg(%s) = (%q, %v)", r, got, err) } @@ -2075,6 +2187,17 @@ func TestAMD64CtxAliasAndFPFallbackCoverage(t *testing.T) { {CH, "35"}, {DL, "20"}, {DH, "36"}, + {BPB, "22"}, + {SIB, "23"}, + {DIB, "24"}, + {R8B, "25"}, + {R9B, "26"}, + {R10B, "27"}, + {R11B, "28"}, + {R12B, "29"}, + {R13B, "30"}, + {R14B, "31"}, + {R15B, "32"}, } { if err := c.storeReg(tc.r, tc.v); err != nil { t.Fatalf("storeReg(%s) error = %v", tc.r, err) diff --git a/amd64_horizontal_float_arithmetic_type_test.go b/amd64_horizontal_float_arithmetic_type_test.go new file mode 100644 index 000000000..9cf4ca69a --- /dev/null +++ b/amd64_horizontal_float_arithmetic_type_test.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The arithmetic carrier must retain the same lane type as the horizontal +// shuffles. These are the complete register rows of Go's yxm/_yvaddsubpd +// tables, independently enumerated rather than derived from the emitter. +func TestX86HorizontalFloatArithmeticPreservesTypedLaneWidth(t *testing.T) { + for _, opcode := range []string{ + "HADDPS", "HADDPD", "HSUBPS", "HSUBPD", + "VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD", + } { + banks := []string{"X"} + vector := strings.HasPrefix(opcode, "V") + if vector { + banks = append(banks, "Y") + } + for _, bank := range banks { + t.Run(opcode+"/"+bank, func(t *testing.T) { + row := fmt.Sprintf("%s %s0,%s1", opcode, bank, bank) + if vector { + row += "," + bank + "2" + } + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + bits, element, width := 32, "float", 16 + if strings.HasSuffix(opcode, "PD") { + bits, element = 64, "double" + } + if bank == "Y" { + width = 32 + } + operation := "fadd" + if strings.Contains(opcode, "SUB") { + operation = "fsub" + } + lanes := width * 8 / bits + call := fmt.Sprintf("call <%d x %s> @llvm.experimental.constrained.%s.v%df%d(", lanes, element, operation, lanes, bits) + if strings.Count(ir, call) != 1 { + t.Fatalf("%s lost its complete typed arithmetic carrier %q", row, call) + } + }) + } + } +} diff --git a/amd64_horizontal_float_forms.go b/amd64_horizontal_float_forms.go new file mode 100644 index 000000000..152ba508f --- /dev/null +++ b/amd64_horizontal_float_forms.go @@ -0,0 +1,74 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86HorizontalFloatForm struct { + spec amd64HorizontalFloatingSpec + first, second, destination, memory int + byteWidth int +} + +// Each V-prefixed opcode has exactly the two _yvaddsubpd VEX rows, not +// EVEX rows. Sharing that ytab does not confer VADDSUB's alternating effect. +func x86HorizontalFloatProperties(ins Instr, goarch string) (x86HorizontalFloatForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := amd64HorizontalFloatingSpecs[Op(base)] + if !recognized { + return x86HorizontalFloatForm{}, false, nil + } + form := x86HorizontalFloatForm{spec: spec, first: 0, second: 1, destination: len(ins.Args) - 1, memory: -1, byteWidth: 16} + fail := func() (x86HorizontalFloatForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go horizontal floating grammar: %q", goarch, ins.Op, ins.Raw) + } + if dotted || goarch != "386" && goarch != "amd64" || !spec.vector && len(ins.Args) != 2 || spec.vector && len(ins.Args) != 3 { + return fail() + } + if spec.vector { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if form.byteWidth != 16 && form.byteWidth != 32 { + return fail() + } + } + register := func(operand Operand) bool { + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + if goarch == "386" && spec.vector { + for _, operand := range ins.Args { + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + if operand.Kind == OpReg && valid && index >= 8 && index < 16 { + // Go's named VEX ytab can emit these encodings in mode32, + // but that does not establish extra physical SIMD registers. + return form, true, fmt.Errorf("%w: 386 horizontal VEX register %s has no proved physical mode32 binding", ErrProbeNeedsContext, operand.Reg) + } + } + } + if !register(ins.Args[form.second]) || !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory = 0 + } else if !register(first) { + return fail() + } + return form, true, nil +} diff --git a/amd64_horizontal_float_forms_test.go b/amd64_horizontal_float_forms_test.go new file mode 100644 index 000000000..31b018d06 --- /dev/null +++ b/amd64_horizontal_float_forms_test.go @@ -0,0 +1,270 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86HorizontalFloatEveryGoOpcodeAttribute(t *testing.T) { + if len(amd64HorizontalFloatingSpecs) != 8 { + t.Fatal("incomplete horizontal floating family") + } + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return // older source builds still execute grammar, object and bit tests + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, op := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD", "VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD"} { + bits, prefix, opcode := 32, "vexF2", "0x7C" + if strings.HasSuffix(op, "PD") { + bits, prefix = 64, "vex66" + } + if strings.Contains(op, "SUB") { + opcode = "0x7D" + } + spec := amd64HorizontalFloatingSpecs[Op(op)] + if spec.laneBits != bits || spec.subtract != strings.Contains(op, "SUB") || spec.vector != strings.HasPrefix(op, "V") { + t.Fatalf("%s independent opcode attributes differ: %+v", op, spec) + } + if !spec.vector { + if !strings.Contains(string(legacy), "{A"+op+", yxm,") { + t.Fatalf("Go legacy yxm changed: %s", op) + } + continue + } + start := strings.Index(string(avx), "{as: A"+op+",") + if start < 0 { + t.Fatalf("Go omits %s", op) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvaddsubpd,") || strings.Contains(row, "evex") || strings.Contains(row, "Bcst") || strings.Contains(row, "Sae") { + t.Fatalf("Go VEX-only grammar changed: %s", row) + } + for _, width := range []string{"vex128", "vex256"} { + if !strings.Contains(row, "avxEscape | "+width+" | "+prefix+" | vex0F | vexW0, "+opcode) { + t.Fatalf("%s complete encoding attributes changed: %s", op, row) + } + } + } +} + +func TestX86HorizontalFloatDispatchIdentityBeforeEmission(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + op := file.Funcs[0].Instrs[1].Op + for _, identity := range []Op{"", op, Op(strings.ToLower(string(op))), "UNKNOWN", "VHADDPS.Z"} { + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + original := ins + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerHorizontalFloating(op, ins) + conflict := identity == "UNKNOWN" || identity == "VHADDPS.Z" + if !known || term || (err != nil) != conflict || conflict && output.String() != before || !reflect.DeepEqual(ins, original) { + t.Fatalf("%s/%s source identity: %t/%t/%v", op, identity, known, term, err) + } + } + } +} + +func TestX86HorizontalFloatMalformedFormsBeforeEmission(t *testing.T) { + for _, row := range []string{ + "HADDPS X0,Y1", "HSUBPD X0,X16", "HADDPD.SAE X0,X1", "HSUBPS $data(SB),X1", + "VHADDPS X0,X1", "VHADDPD X0,Y1,Y2", "VHSUBPS Z0,Z1,Z2", "VHSUBPD X0,X1,X16", + "VHADDPS X0,X1,K1,X2", "VHADDPD.Z X0,X1,X2", "VHSUBPS.BCST (AX),X1,X2", + "VHSUBPD.RN_SAE Y0,Y1,Y2", "VHADDPS. X0,X1,X2", "VHADDPD (AX)(SP*2),X1,X2", + "VHSUBPS 2147483648(AX),X1,X2", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerHorizontalFloating(ins.Op, ins) + if !known || term || err == nil || output.String() != before { + t.Fatalf("invalid form emitted IR: %t/%t/%v", known, term, err) + } + }) + } +} + +func TestX86HorizontalFloatGoFormatsFiveLLVMTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + sigs := map[string]FuncSig{} + for index, row := range x86HorizontalFloatFormats() { + name := fmt.Sprintf("horizontal%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-formats.ll", "horizontal-formats.o", ir) + } + } +} + +func TestX86HorizontalFloatLegacyMode32ClassesMatchGo(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + // NOSPLIT is a CPP macro, not an assembler constant without its header. + goAssemblerOracleOutput(t, arch, + "TEXT horizontal_floating_complete_forms(SB),NOSPLIT,$0-0\nHADDPS X0,X1\nRET\n", "", false) + } + // Keep the former complete-form fixture's rejected source, rather than + // treating mode32 X15 as a translator-supported ordinary register. + for _, opcode := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD"} { + for _, instruction := range []string{opcode + " X15,X1", opcode + " 8(BX),X15"} { + t.Run(strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + instruction + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}, + }); err == nil { + t.Fatal("translator accepted the original Go-rejected legacy X15 form") + } + }) + } + } +} + +func TestX86HorizontalFloat386VEXHighRegistersKeepPhysicalContext(t *testing.T) { + // Execute each compatibility Go assembler rather than trusting the current + // optab-only oracle. Actual objects are syntax evidence, not mode32 runtime. + const source = "TEXT f(SB),4,$0-0\nVHADDPS X8,X9,X10\nVHSUBPD Y8,Y9,Y10\nRET\n" + dir := t.TempDir() + asm, object := filepath.Join(dir, "high.s"), filepath.Join(dir, "high.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "oracle", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=386") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), true, "386"); failure != nil { + t.Fatalf("actual named high-register fixture: %v\n%s", failure, output) + } + if !bytes.Contains(output, []byte("c4 41 33 7c d0 c4 41 35 7d d0 c3")) { + t.Fatalf("actual Go mode32 source bytes changed:\n%s", output) + } + for _, row := range []struct{ code, op, bank string }{{"c441337cd0", "VHADDPS", "X"}, {"c441357dd0", "VHSUBPD", "Y"}} { + code, err := hex.DecodeString(row.code) + if err != nil { + t.Fatal(err) + } + ins, length, recognized, err := decodedX86VEXHorizontalFloatInstruction(code, 64) + if !recognized || err != nil || length != len(code) || string(ins.Op) != row.op || + ins.Args[0].Reg != Reg(row.bank+"8") || ins.Args[1].Reg != Reg(row.bank+"9") || ins.Args[2].Reg != Reg(row.bank+"10") { + t.Fatalf("source bytes lost physical register identities: %+v/%d/%t/%v", ins, length, recognized, err) + } + if _, _, known, err := decodedX86VEXHorizontalFloatInstruction(code, 32); !known || err == nil { + t.Fatal("mode32 decoder invented high physical SIMD registers") + } + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("named Go object laundered as physical mode32 success: %v", err) + } + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\nHADDPS X8,X9\nRET\n", "", false) +} + +func TestX86HorizontalFloatCompleteTypedFPSpanBeforeEmission(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + if !strings.Contains(row, "(BX)") { + continue + } + width := 16 + if strings.Contains(row, "Y") { + width = 32 + } + row = strings.Replace(row, "(BX)", "a+0(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-32\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, missing := range []int{-1, 0, width/8 - 1} { + sig := FuncSig{Name: "f", Ret: Void} + for index := 0; index < width/8; index++ { + if index == missing { + continue + } + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(index * 8), Type: I64, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I64) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + ins := file.Funcs[0].Instrs[1] + before := output.String() + known, term, err := ctx.lowerHorizontalFloating(ins.Op, ins) + if !known || term || (err != nil) != (missing >= 0) || missing >= 0 && output.String() != before { + t.Fatalf("%s missing typed slot %d: %t/%t/%v", row, missing, known, term, err) + } + } + } +} + +func TestX86HorizontalFloatPairsUseSingleStrictVectorOperation(t *testing.T) { + for _, row := range x86HorizontalFloatFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + // Pair projection may shuffle integers/floats, but must not add more + // arithmetic, reassociate, or split one strict vector into scalar pairs. + if got := strings.Count(ir, "metadata !\"fpexcept.strict\")"); got != 1 || !strings.Contains(ir, "= call <") { + t.Fatalf("%s split strict horizontal operation into %d scalar operations", row, got) + } + } +} diff --git a/amd64_horizontal_float_go_oracle_test.go b/amd64_horizontal_float_go_oracle_test.go new file mode 100644 index 000000000..397b8c5b3 --- /dev/null +++ b/amd64_horizontal_float_go_oracle_test.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86HorizontalFloatOriginalGoBitOracle(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import ("encoding/binary";"testing") +func TestOriginalHorizontalGolden(t *testing.T) { +`) + for index, opcode := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD"} { + name, bits, move := fmt.Sprintf("horizontal%d", index), 32, "MOVUPS" + if strings.HasSuffix(opcode, "PD") { + bits, move = 64, "MOVUPD" + } + fmt.Fprintf(&declarations, "func %s(out *[20]byte,a,b *[16]byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\n%s (BX),X0\n%s (CX),X1\n%s X0,X1\nMOVUPS X1,(AX)\nSTMXCSR 16(AX)\nLDMXCSR saved-4(SP)\nRET\n", name, move, move, opcode) + // Different qNaN/sNaN payloads in both halves establish horizontal + // pair priority independently of the production binary helper. + left, right, quiet := uint64(0x7f812345), uint64(0xffc45678), uint64(0x7fc12345) + if bits == 64 { + left, right, quiet = 0x7ff0123456789abc, 0xfff8fedcba987654, 0x7ff8123456789abc + } + fmt.Fprintf(&tests, `{ + var a,b [16]byte;var out [20]byte + for lane:=0;lane<16/%[2]d;lane++ { + value:=uint64(0x%[3]x);if lane%%2!=0 {value=0x%[4]x} + if %[5]d==32 {binary.LittleEndian.PutUint32(a[lane*4:],uint32(value));binary.LittleEndian.PutUint32(b[lane*4:],uint32(value))} else {binary.LittleEndian.PutUint64(a[lane*8:],value);binary.LittleEndian.PutUint64(b[lane*8:],value)} + } + %[1]s(&out,&a,&b,0x1f80) + for lane:=0;lane<16/%[2]d;lane++ { + var got uint64;if %[5]d==32 {got=uint64(binary.LittleEndian.Uint32(out[lane*4:]))}else{got=binary.LittleEndian.Uint64(out[lane*8:])} + if got!=0x%[6]x {t.Fatalf("%[1]s pair priority lane=%%d got=%%x",lane,got)} + } + if flags:=binary.LittleEndian.Uint32(out[16:]);flags!=0x1f81 {t.Fatalf("%[1]s sNaN flags=%%x",flags)} +} +`, name, bits/8, left, right, bits, quiet) + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go horizontal bit/flags oracle: %v\n%s", err, output) + } +} diff --git a/amd64_horizontal_float_numerical_test.go b/amd64_horizontal_float_numerical_test.go new file mode 100644 index 000000000..c66db7827 --- /dev/null +++ b/amd64_horizontal_float_numerical_test.go @@ -0,0 +1,133 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This list follows Go's four yxm and four _yvaddsubpd opcode rows. It is +// deliberately independent of the production parser and emitter. +func x86HorizontalFloatFormats() []string { + var rows []string + for _, op := range []string{"HADDPS", "HADDPD", "HSUBPS", "HSUBPD"} { + rows = append(rows, op+" X0,X1", op+" (BX),X1") + } + for _, op := range []string{"VHADDPS", "VHADDPD", "VHSUBPS", "VHSUBPD"} { + for _, bank := range []string{"X", "Y"} { + rows = append(rows, fmt.Sprintf("%s %s0,%s1,%s2", op, bank, bank, bank), fmt.Sprintf("%s (BX),%s1,%s2", op, bank, bank)) + } + } + return rows +} + +func x86HorizontalFloatSemanticsFormats() []string { + rows := x86HorizontalFloatFormats() + for _, row := range x86HorizontalFloatFormats() { + if !strings.HasPrefix(row, "V") || strings.Contains(row, "(BX)") { + continue + } + for _, destination := range []string{"0", "1"} { + rows = append(rows, strings.TrimSuffix(row, "2")+destination) + } + } + return rows +} + +// A real SSE3 horizontal instruction supplies the entire 128-bit result, +// not a scalar ADD/SUB reconstruction. VEX Y results concatenate two such +// independent blocks. MXCSR is observed before restoring the caller state. +const x86HorizontalFloatSSEOracle = `#include +#include +#include +typedef uint64_t vector128 __attribute__((vector_size(16))); +static unsigned last_horizontal_csr; +static void horizontal_result(void *out,const void *second,const void *first,unsigned bits,int sub,unsigned csr) { + unsigned saved;vector128 a,b;memcpy(&a,second,16);memcpy(&b,first,16); + __asm__ volatile("stmxcsr %0":"=m"(saved));__asm__ volatile("ldmxcsr %0"::"m"(csr)); + if(bits==32) { + if(sub)__asm__ volatile("hsubps %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("haddps %1,%0":"+x"(a):"x"(b)); + } else { + if(sub)__asm__ volatile("hsubpd %1,%0":"+x"(a):"x"(b)); + else __asm__ volatile("haddpd %1,%0":"+x"(a):"x"(b)); + } + memcpy(out,&a,16);__asm__ volatile("stmxcsr %0":"=m"(last_horizontal_csr)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); +} +` + +func TestCrossLinuxRuntimeMatrixX86HorizontalFloatCompletePhysicalSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for index, row := range x86HorizontalFloatSemanticsFormats() { + op := strings.Fields(row)[0] + bits, width, dest, legacy := 32, 16, 2, !strings.HasPrefix(op, "V") + if strings.HasSuffix(op, "PD") { + bits = 64 + } + if strings.Contains(row, "Y") { + width = 32 + } + if legacy { + dest = 1 + } else { + dest = int(row[len(row)-1] - '0') + } + name := fmt.Sprintf("horizontal%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", name, row, dest) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*);\n", name) + fmt.Fprintf(&checks, ` + for(unsigned vector=0;vector<20;vector++)for(unsigned mode=0;mode<4;mode++)for(unsigned control=0;control<6;control++) { + _Alignas(16) uint8_t aStorage[96],bStorage[96];uint8_t out[67],expected[64],beforeA[64],beforeB[64]; + unsigned offset=%[7]d?17:16;uint8_t *a=aStorage+offset,*b=bStorage+offset;memset(aStorage,0xa7,96);memset(bStorage,0xb9,96);memset(out,0xc3,67); + for(unsigned lane=0;lane<64/%[2]d;lane++) { + uint64_t av=edges%[3]d[(vector+lane)%%20],bv=edges%[3]d[(vector*3+lane*7)%%20]; + memcpy(a+lane*%[2]d,&av,%[2]d);memcpy(b+lane*%[2]d,&bv,%[2]d); + } + memcpy(beforeA,a,64);memcpy(beforeB,b,64); + unsigned csr=controls[control]|(mode<<13),observed,flags=0; + __asm__ volatile("ldmxcsr %%0"::"m"(csr));%[1]s(out+1,a,b); + __asm__ volatile("stmxcsr %%0":"=m"(observed)); + if(%[4]d)memcpy(expected,beforeB,64);else memset(expected,0,64); + for(unsigned block=0;block<%[5]d;block+=16) { + horizontal_result(expected+block,beforeB+block,beforeA+block,%[3]d,%[6]d,csr);flags|=last_horizontal_csr&63; + } + for(unsigned byte=0;byte<64;byte++)if(out[1+byte]!=expected[byte]) { + fprintf(stderr,"%[1]s vector/mode/csr=%%u/%%u/%%x byte=%%u got=%%x want=%%x\n",vector,mode,csr,byte,out[1+byte],expected[byte]);return 1; + } + if(observed!=(csr|flags)){fprintf(stderr,"%[1]s MXCSR=%%x want=%%x\n",observed,csr|flags);return 2;} + if(out[0]!=0xc3 || out[65]!=1 || out[66]!=0xc3)return 3; + if(memcmp(a,beforeA,64) || memcmp(b,beforeB,64))return 5; + for(unsigned byte=0;byte 1 || rule.Ytab != "_yvaddsubpd" || rule.Prefix != "Pavx" || rule.ZCase != "Zvex_rm_v_r" || rule.ZOffset != 2 || rule.EncodingOffset != 2*index || !reflect.DeepEqual(rule.Classes, classes[index]) || !reflect.DeepEqual(rule.OpBytes, bytes) { + t.Fatalf("unmodeled complete horizontal VEX row: %+v", rule) + } + bank := []string{"X", "Y"}[index] + rows = append(rows, rule.Opcode+" "+bank+"0,"+bank+"1,"+bank+"7", rule.Opcode+" -7(BX)(CX*4),"+bank+"1,"+bank+"7") + counts[rule.Opcode]++ + } + for _, op := range x86HorizontalIntegerOpcodes { + if counts["V"+op] != 2 { + t.Fatalf("missing complete original encoding rows for %s: %v", op, counts) + } + bits := 16 + if op == "PHADDD" || op == "PHSUBD" { + bits = 32 + } + for _, name := range []string{op, "V" + op} { + spec := amd64HorizontalIntegerSpecs[Op(name)] + if spec.laneBits != bits || spec.subtract != strings.Contains(name, "SUB") || spec.saturate != strings.HasSuffix(name, "SW") || spec.vector != strings.HasPrefix(name, "V") || spec.allowMMX != (name == "PHADDD") { + t.Fatalf("independent horizontal arithmetic attributes differ: %s %+v", name, spec) + } + } + } + return rows +} + +func TestX86HorizontalIntegerCompleteEncoderRowsThreeAPIsFiveTargets(t *testing.T) { + rows := x86OfficialHorizontalIntegerRows(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT horizontal(SB),4,$0-0\n" + strings.Join(rows, "\n") + "\nRET\n" + if arch == "386" { + source = strings.Replace(source, "\nRET\n", "\nVPHSUBD 2147483648(AX),X1,X2\nRET\n", 1) + } + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"horizontal": {Name: "horizontal", Ret: Void}}} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-text.ll", "horizontal-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-module.ll", "horizontal-module.o", module.String()) + translated, err := translateGoModuleInContext(ctx, x86SADGoPackage(t, arch, "()"), []byte(strings.Replace(source, "horizontal(SB)", "·sad(SB)", 1)), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "horizontal-go.ll", "horizontal-go.o", translated.Module.String()) + }) + } +} + +func TestX86HorizontalIntegerGrammarRejectsBeforeEveryConsumer(t *testing.T) { + for _, row := range []string{ + "PHADDW M0,M1", "PHADDSW X0,M1", "PHSUBD Y0,Y1", "PHADDD.Z X0,X1", "PHADDW X0,X16", + "VPHADDD X0,X1", "VPHADDW X0,Y1,Y2", "VPHSUBD Z0,Z1,Z2", "VPHSUBSW X0,X1,K1,X2", "VPHSUBW.Z X0,X1,X2", + "VPHADDD.BCST (AX),X1,X2", "VPHADDW.SAE X0,X1,X2", "VPHSUBW.RN_SAE Y0,Y1,Y2", "VPHADDD (AX)(SP*2),X1,X2", "VPHSUBD 4294967296(AX),X1,X2", "PHADDW $table(SB),X1", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT sad(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerHorizontalInteger(ins.Op, ins) + if !known || term || err == nil || output.String() != before || x86CallerVectorMoveKnown(ins) { + t.Fatal("invalid horizontal grammar emitted or registered", known, term, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": ^uint64(0)}} + state = x86CloneCallerState(state) + original := x86CloneCallerState(state) + if x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, original) { + t.Fatal("invalid grammar mutated caller state") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + goAssemblerOracleOutput(t, arch, source, "", false) + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"sad": {Ret: Void}}} + if ir, err := Translate(file, options); err == nil || ir != "" { + t.Fatal("invalid text API form accepted") + } + llvmCtx := llvm.NewContext() + defer llvmCtx.Dispose() + module, err := TranslateModuleInContext(llvmCtx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil { + t.Fatal("invalid module API form accepted") + } + translated, err := translateGoModuleInContext(llvmCtx, x86SADGoPackage(t, arch, "()"), []byte(strings.Replace(source, "sad(SB)", "·sad(SB)", 1)), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err == nil { + t.Fatal("invalid Go API form accepted") + } + }) + } + }) + } +} + +func TestX86HorizontalIntegerDispatchAndCompleteFPSpanBeforeEmission(t *testing.T) { + for _, row := range x86OfficialHorizontalIntegerRows(t) { + ins := x86YMMReadSource(t, row) + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "UNKNOWN", "VPHADDD.Z"} { + local := ins + local.Op = identity + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := output.String() + _, _, err := ctx.lowerHorizontalInteger(ins.Op, local) + if identity == "UNKNOWN" || identity == "VPHADDD.Z" { + if err == nil || output.String() != before { + t.Fatal("dispatch conflict emitted", row, identity) + } + } else if err != nil { + t.Fatal(row, identity, err) + } + } + form, _, err := x86HorizontalIntegerProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", Func{Instrs: []Instr{ins}}) + wantFeatures := "" + if form.memory >= 0 && !form.spec.vector && !form.mmx { + wantFeatures = "+sse2,+ssse3" + } + if features != wantFeatures { + t.Fatalf("%s native carrier features %s != %s", row, features, wantFeatures) + } + if !strings.Contains(row, "(AX)") && !strings.Contains(row, "(BX)") { + continue + } + row = regexp.MustCompile(`(?:-7\(BX\)\(CX\*4\)|\(AX\))`).ReplaceAllString(row, "a+0(FP)") + ins = x86YMMReadSource(t, row) + form, _, err = x86HorizontalIntegerProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for hole := -1; hole < form.byteWidth; hole++ { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-32\n"+row+"\nRET\n") + sig := FuncSig{Ret: Void} + for at := 0; at < form.byteWidth; at++ { + if at == hole { + continue + } + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(at), Type: I8, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I8) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + _, _, err := ctx.lowerHorizontalInteger(ins.Op, ins) + wantContext := !form.spec.vector && !form.mmx + if (err != nil) != (hole >= 0 || wantContext) || err != nil && output.String() != before || wantContext && hole < 0 && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s byte hole %d: %v", row, hole, err) + } + } + } +} + +func TestX86HorizontalIntegerEveryInputByteAndAliasEffect(t *testing.T) { + for _, op := range x86HorizontalIntegerOpcodes { + for _, bank := range []string{"legacy", "X", "Y"} { + for destination := 0; destination < 3; destination++ { + row := op + " X0,X1" + if bank != "legacy" { + row = fmt.Sprintf("V%s %s0,%s1,%s%d", op, bank, bank, bank, destination) + } + ins := x86YMMReadSource(t, row) + form, known, err := x86HorizontalIntegerProperties(ins, "amd64") + if !known || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatal("missing whole-family registration", row, err) + } + for operand := 0; operand < 2; operand++ { + for hole := -1; hole < form.byteWidth; hole++ { + state := x86SourceStackState{bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + known: map[Reg]uint8{AX: 0xff}, masks: map[Reg]bool{"K1": true}, + vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0), "Z2": 0x55, "Z31": 0x1234}} + if hole >= 0 { + state.vectors[x86CallerVectorRoot(ins.Args[operand].Reg)] &^= uint64(1) << uint(hole) + } + state = x86CloneCallerState(state) + before := x86CloneCallerState(state) + accepted := x86CallerVectorMoveStep(&ins, &state, nil) + if accepted != (hole < 0) || !accepted && !reflect.DeepEqual(state, before) { + t.Fatalf("%s input %d byte hole %d invented input/changed state", row, operand, hole) + } + if accepted { + want := x86CloneCallerState(before) + root := x86CallerVectorRoot(ins.Args[form.destination].Reg) + want.vectors[root] = ^uint64(0) + if !form.spec.vector { + want.vectors[root] = before.vectors[root] | x86CallerVectorByteMask(int64(form.byteWidth)) + } + if !reflect.DeepEqual(state, want) { + t.Fatal("horizontal effect altered unrelated state", row) + } + } + } + } + for _, upper := range []uint64{0, 0xff00000000000000, ^x86CallerVectorByteMask(int64(form.byteWidth))} { + mask := x86CallerVectorByteMask(int64(form.byteWidth)) + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": mask | upper, "Z1": mask | upper, "Z2": upper}} + if !x86CallerVectorMoveStep(&ins, &state, nil) { + t.Fatal("initialized complete input failed", row) + } + want := ^uint64(0) + if !form.spec.vector { + want = mask | upper + } + if state.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] != want { + t.Fatal("legacy preservation/VEX upper clearing diverged", row) + } + } + } + } + } +} diff --git a/amd64_horizontal_integer_forms.go b/amd64_horizontal_integer_forms.go new file mode 100644 index 000000000..cbc96184b --- /dev/null +++ b/amd64_horizontal_integer_forms.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64HorizontalIntegerSpec struct { + laneBits int + subtract bool + saturate bool + vector bool + allowMMX bool + // VEX.66.0F38 opcode, shared by raw decoding and typed lowering. + opcode byte +} + +// The complete PHADD/PHSUB family has one exceptional legacy MMX row: +// PHADDD's ymmxmm0f38. Its siblings use yxm_q4; VEX uses X/Y only. +var amd64HorizontalIntegerSpecs = map[Op]amd64HorizontalIntegerSpec{ + "PHADDD": {laneBits: 32, allowMMX: true}, + "PHADDSW": {laneBits: 16, saturate: true}, + "PHADDW": {laneBits: 16}, + "PHSUBD": {laneBits: 32, subtract: true}, + "PHSUBSW": {laneBits: 16, subtract: true, saturate: true}, + "PHSUBW": {laneBits: 16, subtract: true}, + "VPHADDD": {laneBits: 32, vector: true, opcode: 0x02}, + "VPHADDSW": {laneBits: 16, saturate: true, vector: true, opcode: 0x03}, + "VPHADDW": {laneBits: 16, vector: true, opcode: 0x01}, + "VPHSUBD": {laneBits: 32, subtract: true, vector: true, opcode: 0x06}, + "VPHSUBSW": {laneBits: 16, subtract: true, saturate: true, vector: true, opcode: 0x07}, + "VPHSUBW": {laneBits: 16, subtract: true, vector: true, opcode: 0x05}, +} + +type x86HorizontalIntegerForm struct { + spec amd64HorizontalIntegerSpec + first, second, destination, memory int + byteWidth int + mmx bool +} + +// Forms describe ordered bytes and encoding only; they confer no caller, +// stack, alignment, MMX state or address permission. +func x86HorizontalIntegerProperties(ins Instr, arch string) (x86HorizontalIntegerForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := amd64HorizontalIntegerSpecs[Op(base)] + if !known { + return x86HorizontalIntegerForm{}, false, nil + } + form := x86HorizontalIntegerForm{spec: spec, first: 0, second: 1, destination: len(ins.Args) - 1, memory: -1, byteWidth: 16} + fail := func() (x86HorizontalIntegerForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go horizontal integer grammar: %q", arch, ins.Op, ins.Raw) + } + if dotted || arch != "amd64" && arch != "386" || !spec.vector && len(ins.Args) != 2 || spec.vector && len(ins.Args) != 3 { + return fail() + } + if spec.allowMMX { + legacy, _, err := x86LegacyMMXReadProperties(ins, arch) + if err != nil { + return form, true, err + } + if err := x86LegacyMMXReadEmitterContext(ins, arch); err != nil { + return form, true, err + } + form.mmx, form.byteWidth = legacy.mmx, legacy.byteWidth + if legacy.memoryBytes != 0 { + form.memory = 0 + } + return form, true, nil + } + if spec.vector { + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + if form.byteWidth != 16 && form.byteWidth != 32 { + return fail() + } + } + register := func(operand Operand) bool { + index, valid := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + return operand.Kind == OpReg && valid && index < 16 && (spec.vector || arch == "amd64" || index < 8) + } + if !register(ins.Args[form.second]) || !register(ins.Args[form.destination]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) && x86PackedShiftMemoryValid(first, arch) { + form.memory = form.first + } else if !register(first) { + return fail() + } + if spec.vector { + if err := x86Mode32VectorRegisterContext(ins, arch); err != nil { + return form, true, err + } + } + return form, true, nil +} + +func x86CallerHorizontalIntegerStep(ins *Instr, form x86HorizontalIntegerForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if form.mmx { + return false + } + required := x86CallerVectorByteMask(int64(form.byteWidth)) + // Require all potential input bytes before certifying memory or writing + // aliases. Horizontal subtract with equal operands is not a zero idiom. + for _, index := range []int{form.first, form.second} { + if index != form.memory && state.vectors[x86CallerVectorRoot(ins.Args[index].Reg)]&required != required { + return false + } + } + if form.memory >= 0 { + // No current caller producer supplies the actual legacy m128 alignment. + if !form.spec.vector || !x86CallerOrdinaryMemoryStep(ins, form.memory, state, proof, int64(form.byteWidth), false) { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[form.destination].Reg) + known := ^uint64(0) + if !form.spec.vector { + known = state.vectors[destination] | required + } + state.vectors[destination] = known + return true +} diff --git a/amd64_horizontal_integer_full_alias_runtime_test.go b/amd64_horizontal_integer_full_alias_runtime_test.go new file mode 100644 index 000000000..59874989d --- /dev/null +++ b/amd64_horizontal_integer_full_alias_runtime_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// All three physical vector-number views are observed after the instruction. +// The scalar reference does not read the production grammar or lane emitter. +// This executes portable LLVM, not original AVX/AVX512 on the host. +func TestCrossLinuxRuntimeMatrixX86HorizontalIntegerFullAliasSentinels(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, prototypes, checks strings.Builder + declarations.WriteString("package narroworacle\n") + count := 0 + for _, entry := range []struct { + legacy, vex string + lane, subtract, saturate int + }{ + {"PHADDW", "VPHADDW", 2, 0, 0}, + {"PHADDD", "VPHADDD", 4, 0, 0}, + {"PHADDSW", "VPHADDSW", 2, 0, 1}, + {"PHSUBW", "VPHSUBW", 2, 1, 0}, + {"PHSUBD", "VPHSUBD", 4, 1, 0}, + {"PHSUBSW", "VPHSUBSW", 2, 1, 1}, + } { + for _, legacy := range []bool{true, false} { + widths := []int{16} + if !legacy { + widths = append(widths, 32) + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y"}[width] + for source := 0; source < 3; source++ { + // Distinct registers, both inputs the destructive source, + // and full-width memory each have independent witnesses. + first := bank + "0" + if source == 1 { + first = bank + "1" + } else if source == 2 { + first = "(CX)" + } + for destination := 0; destination < 3; destination++ { + if legacy && destination != 1 { + continue + } + op := entry.vex + args := first + "," + bank + "1" + if legacy { + op = entry.legacy + } else { + args += fmt.Sprintf(",%s%d", bank, destination) + } + name := fmt.Sprintf("H%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[256]byte,a,b,old *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-44\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nLDMXCSR csr+40(FP)\nPUSHQ flags+32(FP)\nPOPFQ\n%s %s\nVMOVUPS Z0,(AX)\nVMOVUPS Z1,64(AX)\nVMOVUPS Z2,128(AX)\nVMOVUPS Y%d,192(AX)\nVMOVUPS X%d,224(AX)\nPUSHFQ\nPOPQ R9\nMOVQ R9,240(AX)\nSTMXCSR 248(AX)\nRET\n", name, op, args, destination, destination) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,%d,%d,%d,%d,%d))return 1;\n", name, op+" "+args, width, entry.lane, entry.subtract, entry.saturate, x86FloatAddSubBoolInt(legacy), source, destination) + } + } + } + } + } + if count != 126 { + t.Fatal("independent legacy/VEX source and alias universe changed", count) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + reference := x86HorizontalIntegerFullAliasC + prototypes.String() + "int main(void){\n" + checks.String() + fmt.Sprintf("printf(\"horizontal full-alias kernels %d cases %d failures 0\\n\");return 0;}\n", count, count*97*8) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_default_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run("backend"+backend, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "horizontal-full-alias", text, reference, backend) + want := fmt.Sprintf("horizontal full-alias kernels %d cases %d failures 0\n", count, count*97*8) + if string(output) != want { + t.Fatal("full-byte alias oracle did not complete", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } + t.Log("126 independently Go-accepted legacy/VEX kernels; 97 boundary/random samples and 8 six-flag seeds per axis. All64 bytes of Z0/Z1/Z2 plus destination X/Y views, untouched sources, input/output guards and complete MXCSR; portable LLVM/Rosetta is not original AVX execution.") +} + +const x86HorizontalIntegerFullAliasC = ` +#include +#include +#include +#include +typedef void(*Program)(uint8_t*,const uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t); +struct Input {_Alignas(64) uint8_t before[64],value[64],after[64];}; +struct Output {_Alignas(64) uint8_t before[64],value[256],after[64];}; +static const uint32_t edges16[]={0,1,2,0x7ffe,0x7fff,0x8000,0x8001,0xfffe,0xffff}; +static const uint32_t edges32[]={0,1,2,0x7ffffffe,0x7fffffff,0x80000000,0x80000001,0xfffffffe,0xffffffff}; +static const uint64_t flagSeeds[]={0,0x8d5,1,4,0x10,0x40,0x80,0x800}; +static int64_t readLane(const uint8_t *p,int bytes) { + uint64_t value=0;for(int i=0;i=81) {x=UINT32_C(0x9e3779b9)*(unsigned)(sample*17+lane*31);y=UINT32_C(0x85ebca6b)*(unsigned)(sample*43+lane*7);} + for(int i=0;i>(i*8));b[lane*bytes+i]=(uint8_t)(y>>(i*8));} + } +} +static int check(Program program,const char *name,int width,int lane,int sub,int sat,int legacy,int source,int destination) { + for(int sample=0;sample<97;sample++) for(unsigned state=0;state32767)value=32767;} + for(int i=0;i>(8*i)); + } + if(!legacy)memset(result+width,0,64-width); + uint64_t seed=flagSeeds[state],flags=0;uint32_t csr=0; + uint32_t profile=(uint32_t)(0x1f80|((sample%4)<<13)|(sample%64)),savedCSR=_mm_getcsr(); + _mm_setcsr(profile);program(out.value,a.value,b.value,old.value,seed,profile); + uint32_t afterCSR=_mm_getcsr();_mm_setcsr(savedCSR); + memcpy(&flags,out.value+240,8);memcpy(&csr,out.value+248,4); + if((flags&0x8d5)!=seed||csr!=profile||afterCSR!=profile||memcmp(&a,&savedA,sizeof(a))||memcmp(&b,&savedB,sizeof(b))||memcmp(&old,&savedOld,sizeof(old))) { + fprintf(stderr,"%s sample%d state%u flags/CSR/input guard mismatch\n",name,sample,state);return 1; + } + for(int i=0;i<192;i++)if(out.value[i]!=want[i]) { + fprintf(stderr,"%s sample%d state%u physical Z%d byte%d got%x want%x\n",name,sample,state,i/64,i%64,out.value[i],want[i]);return 2; + } + for(int i=0;i<32;i++)if(out.value[192+i]!=result[i]) { + fprintf(stderr,"%s sample%d state%u Y%d byte%d got%x want%x\n",name,sample,state,destination,i,out.value[192+i],result[i]);return 3; + } + for(int i=0;i<16;i++)if(out.value[224+i]!=result[i]) { + fprintf(stderr,"%s sample%d state%u X%d byte%d got%x want%x\n",name,sample,state,destination,i,out.value[224+i],result[i]);return 4; + } + for(int i=0;i<64;i++)if(out.before[i]!=0xc3||out.after[i]!=0xc3)return 5; + for(int i=252;i<256;i++)if(out.value[i]!=0xc3)return 6; + } + return 0; +} +` diff --git a/amd64_horizontal_integer_memory_runtime_test.go b/amd64_horizontal_integer_memory_runtime_test.go new file mode 100644 index 000000000..a9010ddef --- /dev/null +++ b/amd64_horizontal_integer_memory_runtime_test.go @@ -0,0 +1,205 @@ +//go:build unix + +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86HorizontalIntegerOriginalNativeMemory(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, originals, checks strings.Builder + sigs := map[string]FuncSig{} + // This independent list fixes the original SSSE3 bytes and arithmetic; + // neither native oracle nor expected values consume the production spec. + for index, entry := range []struct { + op string + opcode byte + lane, subtract, clamp int + }{ + {"PHADDW", 1, 2, 0, 0}, {"PHADDD", 2, 4, 0, 0}, {"PHADDSW", 3, 2, 0, 1}, + {"PHSUBW", 5, 2, 1, 0}, {"PHSUBD", 6, 4, 1, 0}, {"PHSUBSW", 7, 2, 1, 1}, + } { + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+entry.op+" (SI),X1\nRET\n") + if !bytes.Equal(code, []byte{0x66, 0x0f, 0x38, entry.opcode, 0x0e, 0xc3}) { + t.Fatalf("original horizontal m128 bytes changed: %s %x", entry.op, code) + } + for _, dead := range []bool{false, true} { + name := fmt.Sprintf("horizontalmemory%d_%t", index, dead) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ out+0(FP),AX\nMOVQ input+8(FP),BX\nMOVOU (AX),X1\nCMPQ BX,BX\nSTC\n%s (BX),X1\n", name, entry.op) + tail := "" + if !dead { + source.WriteString("MOVOU X1,16(AX)\n") + tail = "; movdqu %%xmm1,16(%%rdi)" + } + source.WriteString("SETCS 32(AX)\nSETNE 33(AX)\nSTMXCSR 36(AX)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*);\n", name) + fmt.Fprintf(&originals, `static void original_%s(uint8_t *out,const uint8_t *input) { + __asm__ volatile("movdqu (%%%%rdi),%%%%xmm1; cmpq %%%%rsi,%%%%rsi; stc; .byte 0x66,0x0f,0x38,0x%02x,0x0e%s; setc 32(%%%%rdi); setnz 33(%%%%rdi); stmxcsr 36(%%%%rdi)" :: + "D"(out),"S"(input) : "xmm1","cc","memory"); +} +`, name, entry.opcode, tail) + fmt.Fprintf(&checks, ` + if(compare(original_%[1]s,%[1]s,region,0,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,boundary-16,0,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,region+1,alignmentFault,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,boundary-15,1,%[2]d,%[3]d,%[4]d,%[5]d) || + compare(original_%[1]s,%[1]s,0,1,%[2]d,%[3]d,%[4]d,%[5]d)) return 1; +`, name, entry.lane, entry.subtract, entry.clamp, x86FloatAddSubBoolInt(dead)) + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + originals.String() + ` +typedef void (*memoryfn)(uint8_t*,const uint8_t*); +static int64_t laneValue(const uint8_t *p,int lane) { + if(lane==2) {int16_t v;memcpy(&v,p,2);return v;} + int32_t v;memcpy(&v,p,4);return v; +} +static int observe(memoryfn fn,const uint8_t *input,int lane,int sub,int clamp,int dead) { + pid_t child=fork(); + if(child<0) return -1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + uint8_t out[48],expected[48];unsigned csr,after; + for(unsigned i=0;i32767)value=32767;} + uint64_t bits=(uint64_t)value; + for(int byte=0;byte>(8*byte)); + } + } + expected[33]=1;expected[34]=0;memcpy(expected+37,&csr,4); + __asm__ volatile("stmxcsr %0":"=m"(after)); + if(memcmp(out,expected,sizeof(out)) || after!=csr) _exit(3); + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status)) return WEXITSTATUS(status); + if(WIFSIGNALED(status)) return 128+WTERMSIG(status); + return -1; +} +static int compare(memoryfn reference,memoryfn translated,const uint8_t *input,int fault,int lane,int sub,int clamp,int dead) { + int native=observe(reference,input,lane,sub,clamp,dead),got=observe(translated,input,lane,sub,clamp,dead); + int memoryFault=native==128+SIGSEGV || native==128+SIGBUS; + int valid=fault==1 ? memoryFault : fault==-1 ? native==0 || memoryFault : native==0; + if(native<0 || native!=got || !valid) { + fprintf(stderr,"horizontal native=%d LLVM=%d input=%p fault=%d lane=%d sub=%d clamp=%d dead=%d\n",native,got,input,fault,lane,sub,clamp,dead); + return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 1; + uint8_t *region=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,(size_t)page,PROT_NONE)) return 1; + for(long i=0;i +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +static int check(void (*program)(const uint8_t*),const uint8_t *input,int fault) { + pid_t child=fork();if(child<0)return 1; + if(child==0) {struct rlimit limit={0,0};if(setrlimit(RLIMIT_CORE,&limit))_exit(2);program(input);_exit(0);} + int status;if(waitpid(child,&status,0)!=child)return 1; + int got=WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS); + if(fault ? !got : !WIFEXITED(status) || WEXITSTATUS(status)!=0) { + fprintf(stderr,"horizontal dead read input=%p fault=%d status=%x\n",input,fault,status);return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE);if(page<=0)return 1; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED || mprotect(memory+page,(size_t)page,PROT_NONE))return 1; + uint8_t *boundary=memory+page; +` + checks.String() + `return munmap(memory,(size_t)page*2)!=0;} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "horizontal-vex-dead-full-read", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_immediate_shuffle_class_boundaries_test.go b/amd64_immediate_shuffle_class_boundaries_test.go new file mode 100644 index 000000000..6c1ea3b42 --- /dev/null +++ b/amd64_immediate_shuffle_class_boundaries_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86ImmediateShuffleRejectsNonIntegerClassesEveryConsumer(t *testing.T) { + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, literal := range []string{"$0.0", "$1.0", "$UNKNOWN"} { + bank := "X" + if op == "PSHUFW" { + bank = "M" + } + row := fmt.Sprintf("%s %s,%s0,%s1", op, literal, bank, bank) + t.Run(row, func(t *testing.T) { + assembly := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + for _, arch := range []string{"amd64", "386"} { + goAssemblerOracleOutput(t, arch, assembly, "", false) + ins := x86YMMReadSource(t, row) + if _, known, err := x86ImmediateShuffleProperties(ins, arch); !known || err == nil { + t.Error("non-integer class acquired integer grammar", arch, literal, err) + } + var output strings.Builder + ctx := &amd64Ctx{goarch: arch, b: &output} + if known, _, err := ctx.lowerPackedWordShuffle(ins.Op, ins); !known || err == nil || output.Len() != 0 { + t.Error("non-integer class emitted IR", arch, err, output.String()) + } + state := x86CloneCallerState(x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0)}}) + before := x86CloneCallerState(state) + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, before) { + t.Error("non-integer class acquired caller effects or changed state", arch) + } + result := x86ResultState{} + result.writeVector(0, 64, ^uint64(0), false) + result.writeVector(1, 64, ^uint64(0), false) + beforeResult := result.clone() + if x86ResultTransfer(ins, &result, nil, 0, arch) || !reflect.DeepEqual(result, beforeResult) { + t.Error("non-integer class acquired result effects or changed state", arch) + } + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}} + if _, err := Translate(file, options); err == nil { + t.Error("text API accepted non-integer immediate") + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err == nil { + t.Error("module API accepted non-integer immediate") + } + pkg := mustGoPackage(t, "example.invalid/shuffle", "package shuffle\nfunc f()\n") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: options.TargetTriple, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err == nil { + t.Error("actual typed declaration API accepted non-integer immediate") + } + }) + } + } +} + +func TestX86ImmediateShuffleGoMode32ConstantClassification(t *testing.T) { + // Go's oclass truncates TYPE_CONST to int32 only in I386 mode. This is + // source classification, not permission to invent an encoded raw operand. + values := []int64{4294967296, -4294967296, 4294967168, 4294967423, 4294967424, 4294967551, 4294967552, -4294967297} + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + banks := []string{"X"} + if op == "PSHUFW" { + banks = []string{"M"} + } else if strings.HasPrefix(op, "V") { + banks = append(banks, "Z") + } + for _, bank := range banks { + for _, value := range values { + classified := value + if arch == "386" { + classified = int64(int32(value)) + } + accepted := classified >= 0 && classified <= 255 + if op == "PSHUFW" { + accepted = classified >= -128 && classified <= 127 + } else if strings.HasPrefix(op, "V") && bank == "X" { + accepted = classified >= -128 && classified <= 255 + } + row := fmt.Sprintf("%s $%d,%s0,%s1", op, value, bank, bank) + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", accepted) + ins := x86YMMReadSource(t, row) + before := fmt.Sprintf("%#v", ins) + form, known, err := x86ImmediateShuffleProperties(ins, arch) + if !known || accepted != (err == nil) || accepted && form.immediate != uint8(classified) { + t.Errorf("Go %s accepted=%v for %s, grammar=%+v error=%v", arch, accepted, row, form, err) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("classification changed original constant/source") + } + } + } + } + }) + } +} + +func TestX86ImmediateShuffleRawMMXCompleteImmediateBits(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + var expected []byte + for value := 0; value < 256; value++ { + code := []byte{0x0f, 0x70, 0xc8, byte(value)} + fmt.Fprintf(&source, "PSHUFW $%d,M0,M1\n", int8(value)) + expected = append(expected, code...) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil || len(decoded.Instrs) != 1 { + t.Fatalf("physical imm8 %x lost instruction identity: %+v %v", code, decoded, err) + } + form, known, err := x86ImmediateShuffleProperties(decoded.Instrs[0], arch) + if !known || err != nil || form.immediate != uint8(value) || !form.spec.mmx { + t.Errorf("raw PSHUFW imm8=%d lost Go Yi8 spelling: %+v %v", value, form, err) + } + } + source.WriteString("RET\n") + expected = append(expected, 0xc3) + if got := assembleX87ControlBytes(t, arch, source.String()); !bytes.HasPrefix(got, expected) { + t.Fatal("independent Go signed source did not encode the original physical imm8 bytes") + } + }) + } +} diff --git a/amd64_immediate_shuffle_effects_test.go b/amd64_immediate_shuffle_effects_test.go new file mode 100644 index 000000000..ed6850195 --- /dev/null +++ b/amd64_immediate_shuffle_effects_test.go @@ -0,0 +1,239 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86ImmediateShuffleCompleteLegacyOriginalRows(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + pattern := regexp.MustCompile(`\{A(PSHUFW|PSHUFL|PSHUFD|PSHUFHW|PSHUFLW), (\w+), (\w+), opBytes\{([^}]+)\}\}`) + rows := pattern.FindAllStringSubmatch(string(source), -1) + if len(rows) != 5 { + t.Fatalf("complete legacy original shuffle denominator changed: %d", len(rows)) + } + for _, row := range rows { + prefix := map[string]string{"PSHUFW": "Pm", "PSHUFL": "Pq", "PSHUFD": "Pq", "PSHUFHW": "Pf3", "PSHUFLW": "Pf2"}[row[1]] + table := "yxshuf" + classes := "{Zibm_r,2,argList{Yu8,Yxm,Yxr}}," + if row[1] == "PSHUFW" { + table, classes = "ymshuf", "{Zibm_r,2,argList{Yi8,Ymm,Ymr}}," + } + definition := regexp.MustCompile(`(?s)var ` + table + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(string(source)) + if len(definition) != 2 || strings.Join(strings.Fields(definition[1]), "") != classes || + row[2] != table || row[3] != prefix || row[4] != "0x70, 00" && row[4] != "0x70, 0" { + t.Fatal("unmodeled complete legacy prefix/operands/opcode", row) + } + } +} + +func TestX86ImmediateShuffleByteKnowledgeAndUnaffectedState(t *testing.T) { + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + widths := []int{16} + if strings.HasPrefix(op, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + for _, mode := range []string{"plain", "merge", "zero"} { + if !strings.HasPrefix(op, "V") && mode != "plain" { + continue + } + for _, alias := range []bool{false, true} { + for _, immediate := range []int{0, 27, 228, 255} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + destination := 1 + if alias { + destination = 0 + } + row := op + if mode == "zero" { + row += ".Z" + } + row += fmt.Sprintf(" $%d,%s0,", immediate, bank) + if mode != "plain" { + row += "K1," + } + row += fmt.Sprint(bank, destination) + t.Run(row, func(t *testing.T) { + ins := x86YMMReadSource(t, row) + for seed := 0; seed < 32; seed++ { + initial := x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, gOriginal: true, numericFlags: true, + callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, known: map[Reg]uint8{AX: 0xff}, + vectors: map[Reg]uint64{"Z0": uint64(seed)*0x1937a73b15d94ca7 ^ 0x31c097642eaf859b, + "Z1": uint64(seed)*0xf03755e989ab7361 ^ 0xae78f9234cbad619, "Z31": 0x55}, + masks: map[Reg]bool{"K1": true, "K0": false}, + } + state, want := x86CloneCallerState(initial), x86CloneCallerState(initial) + var expected uint64 + lane := 4 + if strings.Contains(op, "HW") || strings.Contains(op, "LW") { + lane = 2 + } + old := initial.vectors[Reg(fmt.Sprint("Z", destination))] + for output := 0; output < 64; output++ { + known := old>>uint(output)&1 != 0 + if output >= width { + known = strings.HasPrefix(op, "V") || known + } else { + at := output + if lane == 4 { + at = output/16*16 + (immediate>>(2*(output%16/4))&3)*4 + output%4 + } else if word := output % 16 / 2; word < 4 && strings.Contains(op, "LW") || word >= 4 && strings.Contains(op, "HW") { + half := 0 + if strings.Contains(op, "HW") { + half = 8 + } + at = output/16*16 + half + (immediate>>(2*(word%4))&3)*2 + output%2 + } + known = initial.vectors["Z0"]>>uint(at)&1 != 0 + if mode == "merge" { + known = known && old>>uint(output)&1 != 0 + } + } + if known { + expected |= uint64(1) << uint(output) + } + } + want.vectors[Reg(fmt.Sprint("Z", destination))] = expected + // The independent result observer must consume the same + // family, not reject it after correct caller/lowering work. + result := x86ResultState{flags: 0x3f, fp: []bool{true, false}, auto: []bool{false, true}} + result.gp[0], result.pointerGP[7], result.masks[1] = 0xff, true, true + for _, index := range []int{0, 1, 31} { + result.writeVector(index, 64, initial.vectors[Reg(fmt.Sprint("Z", index))], false) + } + wantResult := result.clone() + wantResult.writeVector(destination, 64, expected, false) + if !x86ResultTransfer(ins, &result, nil, 0, "amd64") || !reflect.DeepEqual(result, wantResult) { + t.Fatalf("%s seed=%d lost result bytes or unrelated state: got=%+v want=%+v", row, seed, result, wantResult) + } + if !x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, want) { + t.Fatalf("%s seed=%d changed byte dependencies or unrelated state: got=%+v want=%+v", row, seed, state, want) + } + state = x86CloneCallerState(initial) + state.masks["K1"] = false + before := x86CloneCallerState(state) + if mode != "plain" && (x86CallerVectorMoveStep(&ins, &state, nil) || !reflect.DeepEqual(state, before)) { + t.Fatal("unknown mask acquired data or mutated state") + } + result.masks[1] = false + beforeResult := result.clone() + if mode != "plain" && (x86ResultTransfer(ins, &result, nil, 0, "amd64") || !reflect.DeepEqual(result, beforeResult)) { + t.Fatal("unknown result mask acquired data or mutated state") + } + } + }) + } + } + } + } + } +} + +func TestX86ImmediateShuffleActualCallerThreeAPIs(t *testing.T) { + var body strings.Builder + rows := x86OfficialImmediateShuffleRows(t) + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + rows = append(rows, op+" $255,X0,X1") + } + for _, row := range rows { + destination := strings.Split(row, ",") + vector := destination[len(destination)-1] + index := vector[1:] + fmt.Fprintf(&body, "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z%s\nKMOVQ AX,K1\n%s\nVMOVSS X%s,(DX)\n", index, row, index) + } + // Broadcast, full tuple, merge and zeroing use actual caller input regions. + for _, row := range []string{"VPSHUFD.BCST.Z $255,(DI),K1,Z1", "VPSHUFHW.Z $0,(DI),K1,Z1", "VPSHUFLW $27,(DI),K1,Y1"} { + fmt.Fprintf(&body, "VMOVUPS (SI),Z1\nKMOVQ AX,K1\n%s\nVMOVSS X1,(DX)\n", row) + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body.String(), 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "immediate-shuffle-text.ll", "immediate-shuffle-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "immediate-shuffle-module.ll", "immediate-shuffle-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("actual selected caller failed", err) + } + }) + } +} + +func TestX86ImmediateShufflePartialSourceActualCaller(t *testing.T) { + for _, test := range []struct { + name, body string + accepted bool + }{ + {"only selected dword", "MOVSS (DI),X0\nVPSHUFD $0,X0,Y1", false}, + {"selected low dword", "MOVSS (DI),X2\nMOVSS X2,X0\nVPSHUFD $0,X0,X1", true}, + {"selected unknown dword", "MOVSS (DI),X2\nMOVSS X2,X0\nVPSHUFD $255,X0,X1", false}, + {"unused upper words", "MOVSS (DI),X2\nMOVSS X2,X0\nVPSHUFLW $0,X0,X1", true}, + {"whole unknown vector", "VPSHUFD $0,X0,X1", false}, + {"unknown mask", "VMOVUPS (DI),Z0\nVPSHUFD.Z $0,X0,K1,X1", false}, + {"merge unknown destination", "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVPSHUFD $0,X0,K1,X1", false}, + {"zeroing no old destination", "VMOVUPS (DI),Z0\nKMOVQ AX,K1\nVPSHUFD.Z $0,X0,K1,X1", true}, + {"legacy alignment unbound", "PSHUFL $0,(DI),X1", false}, + } { + t.Run(test.name, func(t *testing.T) { + // The width-mismatch case is a syntax negative, not caller Context. + if test.name == "only selected dword" { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+test.body+"\nRET\n", "", false) + if _, known, err := x86ImmediateShuffleProperties(x86YMMReadSource(t, "VPSHUFD $0,X0,Y1"), "amd64"); !known || err == nil { + t.Fatal("width mismatch acquired shuffle grammar") + } + return + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", test.body+"\nVMOVSS X1,(DX)", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("partial caller accepted=%v got %v", test.accepted, err) + } + }) + } + + ins := x86YMMReadSource(t, "vpshufd $27,X0,X1") + var output strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &output} + if known, _, err := ctx.lowerPackedWordShuffle("VPSHUFHW", ins); !known || err == nil || output.Len() != 0 { + t.Fatal("contradictory dispatch consumed source or emitted IR") + } +} diff --git a/amd64_immediate_shuffle_family_test.go b/amd64_immediate_shuffle_family_test.go new file mode 100644 index 000000000..0e71bb8e4 --- /dev/null +++ b/amd64_immediate_shuffle_family_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// Read every original encoding row and its opcode-specific attributes. Sharing +// _yvpshufd does not give the word shuffles dword broadcast permissions. +func x86OfficialImmediateShuffleRows(t *testing.T) []string { + t.Helper() + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + var rows []string + counts := map[string]int{} + for _, rule := range rules { + pp := map[string]string{"VPSHUFD": "66", "VPSHUFHW": "F3", "VPSHUFLW": "F2"}[rule.Opcode] + if pp == "" { + continue + } + index := rule.RowIndex + classes := [][]string{ + {"Yu8", "Yxm", "Yxr"}, {"Yi8", "Yxm", "Yxr"}, + {"Yu8", "Yym", "Yyr"}, {"Yi8", "Yym", "Yyr"}, + {"Yu8", "YxmEvex", "YxrEvex"}, {"Yu8", "YxmEvex", "Yknot0", "YxrEvex"}, + {"Yu8", "YymEvex", "YyrEvex"}, {"Yu8", "YymEvex", "Yknot0", "YyrEvex"}, + {"Yu8", "Yzm", "Yzr"}, {"Yu8", "Yzm", "Yknot0", "Yzr"}, + } + zcase := "Zvex_i_rm_r" + if index >= 4 { + zcase = "Zevex_i_rm_r" + if index%2 == 1 { + zcase = "Zevex_i_rm_k_r" + } + } + offsets := []int{0, 0, 2, 2, 4, 4, 7, 7, 10, 10} + advances := []int{0, 2, 0, 2, 0, 3, 0, 3, 0, 3} + if index < 0 || index >= len(classes) || rule.Ytab != "_yvpshufd" || rule.Prefix != "Pavx" || + !reflect.DeepEqual(rule.Classes, classes[index]) || rule.ZCase != zcase || + rule.EncodingOffset != offsets[index] || rule.ZOffset != advances[index] || len(rule.OpBytes) != 13 { + t.Fatalf("unmodeled original shuffle row: %+v", rule) + } + generation, bits := "vex", "128" + if index == 2 || index == 3 || index == 6 || index == 7 { + bits = "256" + } else if index >= 8 { + bits = "512" + } + if index >= 4 { + generation = "evex" + } + terms := []string{"avxEscape", generation + bits, generation + pp, generation + "0F", generation + "W0"} + if !reflect.DeepEqual(strings.Split(rule.OpBytes[rule.EncodingOffset], " | "), terms) { + t.Fatalf("unmodeled original shuffle prefix: %+v", rule) + } + opcodeOffset := rule.EncodingOffset + 1 + if generation == "evex" { + attributes := "evexN" + map[string]string{"128": "16", "256": "32", "512": "64"}[bits] + if rule.Opcode == "VPSHUFD" { + attributes += " | evexBcstN4" + } + attributes += " | evexZeroingEnabled" + if rule.OpBytes[opcodeOffset] != attributes { + t.Fatalf("unmodeled original shuffle attributes: %+v", rule) + } + opcodeOffset++ + } + if rule.OpBytes[opcodeOffset] != "0x70" { + t.Fatalf("unmodeled original shuffle opcode: %+v", rule) + } + bank := map[string]string{"128": "X", "256": "Y", "512": "Z"}[bits] + immediate := "255" + if rule.Classes[0] == "Yi8" { + immediate = "-128" + } + args := "$" + immediate + "," + bank + "0," + if len(rule.Classes) == 4 { + args += "K1," + } + // High banks force the unmasked X/Y EVEX row instead of VEX selection. + destination := "1" + if generation == "evex" && bits != "512" { + destination = "17" + } + rows = append(rows, rule.Opcode+" "+args+bank+destination) + counts[rule.Opcode]++ + } + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + if counts[op] != 10 { + t.Fatalf("%s complete original-row denominator changed: %d", op, counts[op]) + } + } + if len(rows) != 30 { + t.Fatalf("complete original shuffle denominator changed: %d", len(rows)) + } + return rows +} + +func TestX86ImmediateShuffleOfficialCallerRegistration(t *testing.T) { + rows := x86OfficialImmediateShuffleRows(t) + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + rows = append(rows, op+" $255,X0,X1") + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if !x86CallerVectorMoveKnown(file.Funcs[0].Instrs[1]) { + t.Fatal("Go-accepted original shuffle row has no caller effect") + } + }) + } +} + +func TestX86ImmediateShuffleMemoryShapeIsPhysicalTuple(t *testing.T) { + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for bank, width := range map[string]int{"X": 16, "Y": 32, "Z": 64} { + for _, suffix := range []string{"", ".Z"} { + row := fmt.Sprintf("%s%s $0,(AX),K1,%s1", op, suffix, bank) + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + bytes, write, valid := x86InstructionMemoryShape(file.Funcs[0].Instrs[1], 1, "amd64") + if !valid || write || bytes != int64(width) { + t.Fatalf("physical tuple read got %d/%v/%v, want %d/false/true", bytes, write, valid, width) + } + }) + } + } + } +} + +func TestCrossLinuxRuntimeMatrixX86ImmediateShuffleCompleteBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + widths := []int{16} + if strings.HasPrefix(op, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + for _, mode := range []int{0, 1, 2} { + if !strings.HasPrefix(op, "V") && mode != 0 { + continue + } + broadcasts := []bool{false} + if op == "VPSHUFD" { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("immshuffle%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + row := op + if broadcast { + row += ".BCST" + } + if mode == 2 { + row += ".Z" + } + input := bank + "0" + if broadcast { + input = "(BX)" + } + row += " $27," + input + "," + if mode != 0 { + row += "K1," + } + row += bank + "1" + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ in+8(FP),BX\nMOVQ mask+16(FP),DX\n", name) + fmt.Fprintf(&source, "VMOVUPS (BX),Z0\nVMOVUPS (AX),Z1\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z1,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", row) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}} + kind := 0 + if strings.Contains(op, "HW") { + kind = 2 + } else if strings.Contains(op, "LW") { + kind = 1 + } + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,\"%s\",%d,%d,%d,%d,%d)) return 1;\n", name, name, width, kind, mode, + x86FloatAddSubBoolInt(strings.HasPrefix(op, "V")), x86FloatAddSubBoolInt(broadcast)) + } + } + } + } + if count != 40 { + t.Fatalf("independent complete runtime denominator changed: %d", count) + } + oracle := `#include +#include +#include +` + declarations.String() + ` +static int check(void (*program)(uint8_t*,const uint8_t*,uint64_t),const char *name, + unsigned width,int kind,int mode,int vector,int broadcast) { + const uint64_t masks[]={0,1,UINT64_MAX,UINT64_C(0x55aa55aa55aa55aa)}; + for(unsigned m=0;m<4;m++) { + uint8_t input[66],original[66],output[68],before[68]; + for(unsigned i=0;i<66;i++) input[i]=(uint8_t)(i*29+197); + for(unsigned i=0;i<68;i++) output[i]=(uint8_t)(i*13+71); + memcpy(original,input,sizeof(input)); memcpy(before,output,sizeof(output)); + program(output+1,input+1,masks[m]); + if(output[0]!=before[0] || output[67]!=before[67] || + output[65]!=1 || output[66]!=0 || memcmp(input,original,sizeof(input))) return 1; + unsigned lane=kind ? 2 : 4; + for(unsigned i=0;i<64;i++) { + uint8_t want=before[1+i]; + if(i>=width) { if(vector) want=0; } + else if(mode && !(masks[m]>>(i/lane)&1)) { if(mode==2) want=0; } + else { + unsigned at=i; + if(broadcast) at=i%4; + else if(kind==0) at=(i/16)*16+((27>>(2*((i%16)/4)))&3)*4+i%4; + else { + unsigned word=(i%16)/2; + if((kind==1 && word<4) || (kind==2 && word>=4)) + at=(i/16)*16+(kind==2 ? 8 : 0)+((27>>(2*(word%4)))&3)*2+i%2; + } + want=input[1+at]; + } + if(output[1+i]!=want) { + fprintf(stderr,"%s mask=%llx byte=%u got=%x want=%x\n",name,(unsigned long long)masks[m],i,output[1+i],want); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "immediate-shuffle-complete-bytes", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_immediate_shuffle_fault_test.go b/amd64_immediate_shuffle_fault_test.go new file mode 100644 index 000000000..569cbed92 --- /dev/null +++ b/amd64_immediate_shuffle_fault_test.go @@ -0,0 +1,260 @@ +//go:build unix + +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ImmediateShuffleLegacyNativeMemory(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, references, checks strings.Builder + sigs := map[string]FuncSig{} + for index, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + // The C oracle executes bytes independently observed from original Go, + // with fixed RSI/RDI inputs. It does not reuse the production grammar. + original := "TEXT f(SB),4,$0-0\n" + op + " $27,(SI),X1\nRET\n" + listing := x86InsertExtractGoListing(t, "amd64", original) + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + prefixByte := byte(0x66) + if op == "PSHUFHW" { + prefixByte = 0xf3 + } else if op == "PSHUFLW" { + prefixByte = 0xf2 + } + if !bytes.Equal(code, []byte{prefixByte, 0x0f, 0x70, 0x0e, 27, 0xc3}) { + t.Fatalf("independent original Go m128 opcode changed: %s %x", op, code) + } + var literals []string + for _, value := range code[:5] { + literals = append(literals, fmt.Sprintf("0x%02x", value)) + } + kind := 0 + if op == "PSHUFHW" { + kind = 2 + } else if op == "PSHUFLW" { + kind = 1 + } + for _, dead := range []bool{false, true} { + name := fmt.Sprintf("legacymemory%d_%t", index, dead) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ out+0(FP),AX\nMOVQ input+8(FP),BX\nCMPQ BX,BX\nSTC\n%s $27,(BX),X1\n", name, op) + tail := "" + if !dead { + source.WriteString("MOVOU X1,(AX)\nSETCS 16(AX)\nSETNE 17(AX)\n") + tail = "; movdqu %%xmm1,(%%rdi); setc 16(%%rdi); setnz 17(%%rdi)" + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*);\n", name) + fmt.Fprintf(&references, `static void original_%s(uint8_t *out,const uint8_t *input) { + __asm__ volatile("cmpq %%%%rsi,%%%%rsi; stc; .byte %s%s" :: + "D"(out), "S"(input) : "xmm1", "cc", "memory"); +} +`, name, strings.Join(literals, ","), tail) + fmt.Fprintf(&checks, ` + if(compare(original_%[1]s,%[1]s,region,0,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,boundary-16,0,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,region+1,-1,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,boundary-15,1,%[2]d,%[3]d) || + compare(original_%[1]s,%[1]s,0,1,%[2]d,%[3]d)) return 1; +`, name, kind, x86FloatAddSubBoolInt(dead)) + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := `#include +#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + references.String() + ` +typedef void (*memoryfn)(uint8_t*,const uint8_t*); +static int observe(memoryfn fn,const uint8_t *input,int kind,int dead) { + pid_t child=fork(); + if(child<0) return -1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + uint8_t result[20], before[16]; + memset(result,0xc3,sizeof(result)); + // Copy only for a successful readable input; boundary and NULL probes + // must fault in the tested opcode, not in the independent reference. + if(input && !((uintptr_t)input & 15)) memcpy(before,input,16); + fn(result+1,input); + if(result[0]!=0xc3 || result[19]!=0xc3) _exit(3); + if(dead) { + for(unsigned i=1;i<19;i++) if(result[i]!=0xc3) _exit(4); + } else { + if(result[17]!=1 || result[18]!=0) _exit(5); + for(unsigned i=0;i<16;i++) { + unsigned at=i; + if(kind==0) at=((27>>(2*(i/4)))&3)*4+i%4; + else { + unsigned word=i/2; + if((kind==1 && word<4) || (kind==2 && word>=4)) + at=(kind==2 ? 8 : 0)+((27>>(2*(word%4)))&3)*2+i%2; + } + if(result[i+1]!=input[at]) _exit(6); + } + if(!((uintptr_t)input & 15) && memcmp(before,input,16)) _exit(7); + } + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status)) return WEXITSTATUS(status); + if(WIFSIGNALED(status)) return 128+WTERMSIG(status); + return -1; +} +static int compare(memoryfn reference,memoryfn translated,const uint8_t *input, + int fault,int kind,int dead) { + int native=observe(reference,input,kind,dead); + int got=observe(translated,input,kind,dead); + int memoryFault=native==128+SIGSEGV || native==128+SIGBUS; + int valid=fault==1 ? memoryFault : fault==-1 ? native==0 || memoryFault : native==0; + if(native<0 || native!=got || !valid) { + fprintf(stderr,"legacy shuffle input=%p native=%d LLVM=%d requiredFault=%d\n",input,native,got,fault); + return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 1; + uint8_t *region=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,(size_t)page,PROT_NONE)) return 1; + for(long i=0;i +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +static int check(void (*program)(const uint8_t*),const uint8_t *input,int fault) { + pid_t child=fork(); + if(child<0) return 1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + program(input); + _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return 1; + int got=WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS); + if(fault ? !got : !WIFEXITED(status) || WEXITSTATUS(status)!=0) { + fprintf(stderr,"shuffle physical-read input=%p fault=%d status=%x\n",input,fault,status); + return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<=0) return 1; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED || mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary=memory+page; +` + checks.String() + ` + return munmap(memory,(size_t)page*2)!=0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "immediate-shuffle-native-reads", triple, text, oracle, prefix) + }) + } +} diff --git a/amd64_immediate_shuffle_forms.go b/amd64_immediate_shuffle_forms.go new file mode 100644 index 000000000..e6de0ce04 --- /dev/null +++ b/amd64_immediate_shuffle_forms.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ImmediateShuffleSpec struct { + laneBytes int + high, legacy, mmx bool +} + +// Go's ymshuf/yxshuf and complete _yvpshufd rows share the four-element +// selector grammar. Only the dword EVEX spelling permits m32 broadcast. +var x86ImmediateShuffleSpecs = map[Op]x86ImmediateShuffleSpec{ + "PSHUFW": {2, false, true, true}, + "PSHUFL": {4, false, true, false}, + "PSHUFD": {4, false, true, false}, + "PSHUFHW": {2, true, true, false}, + "PSHUFLW": {2, false, true, false}, + "VPSHUFD": {4, false, false, false}, + "VPSHUFHW": {2, true, false, false}, + "VPSHUFLW": {2, false, false, false}, +} + +type x86ImmediateShuffleForm struct { + spec x86ImmediateShuffleSpec + source Operand + destination, mask, memory int + byteWidth, memoryBytes int + immediate uint8 + broadcast, zeroing bool +} + +// This supplies encoding, native access extent and byte selectors, not source +// values, original pointer/alignment backing or permission to borrow an ABI. +func x86ImmediateShuffleProperties(ins Instr, arch string) (x86ImmediateShuffleForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86ImmediateShuffleSpecs[Op(base)] + if !known { + return x86ImmediateShuffleForm{}, false, nil + } + f := x86ImmediateShuffleForm{spec: spec, destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86ImmediateShuffleForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside Go immediate-shuffle grammar: %q", arch, ins.Op, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || len(ins.Args) != 3 && len(ins.Args) != 4 { + return fail() + } + immediate, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer { + return fail() + } + if spec.legacy { + if dotted || len(ins.Args) != 3 || spec.mmx && !amd64SignedImmediate(immediate, 8) || + !spec.mmx && !amd64UnsignedImmediate(immediate, 8) { + return fail() + } + f.byteWidth = 16 + if spec.mmx { + f.byteWidth = 8 + } + } else { + switch suffix { + case "": + if dotted { + return fail() + } + case "Z": + f.zeroing = true + case "BCST": + f.broadcast = true + case "BCST.Z": + f.broadcast, f.zeroing = true, true + default: + return fail() + } + if immediate.Imm < -128 || immediate.Imm > 255 || f.broadcast && spec.laneBytes != 4 { + return fail() + } + if len(ins.Args) == 4 { + if arch == "386" && !ins.x86Encoded || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + f.mask = 2 + } else if f.zeroing { + return fail() + } + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + } + register := func(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + if spec.mmx { + _, valid := amd64ParseMReg(operand.Reg) + return valid + } + if spec.legacy { + index, valid := amd64ParseXReg(operand.Reg) + return valid && (arch == "amd64" && index < 16 || arch == "386" && index < 8) + } + return amd64GoPackedVectorMoveRegister(operand, f.byteWidth, arch) + } + if !register(ins.Args[f.destination]) { + return fail() + } + local, err := x86LegacyMMXReadInstruction(ins, arch) + if err != nil { + return f, true, err + } + f.source = local.Args[1] + if isAMD64MemoryOperand(f.source) && x86PackedShiftMemoryValid(f.source, arch) { + f.memory, f.memoryBytes = 1, f.byteWidth + if f.broadcast { + f.memoryBytes = 4 + } + } else if f.broadcast || !register(f.source) { + return fail() + } + if !spec.legacy && immediate.Imm < 0 && + (f.broadcast || amd64PackedWordShuffleRequiresEVEX(f.source, ins.Args[f.destination], f.byteWidth, f.mask >= 0)) { + return fail() + } + f.immediate = uint8(immediate.Imm) + return f, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func x86ImmediateShuffleConstant(operand Operand, arch string, encoded bool) (Operand, bool) { + if operand.Kind != OpImm || operand.ImmIsFloat || operand.ImmRaw != "" { + return operand, false + } + // Go's I386 oclass classifies TYPE_CONST after signed int32 truncation; + // retain the original operand, but use its classified value for every row. + if arch == "386" && !encoded { + operand.Imm = int64(int32(operand.Imm)) + } + return operand, true +} + +// Every output byte depends on exactly one source byte; the word half not +// shuffled still comes from the source, never the old destination. +func (f x86ImmediateShuffleForm) sourceByte(destination int) int { + if f.broadcast { + return destination % 4 + } + groupBytes := 16 + if f.spec.mmx { + groupBytes = 8 + } + lane := destination % groupBytes / f.spec.laneBytes + if f.spec.laneBytes == 2 && !f.spec.mmx && (lane < 4) == f.spec.high { + return destination + } + half := 0 + if f.spec.high { + half = 8 + } + return destination/groupBytes*groupBytes + half + + int(f.immediate>>(2*(lane%4))&3)*f.spec.laneBytes + destination%f.spec.laneBytes +} diff --git a/amd64_immediate_shuffle_grammar_test.go b/amd64_immediate_shuffle_grammar_test.go new file mode 100644 index 000000000..6a9203142 --- /dev/null +++ b/amd64_immediate_shuffle_grammar_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86ImmediateShuffleCompleteImmediateDomains(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT probe(SB),4,$0-0\n") + count := 0 + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for value := 0; value < 256; value++ { + immediate, bank := value, "X" + if op == "PSHUFW" { + immediate, bank = value-128, "M" + } + row := fmt.Sprintf("%s $%d,%s0,%s1", op, immediate, bank, bank) + fmt.Fprintln(&source, row) + ins := x86YMMReadSource(t, row) + before := fmt.Sprintf("%#v", ins) + if _, known, err := x86ImmediateShuffleProperties(ins, arch); !known || err != nil { + t.Fatalf("complete imm8 domain missing %s: %v", row, err) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("pure grammar changed original source") + } + count++ + } + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, source.String(), "", true) + if count != 2048 { + t.Fatal("complete independent immediate denominator changed", count) + } + }) + } +} + +func TestX86ImmediateShuffleRejectsSiblingInvalidFormats(t *testing.T) { + var rows []string + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + for _, args := range []string{"$-1,X0,X1", "$256,X0,X1", "$0,X16,X1", "$0,X0,Y1", "$0,AX,X1", "$0,X0,(AX)", "$0,X0,K1,X1"} { + rows = append(rows, op+" "+args) + } + rows = append(rows, op+".Z $0,X0,X1") + } + for _, args := range []string{"$128,M0,M1", "$-129,M0,M1", "$0,X0,M1", "$0,M0,X1", "$0,M8,M1", "$0,M0,(AX)"} { + rows = append(rows, "PSHUFW "+args) + } + rows = append(rows, "PSHUFW.Z $0,M0,M1") + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, args := range []string{"$-129,X0,X1", "$256,X0,X1", "$-1,Z0,Z1", "$-1,X16,X1", "$-1,X0,K1,X1", "$0,X0,Y1", "$0,AX,X1", "$0,X0,(AX)", "$0,X0,K0,X1", "$0,X0,K8,X1", "$0,X0,X32"} { + rows = append(rows, op+" "+args) + } + rows = append(rows, op+".Z $0,X0,X1", op+".BCST $0,X0,X1", op+".SAE $0,(AX),X1") + if op != "VPSHUFD" { + rows = append(rows, op+".BCST $0,(AX),X1") + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + masklessZero := row == "VPSHUFD.Z $0,X0,X1" || row == "VPSHUFHW.Z $0,X0,X1" || row == "VPSHUFLW.Z $0,X0,X1" + historicalReserved := masklessZero && !goToolchainAtLeast(runtime.Version(), 1, 21) + goAssemblerOracleOutput(t, arch, source, "", historicalReserved) + if masklessZero { + // Before Go 1.21 the encoder emitted reserved EVEX.z/aaa=0. + // Observe its exact original bytes; neither named nor raw + // translation may acquire semantics from that old acceptance. + pp := byte(1) + if strings.HasPrefix(row, "VPSHUFHW") { + pp = 2 + } else if strings.HasPrefix(row, "VPSHUFLW") { + pp = 3 + } + code := []byte{0x62, 0xf1, 0x7c | pp, 0x88, 0x70, 0xc8, 0} + if historicalReserved { + listing := x86InsertExtractGoListing(t, arch, source) + var observed []byte + for _, line := range strings.Split(string(listing), "\n") { + observed = append(observed, x86InsertExtractListedBytes(line)...) + } + if !bytes.Equal(observed, append(append([]byte(nil), code...), 0xc3)) { + t.Fatalf("historical reserved shuffle bytes changed: %x", observed) + } + t.Logf("%s/%s historical reserved z=1/aaa=0 observation; not a legal form", runtime.Version(), arch) + } + mode := 64 + if arch == "386" { + mode = 32 + } + if _, _, known, err := decodedX86PackedShuffleInstruction(code, mode); !known || err == nil { + t.Fatal("reserved raw zeroing acquired shuffle semantics", err) + } + x86RequireZeroMaskAPIOutcome(t, arch, source, false) + x86RequireZeroMaskAPIOutcome(t, arch, x86RawINSERTPSSource("f", code), false) + } + ins := x86YMMReadSource(t, row) + if _, known, err := x86ImmediateShuffleProperties(ins, arch); !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go-rejected syntax became a valid/context form", row, arch, err) + } + } + }) + } +} + +func TestX86ImmediateShuffleMode32PhysicalAndMMXBoundaries(t *testing.T) { + for _, op := range []string{"VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + for _, row := range []string{op + " $0,X16,X31", op + " $0,Y16,Y31"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + if _, known, err := x86ImmediateShuffleProperties(x86YMMReadSource(t, row), "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("source-only high bank acquired physical binding", row, err) + } + } + for _, row := range []string{op + " $0,Z8,Z0", op + " $0,X0,K1,X1"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + if _, known, err := x86ImmediateShuffleProperties(x86YMMReadSource(t, row), "386"); !known || err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go-rejected mode32 form acquired physical binding", row, err) + } + } + } + for _, arch := range []string{"amd64", "386"} { + ins := x86YMMReadSource(t, "PSHUFW $27,AX,M1") + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+ins.Raw+"\nRET\n", "", true) + form, known, err := x86ImmediateShuffleProperties(ins, arch) + if !known || err != nil || form.source.Reg != "M0" || !form.spec.mmx { + t.Fatal("original Go Ymm spelling lost its actual MMX bank", arch, form, err) + } + if !errors.Is(x86LegacyMMXReadEmitterContext(ins, arch), ErrProbeNeedsContext) || x86CallerVectorMoveKnown(ins) { + t.Fatal("GP source spelling became initialized MMX data or caller permission") + } + } +} + +func TestX86ImmediateShuffleNativeFeatureScope(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{"PSHUFL $0,(AX),X1", "PSHUFD $0,(AX),X1", "PSHUFHW $0,(AX),X1", "PSHUFLW $0,(AX),X1"} { + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, Func{Instrs: []Instr{x86YMMReadSource(t, row)}}); got != "+sse2" { + t.Fatal("validated native legacy m128 needs its private SSE2 carrier", arch, row, got) + } + } + for _, row := range []string{"PSHUFL $-1,(AX),X1", "PSHUFL $0,X0,X1", "VPSHUFD $0,(AX),X1", "PSHUFW $0,(AX),M1"} { + if got := inferFuncTargetFeaturesForGOARCH(ArchAMD64, arch, Func{Instrs: []Instr{x86YMMReadSource(t, row)}}); got != "" { + t.Fatal("non-native/malformed form acquired features", arch, row, got) + } + } + } +} + +func TestX86ImmediateShuffleNativeMemoryCannotBorrowVirtualBacking(t *testing.T) { + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + for _, operand := range []string{"a+0(FP)", "(SP)", "(BP)", "(AX)(BP*2)"} { + row := op + " $0," + operand + ",X1" + t.Run(row, func(t *testing.T) { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-16\n"+row+"\nRET\n", "", true) + ins := x86YMMReadSource(t, row) + var output strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &output} + if known, _, err := ctx.lowerPackedWordShuffle(ins.Op, ins); !known || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatal("virtual/native frame acquired backing before IR", row, err, output.String()) + } + }) + } + for _, bits := range []int{32, 64} { + ins := x86YMMReadSource(t, op+" $0,(AX),X1") + ins.x86AddressBits = bits + var output strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &output} + if known, _, err := ctx.lowerPackedWordShuffle(ins.Op, ins); !known || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatal("unbound address-size identity acquired native backing", bits, err) + } + } + } +} diff --git a/amd64_immediate_shuffle_objects_test.go b/amd64_immediate_shuffle_objects_test.go new file mode 100644 index 000000000..f117b991d --- /dev/null +++ b/amd64_immediate_shuffle_objects_test.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86ImmediateShuffleOfficialRowsThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + rows := x86OfficialImmediateShuffleRows(t) + var selected []string + for _, row := range rows { + if target.arch == "386" { + if strings.Count(row, ",") == 3 { + // This is an actual frontend refusal, not a skipped test. + goAssemblerOracleOutput(t, target.arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + continue + } + row = strings.NewReplacer("X17", "X1", "Y17", "Y1").Replace(row) + } + selected = append(selected, row) + args := strings.Split(strings.Fields(row)[1], ",") + args[1] = "8(AX)" + selected = append(selected, strings.Fields(row)[0]+" "+strings.Join(args, ",")) + } + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW"} { + selected = append(selected, op+" $255,X0,X1", op+" $0,16(AX),X1") + } + selected = append(selected, "PSHUFW $-128,M0,M1", "PSHUFW $127,8(AX),M1") + if target.arch == "386" { + for _, op := range []string{"PSHUFW", "PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + bank, values := "X", []int{0, 27, 255} + if op == "PSHUFW" { + bank, values = "M", []int{0, -1, 127} + } else if strings.HasPrefix(op, "V") { + values = []int{0, -1, 255} + } + for _, value := range values { + selected = append(selected, fmt.Sprintf("%s $%d,%s0,%s1", op, int64(4294967296)+int64(value), bank, bank)) + } + } + } + for _, bank := range []string{"X", "Y", "Z"} { + selected = append(selected, "VPSHUFD.BCST $0,8(AX),"+bank+"1") + if target.arch == "amd64" { + selected = append(selected, "VPSHUFD.BCST.Z $255,8(AX),K1,"+bank+"1") + } + } + want := 76 + if target.arch == "386" { + want = 79 + } + if len(selected) != want { + t.Fatal("official row/operand variation denominator changed", len(selected), want) + } + var source, declarations strings.Builder + declarations.WriteString("package shuffle\n") + sigs := map[string]FuncSig{} + for index, row := range selected { + name := fmt.Sprintf("F%d", index) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\n%s\nRET\n", name, row) + fmt.Fprintf(&declarations, "func %s()\n", name) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs, ResolveSym: resolve} + before := fmt.Sprintf("%#v %#v", file, sigs) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "shuffle-text.ll", "shuffle-text.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "shuffle-module.ll", "shuffle-module.o", module.String()) + pkg := mustGoPackage(t, "example.invalid/shuffle", declarations.String()) + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "shuffle-go.ll", "shuffle-go.o", translated.Module.String()) + if before != fmt.Sprintf("%#v %#v", file, sigs) { + t.Fatal("object verification mutated original input or signatures") + } + t.Logf("%d original Go forms, three APIs and LLVM22 objects; not a caller/runtime certificate", len(selected)) + }) + } +} diff --git a/amd64_immediate_shuffle_result.go b/amd64_immediate_shuffle_result.go new file mode 100644 index 000000000..d8d3d3c39 --- /dev/null +++ b/amd64_immediate_shuffle_result.go @@ -0,0 +1,40 @@ +package plan9asm + +func x86ResultImmediateShuffleStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86ImmediateShuffleProperties(ins, arch) + if !known || err != nil || form.spec.mmx { + return false, known // Physical MMX does not alias the SIMD result bank. + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false, true + } + } + var source uint64 + if form.memory >= 0 { + // A selected byte or zero mask cannot shrink the native memory tuple. + // Legacy m128 also needs native alignment/backing; a logical FP/AUTO + // allocation is not that witness. Only a paired static read is known. + if _, static := x86StaticSBReadAddress(form.source, arch); form.spec.legacy && !static { + return false, true + } + if !x86ResultRead(form.source, int64(form.memoryBytes), state, owned, arch) { + return false, true + } + source = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + index, _ := amd64VectorRegisterIndex(form.source.Reg, form.byteWidth) + source = state.vectorBytes(index) + } + var defined uint64 + for output := 0; output < form.byteWidth; output++ { + defined |= (source >> uint(form.sourceByte(output)) & 1) << uint(output) + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + if form.mask >= 0 && !form.zeroing { + defined &= state.vectorBytes(destination) + } + state.writeVector(destination, form.byteWidth, defined, form.spec.legacy) + return true, true +} diff --git a/amd64_immediate_shuffle_result_test.go b/amd64_immediate_shuffle_result_test.go new file mode 100644 index 000000000..58e100569 --- /dev/null +++ b/amd64_immediate_shuffle_result_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86ImmediateShuffleResultMemoryExtent(t *testing.T) { + for _, row := range []struct { + instruction string + bytes int + }{ + {"VPSHUFD $0,value+0(FP),X1", 16}, + {"VPSHUFHW.Z $0,value+0(FP),K1,Y1", 32}, + {"VPSHUFLW.Z $0,value+0(FP),K1,Z1", 64}, + {"VPSHUFD.BCST.Z $0,value+0(FP),K1,Z1", 4}, + } { + t.Run(row.instruction, func(t *testing.T) { + ins := x86YMMReadSource(t, row.instruction) + for missing := -1; missing < 64; missing++ { + state := x86ResultState{fp: make([]bool, 64)} + owned := make([]bool, 64) + for index := range owned { + owned[index], state.fp[index] = true, index != missing + } + state.masks[1] = true + accepted := x86ResultTransfer(ins, &state, owned, 64, "amd64") + if want := missing < 0 || missing >= row.bytes; accepted != want { + t.Fatalf("full original tuple bytes=%d missing=%d accepted=%v", row.bytes, missing, accepted) + } + } + }) + } + for _, row := range []string{ + "PSHUFHW $0,value+0(FP),X1", "VPSHUFD $0,(AX),X1", + "VPSHUFD $0,unknown<>(SB),X1", "PSHUFW $0,M0,M1", + } { + state := x86ResultState{fp: make([]bool, 16)} + owned := make([]bool, 16) + for index := range owned { + owned[index], state.fp[index] = true, true + } + state.pointerGP[0] = true + if x86ResultTransfer(x86YMMReadSource(t, row), &state, owned, 16, "amd64") { + t.Fatal("logical storage, pointer or SIMD state acquired native backing/MMX authority", row) + } + } +} + +func amd64ImmediateShuffleResultMatrix() (source, declarations, harness string) { + var asm, decl, prototypes, checks strings.Builder + index := 0 + for _, op := range []string{"PSHUFL", "PSHUFD", "PSHUFHW", "PSHUFLW", "VPSHUFD", "VPSHUFHW", "VPSHUFLW"} { + width, start := 4, 0 + if strings.HasSuffix(op, "HW") || strings.HasSuffix(op, "LW") { + width = 2 + } + if strings.HasSuffix(op, "HW") { + start = 8 + } + for _, immediate := range []int{0, 27, 228, 255} { + for _, destination := range []string{"X0", "X1"} { + name := fmt.Sprint("shuffle", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$0-24\nMOVQ AX,X0\nPINSRQ $1,BX,X0\n%s $%d,X0,%s\nPEXTRQ $1,%s,CX\nMOVQ %s,AX\nXORQ CX,AX\nRET\n", + name, op, immediate, destination, destination, destination) + fmt.Fprintf(&decl, "func %s(a,b uint64) uint64\n", name) + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint64_t,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d)){fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, width, start, immediate, name) + index++ + } + } + } + const reference = ` +static int check(uint64_t (*fn)(uint64_t,uint64_t), unsigned width, + unsigned start, unsigned imm) { + const uint64_t values[]={0,UINT64_MAX,UINT64_C(0x1020304050607080),UINT64_C(0xfedcba9876543210)}; + for(unsigned a=0;a<4;a++) { + for(unsigned b=0;b<4;b++) { + uint8_t input[16],output[16]; + uint64_t low,high; + memcpy(input,&values[a],8); + memcpy(input+8,&values[b],8); + memcpy(output,input,16); + for(unsigned lane=0;lane<4;lane++) { + unsigned selected=(imm>>(2*lane))&3; + memcpy(output+start+width*lane,input+start+width*selected,width); + } + memcpy(&low,output,8); + memcpy(&high,output+8,8); + if(fn(values[a],values[b])!=(low^high)) return 1; + } + } + return 0; +} +` + return asm.String(), decl.String(), "#include \n#include \n#include \n" + + prototypes.String() + reference + "int main(void){\n" + checks.String() + "return 0; }\n" +} + +func TestAMD64ImmediateShuffleRegisterResultsObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + source, declarations, _ := amd64ImmediateShuffleResultMatrix() + requireAMD64RegisterResultGoObject(t, source) + for _, target := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", target, api) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target, ir, "shuffle-result") + }) + } + } +} + +func TestCrossLinuxRuntimeMatrixAMD64ImmediateShuffleRegisterResults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + source, declarations, harness := amd64ImmediateShuffleResultMatrix() + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimize := range []bool{false, true} { + text := ir + if optimize { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "shuffle-results", triple, text, harness, prefix) + } + }) + } +} diff --git a/amd64_insert_extract_forms.go b/amd64_insert_extract_forms.go new file mode 100644 index 000000000..f3559115d --- /dev/null +++ b/amd64_insert_extract_forms.go @@ -0,0 +1,206 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86InsertExtractSpec struct { + insert, scalar, vex, legacy bool + chunk, laneBits int + allowY, allowZ bool +} + +var x86InsertExtractSpecs = map[Op]x86InsertExtractSpec{ + "EXTRACTPS": {scalar: true, legacy: true, chunk: 4, laneBits: 32}, + "INSERTPS": {insert: true, scalar: true, legacy: true, chunk: 4, laneBits: 32}, + "VEXTRACTF128": {vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VEXTRACTI128": {vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VEXTRACTF32X4": {chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VEXTRACTI32X4": {chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VEXTRACTF64X2": {chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VEXTRACTI64X2": {chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VEXTRACTF32X8": {chunk: 32, laneBits: 32, allowZ: true}, + "VEXTRACTI32X8": {chunk: 32, laneBits: 32, allowZ: true}, + "VEXTRACTF64X4": {chunk: 32, laneBits: 64, allowZ: true}, + "VEXTRACTI64X4": {chunk: 32, laneBits: 64, allowZ: true}, + "VINSERTF128": {insert: true, vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VINSERTI128": {insert: true, vex: true, chunk: 16, laneBits: 8, allowY: true}, + "VINSERTF32X4": {insert: true, chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VINSERTI32X4": {insert: true, chunk: 16, laneBits: 32, allowY: true, allowZ: true}, + "VINSERTF64X2": {insert: true, chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VINSERTI64X2": {insert: true, chunk: 16, laneBits: 64, allowY: true, allowZ: true}, + "VINSERTF32X8": {insert: true, chunk: 32, laneBits: 32, allowZ: true}, + "VINSERTI32X8": {insert: true, chunk: 32, laneBits: 32, allowZ: true}, + "VINSERTF64X4": {insert: true, chunk: 32, laneBits: 64, allowZ: true}, + "VINSERTI64X4": {insert: true, chunk: 32, laneBits: 64, allowZ: true}, + "VEXTRACTPS": {scalar: true, chunk: 4, laneBits: 32}, + "VINSERTPS": {insert: true, scalar: true, chunk: 4, laneBits: 32}, +} + +type x86InsertExtractForm struct { + spec x86InsertExtractSpec + source, base, destination, mask, memory int + width, output, offset int + immediate byte + zeroing, memoryWrite bool +} + +func x86InsertExtractProperties(ins Instr, goarch string) (x86InsertExtractForm, bool, error) { + op, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86InsertExtractSpecs[Op(op)] + if !recognized { + return x86InsertExtractForm{}, false, nil + } + f := x86InsertExtractForm{spec: spec, source: 1, base: -1, destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86InsertExtractForm, bool, error) { + return f, true, fmt.Errorf("%s %s operands/suffix outside Go insert/extract grammar: %q", goarch, ins.Op, ins.Raw) + } + count := 3 + if spec.insert { + count, f.base = 4, 2 + if spec.legacy { + count = 3 + } + } + // Go's 386 frontend cannot represent a four-operand named insertion. + // A validated physical VINSERTPS has its independent mode32 register and + // encoding checks; a textual Raw annotation cannot create that identity. + physicalScalarInsert := spec.insert && spec.scalar && ins.x86Encoded + if goarch != "amd64" && goarch != "386" || spec.insert && !spec.legacy && goarch == "386" && !physicalScalarInsert || len(ins.Args) != count && len(ins.Args) != count+1 || + len(ins.Args) == count+1 && (spec.vex || spec.scalar) || dotted && (suffix != "Z" || spec.vex || spec.scalar) || + ins.Args[0].Kind != OpImm || ins.Args[0].ImmRaw != "" || ins.Args[0].ImmIsFloat { + return fail() + } + if len(ins.Args) == count+1 { + f.mask = count - 1 + if !amd64NonzeroKOperand(ins.Args[f.mask]) { + return fail() + } + } + if goarch == "386" && f.mask >= 0 { + return fail() + } + f.zeroing = dotted + if f.zeroing && f.mask < 0 { + return fail() + } + minimum := int64(0) + if !spec.insert && spec.vex { + minimum = -128 + } + if spec.scalar && !spec.insert && !spec.legacy { + index, valid := amd64VectorRegisterIndex(ins.Args[1].Reg, 16) + if valid && index < 16 { + minimum = -128 + } + } + maximum := int64(255) + if spec.legacy && !spec.insert { + maximum = 3 + } + if ins.Args[0].Imm < minimum || ins.Args[0].Imm > maximum { + return fail() + } + f.immediate = byte(ins.Args[0].Imm) + f.width, f.output = 16, spec.chunk + large := ins.Args[f.source] + if spec.insert { + large = ins.Args[f.base] + } + if !spec.scalar { + f.width = amd64VectorByteWidth(large.Reg) + if large.Kind != OpReg || !(f.width == 32 && spec.allowY || f.width == 64 && spec.allowZ) { + return fail() + } + f.offset = (int(f.immediate) & (f.width/spec.chunk - 1)) * spec.chunk + } else { + f.offset = int(f.immediate&3) * 4 + } + if spec.insert { + f.output = f.width + } + highMode32 := Reg("") + register := func(arg Operand, width int) bool { + index, valid := amd64VectorRegisterIndex(arg.Reg, width) + limit := 32 + if spec.vex || spec.legacy { + limit = 16 + } + if goarch == "386" && spec.legacy { + limit = 8 + } + if arg.Kind != OpReg || !valid || index >= limit { + return false + } + if goarch == "386" && index >= 8 { + highMode32 = arg.Reg + } + return true + } + if !register(large, f.width) { + return fail() + } + if spec.insert && !register(ins.Args[f.destination], f.output) { + return fail() + } + transfer := ins.Args[f.destination] + if spec.insert { + transfer = ins.Args[f.source] + } + if isAMD64MemoryOperand(transfer) { + f.memory, f.memoryWrite = f.destination, !spec.insert + if spec.insert { + f.memory = f.source + } + if transfer.Kind == OpSym { + _, absolute := parseInt(strings.TrimSpace(transfer.Sym)) + if strings.HasPrefix(strings.TrimSpace(transfer.Sym), "$") || !strings.HasSuffix(transfer.Sym, "(SB)") && absolute != nil { + return fail() + } + } + if transfer.Kind == OpMem && (!x86MemoryRegistersValidForArch(transfer.Mem, goarch) || + transfer.Mem.Index != "" && (transfer.Mem.Index == SP || transfer.Mem.Scale != 1 && transfer.Mem.Scale != 2 && transfer.Mem.Scale != 4 && transfer.Mem.Scale != 8) || + transfer.Mem.Off != int64(int32(transfer.Mem.Off)) && (goarch != "386" || transfer.Mem.Off != int64(uint32(transfer.Mem.Off)))) { + return fail() + } + } else if !spec.insert && spec.scalar { + if transfer.Kind != OpReg || !isX86YrlRegisterForArch(transfer.Reg, goarch) { + return fail() + } + } else { + width := spec.chunk + if spec.scalar { + width = 16 + } + if !register(transfer, width) { + return fail() + } + } + if f.zeroing && f.memoryWrite { + return fail() + } + if highMode32 != "" { + // Go can encode named VEX/EVEX high SIMD registers in mode32. + // Its syntax/object acceptance does not supply physical register + // identities which the independent mode32 raw decoder rejects. + return f, true, fmt.Errorf("%w: 386 insert/extract register %s has no proved physical mode32 binding", ErrProbeNeedsContext, highMode32) + } + return f, true, nil +} + +func (c *amd64Ctx) bindInsertExtractForm(op Op, ins Instr) (Instr, x86InsertExtractForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86InsertExtractSpecs[Op(base)]; !known { + return ins, x86InsertExtractForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + return ins, x86InsertExtractForm{}, true, fmt.Errorf("insert/extract dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + f, known, err := x86InsertExtractProperties(ins, c.goarch) + if err == nil && f.memory >= 0 && ins.Args[f.memory].Kind == OpFP { + err = c.validateFPVectorSpan(ins.Args[f.memory].FPOffset, f.spec.chunk) + } + return ins, f, known, err +} diff --git a/amd64_insert_extract_forms_test.go b/amd64_insert_extract_forms_test.go new file mode 100644 index 000000000..74ca09280 --- /dev/null +++ b/amd64_insert_extract_forms_test.go @@ -0,0 +1,260 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +// Independent full opcode/ytab axes, not a rendering of the production map. +func x86InsertExtractFormats() []string { + var rows []string + for _, action := range []string{"EXTRACT", "INSERT"} { + for _, kind := range []string{"F", "I"} { + for _, size := range []string{"128", "32X4", "64X2", "32X8", "64X4"} { + op, small := "V"+action+kind+size, "X" + widths := []string{"Y", "Z"} + if size == "128" { + widths = []string{"Y"} + } + if strings.HasSuffix(size, "X8") || strings.HasSuffix(size, "X4") && strings.HasPrefix(size, "64") { + small, widths = "Y", []string{"Z"} + } + for _, wide := range widths { + for _, memory := range []bool{false, true} { + for _, masking := range []string{"", "merge", "zero"} { + if size == "128" && masking != "" || action == "EXTRACT" && memory && masking == "zero" { + continue + } + mnemonic, mask := op, "" + if masking != "" { + mask = ",K1" + } + if masking == "zero" { + mnemonic += ".Z" + } + transfer := small + "0" + if memory { + transfer = "(BX)" + } + if action == "INSERT" { + rows = append(rows, fmt.Sprintf("%s $255,%s,%s1%s,%s2", mnemonic, transfer, wide, mask, wide)) + } else { + rows = append(rows, fmt.Sprintf("%s $255,%s1%s,%s", mnemonic, wide, mask, transfer)) + } + } + } + } + } + } + } + return append(rows, "VEXTRACTPS $255,X1,AX", "VEXTRACTPS $255,X1,(BX)", "VINSERTPS $255,X0,X1,X2", "VINSERTPS $255,(BX),X1,X2", + "EXTRACTPS $3,X1,AX", "EXTRACTPS $3,X1,(BX)", "INSERTPS $255,X0,X1", "INSERTPS $255,(BX),X1") +} + +func TestX86InsertExtractEveryGoOpcodeAttribute(t *testing.T) { + if len(x86InsertExtractSpecs) != 24 { + t.Fatal("incomplete family") + } + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + expected := map[string]string{ + "VEXTRACTF128": "vexW0, 0x19", "VEXTRACTI128": "vexW0, 0x39", + "VEXTRACTF32X4": "evexW0, evexN16", "VEXTRACTI32X4": "evexW0, evexN16", + "VEXTRACTF64X2": "evexW1, evexN16", "VEXTRACTI64X2": "evexW1, evexN16", + "VEXTRACTF32X8": "evexW0, evexN32", "VEXTRACTI32X8": "evexW0, evexN32", + "VEXTRACTF64X4": "evexW1, evexN32", "VEXTRACTI64X4": "evexW1, evexN32", + "VINSERTF128": "vexW0, 0x18", "VINSERTI128": "vexW0, 0x38", + "VINSERTF32X4": "evexW0, evexN16", "VINSERTI32X4": "evexW0, evexN16", + "VINSERTF64X2": "evexW1, evexN16", "VINSERTI64X2": "evexW1, evexN16", + "VINSERTF32X8": "evexW0, evexN32", "VINSERTI32X8": "evexW0, evexN32", + "VINSERTF64X4": "evexW1, evexN32", "VINSERTI64X4": "evexW1, evexN32", + "VEXTRACTPS": "evexW0, evexN4", "VINSERTPS": "evexW0, evexN4", + } + var observed []string + for _, row := range strings.Split(string(data), "{as: A") { + op := strings.SplitN(row, ",", 2)[0] + if !strings.HasPrefix(op, "VEXTRACT") && !strings.HasPrefix(op, "VINSERT") { + continue + } + observed = append(observed, op) + body := strings.SplitN(row, "\n\t}},", 2)[0] + spec, exists := x86InsertExtractSpecs[Op(op)] + if !exists { + t.Fatalf("unregistered actual Go opcode: %s", op) + } + if strings.Contains(body, "Bcst") || strings.Contains(body, "Sae") || + strings.Contains(body, "evexZeroingEnabled") != (!spec.scalar && !spec.vex) { + t.Fatalf("per-opcode flags changed: %s", body) + } + if spec.vex && strings.Contains(body, "evex") || !spec.vex && !strings.Contains(body, "evex") { + t.Fatalf("encoding identity changed: %s", body) + } + if attribute, exists := expected[op]; !exists || !strings.Contains(body, attribute) || + !strings.Contains(body, "0F3A") || spec.allowY != strings.Contains(body, "256") || spec.allowZ != strings.Contains(body, "512") { + t.Fatalf("per-opcode W/tuple/native width attributes changed: %s", body) + } + } + sort.Strings(observed) + if len(observed) != 22 { + t.Fatalf("actual Go family: %v", observed) + } + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{AEXTRACTPS, yextractps, Pq, opBytes{0x3a, 0x17, 0}}", + "{AINSERTPS, yxshuf, Pq, opBytes{0x3a, 0x21, 0}}", + "var yextractps = []ytab{\n\t{Zibr_m, 2, argList{Yu2, Yxr, Yml}}", + "var yxshuf = []ytab{\n\t{Zibm_r, 2, argList{Yu8, Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatalf("actual Go legacy PS row changed: %s", row) + } + } +} + +func TestX86InsertExtractFormatsAndFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + sigs := map[string]FuncSig{} + for index, row := range x86InsertExtractFormats() { + if arch == "386" && (strings.HasPrefix(row, "VINSERT") || strings.Contains(row, "K1")) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + continue + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, known, err := x86InsertExtractProperties(file.Funcs[0].Instrs[1], arch) + if !known || err != nil { + t.Fatalf("%s/%s: %v", arch, row, err) + } + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + name := fmt.Sprintf("transfer%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, row) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "transfer.ll", "transfer.o", ir) + } + } +} + +func TestX86InsertExtractDispatchIdentityAndGrammarBeforeEmission(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "UNKNOWN", "VEXTRACTPS.Z"} { + copy := ins + copy.Op = identity + original := copy + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerInsertExtractFamily(ins.Op, copy) + failure := identity == "UNKNOWN" || identity == "VEXTRACTPS.Z" + if !known || term || (err != nil) != failure || failure && output.String() != before || !reflect.DeepEqual(copy, original) { + t.Fatalf("%s/%s: %t/%t/%v", row, identity, known, term, err) + } + } + } + for _, row := range []string{ + "VEXTRACTF128 $256,Y0,X1", "VEXTRACTF128 $-129,Y0,X1", "VEXTRACTF128 $0,Y16,X1", + "VEXTRACTF64X4 $-1,Z0,Y1", "VEXTRACTF32X4.Z $0,Z0,X1", "VEXTRACTI32X4 $0,Z0,K0,X1", + "VEXTRACTPS $-1,X31,AX", "VEXTRACTPS $0,X0,X1", "VINSERTPS $-1,X0,X1,X2", + "VINSERTI128 $0,X0,Y0,Y16", "VINSERTF32X8.BCST $0,(AX),Z0,Z1", + "VINSERTF64X2.RN_SAE $0,X0,Z0,Z1", "VINSERTI64X4.Z $0,Y0,Z0,Z1", + "VINSERTPS $0,(AX)(SP*2),X0,X1", "VINSERTPS $0,2147483648(AX),X0,X1", + "VEXTRACTF32X4 $0,Z0,$data(SB)", + "VEXTRACTF128 $selector,Y0,X1", "VINSERTPS $1.5,X0,X1,X2", + "INSERTPS $selector,X0,X1", "EXTRACTPS $1.5,X0,AX", "EXTRACTPS $4,X0,AX", + "INSERTPS $0,X16,X0", "EXTRACTPS $0,X16,AX", "INSERTPS.Z $0,X0,X1", + } { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + // Go 1.20 emitted the reserved EVEX.z=1/aaa=0 combination for + // this unchanged negative. Current Go rejects it. Syntax acceptance + // cannot authorize that physical invalid-opcode form. + reservedZeroMask := row == "VEXTRACTF32X4.Z $0,Z0,X1" || row == "VINSERTI64X4.Z $0,Y0,Z0,Z1" + if reservedZeroMask { + x86InsertExtractReservedZeroMaskGoObservation(t, source) + } else { + goAssemblerOracleOutput(t, "amd64", source, "", false) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, _, err := ctx.lowerInsertExtractFamily(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !known || err == nil || output.String() != before { + t.Fatalf("malformed source emitted IR: %s: %v", row, err) + } + } +} + +func TestX86InsertExtractCompleteTypedFPSpanBeforeEmission(t *testing.T) { + for _, row := range x86InsertExtractFormats() { + if !strings.Contains(row, "(BX)") { + continue + } + row = strings.Replace(row, "(BX)", "a+0(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-32\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86InsertExtractProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + width := form.spec.chunk + for _, missing := range []int{-1, 0, width - 1} { + sig := FuncSig{Name: "f", Ret: Void} + for index := 0; index < width; index++ { + if index != missing { + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(index), Type: I8, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I8) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + known, term, err := ctx.lowerInsertExtractFamily(ins.Op, ins) + // Native E6NF requires a physical contiguous destination. Typed + // individual ABI0 shadow cells do not supply it, even if every + // slot is bound. This remains Context, not fabricated memory. + failure := missing >= 0 || form.memoryWrite && form.mask >= 0 + if !known || term || (err != nil) != failure || failure && output.String() != before { + t.Fatalf("%s missing FP byte %d: %t/%t/%v", row, missing, known, term, err) + } + } + } +} diff --git a/amd64_insert_extract_go_oracle_test.go b/amd64_insert_extract_go_oracle_test.go new file mode 100644 index 000000000..0c7d2d329 --- /dev/null +++ b/amd64_insert_extract_go_oracle_test.go @@ -0,0 +1,264 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func x86InsertExtractGoListing(t *testing.T, arch, source string) []byte { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "original.s"), filepath.Join(dir, "original.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "oracle", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH="+arch) + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), true, arch); failure != nil { + t.Fatalf("actual original assembler listing: %v\n%s", failure, output) + } + return output +} + +// Parse only concrete byte columns, not assembly mnemonic text. Go's listing +// starts its columns with 0xNNNN; llvm-objdump uses NNNN:. Both are independent +// physical object observations, rather than a rendering of the typed table. +func x86InsertExtractListedBytes(line string) []byte { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") && !strings.HasSuffix(fields[0], ":") { + return nil + } + var result []byte + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := hex.DecodeString(field) + if err != nil { + break + } + result = append(result, value[0]) + } + return result +} + +func TestX86InsertExtract386HighRegistersKeepPhysicalContext(t *testing.T) { + const source = "TEXT f(SB),4,$0-0\nVEXTRACTF128 $1,Y8,X9\nVEXTRACTPS $3,X8,AX\nRET\n" + listing := x86InsertExtractGoListing(t, "386", source) + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if !bytes.Equal(code, []byte{0xc4, 0x43, 0x7d, 0x19, 0xc1, 1, 0xc4, 0x63, 0x79, 0x17, 0xc0, 3, 0xc3}) { + t.Fatalf("actual Go named mode32 bytes changed: %x", code) + } + for _, op := range []Op{"VEXTRACTF128", "VEXTRACTPS"} { + decode := decodedX86VEXTRACT128Instruction + if op == "VEXTRACTPS" { + decode = decodedX86PackedScalarExtractInstruction + } + ins, size, recognized, err := decode(code, 64) + if !recognized || err != nil || ins.Op != op || ins.Args[1].Reg != map[Op]Reg{"VEXTRACTF128": "Y8", "VEXTRACTPS": "X8"}[op] { + t.Fatalf("actual Go high-register bytes %x lost source identity: %+v/%v", code, ins, err) + } + if _, _, recognized, err := decode(code[:size], 32); !recognized || err == nil { + t.Fatal("mode32 decoder invented high physical SIMD registers") + } + t.Logf("actual Go accepted %s bytes %x; mode32 physical binding remains Context", op, code[:size]) + code = code[size:] + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go's mode32 accepted bytes acquired ordinary physical meaning: %v", err) + } +} + +func TestX86InsertExtractNativeMaskedStorePhysicalObjects(t *testing.T) { + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM22 llc and llvm-objdump are required") + } + var source strings.Builder + sigs := map[string]FuncSig{} + want := map[string]int{} + count := 0 + for _, row := range x86InsertExtractFormats() { + if !strings.HasPrefix(row, "VEXTRACT") || !strings.Contains(row, "K1,(BX)") { + continue + } + listing := x86InsertExtractGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+strings.Replace(row, "1,K1", "0,K1", 1)+"\nRET\n") + matched := false + for _, line := range strings.Split(string(listing), "\n") { + code := x86InsertExtractListedBytes(line) + ins, _, recognized, err := decodedX86VEXTRACT128Instruction(code, 64) + if recognized { + if err != nil || ins.Args[0].Imm != 255 || ins.Args[2].Reg != "K1" || ins.Args[3].Kind != OpMem { + t.Fatalf("Go masked store identity: %+v/%v", ins, err) + } + matched = true + want[string(ins.Op)+":"+string(ins.Args[1].Reg)]++ + } + } + if !matched { + t.Fatalf("Go listing has no concrete E6NF bytes:\n%s", listing) + } + name := fmt.Sprintf("store%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ destination+0(FP),BX\nMOVQ mask+8(FP),AX\nKMOVQ AX,K1\nVPXORD Z1,Z1,Z1\n%s\nRET\n", name, row) + sigs[name] = FuncSig{Ret: Void, Args: []LLVMType{Ptr, I64}, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}}} + count++ + } + if count != 12 { + t.Fatalf("incomplete EVEX masked memory width/opcode family: %d", count) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if strings.Count(ir, "~{zmm0},~{k1},~{memory}") != count || strings.Contains(ir, "target-features") { + t.Fatal("native instruction bridge lost explicit private clobbers or enabled global ISA") + } + dir := t.TempDir() + input, object := filepath.Join(dir, "store.ll"), filepath.Join(dir, "store.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, llc, "-O2", "-mtriple="+triple, "-filetype=obj", input, "-o", object) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("LLVM22 native E6NF object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + command = exec.CommandContext(ctx, objdump, "-d", object) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("LLVM22 native E6NF disassembly: %v\n%s", err, output) + } + observed := 0 + got := map[string]int{} + for _, line := range strings.Split(string(output), "\n") { + code := x86InsertExtractListedBytes(line) + ins, size, recognized, err := decodedX86VEXTRACT128Instruction(code, 64) + if !recognized { + continue + } + if err != nil || size != len(code) || ins.Args[0].Imm != 255 || ins.Args[1].Reg != "Y0" && ins.Args[1].Reg != "Z0" || ins.Args[2].Reg != "K1" || ins.Args[3].Kind != OpMem { + t.Fatalf("LLVM native write-only instruction identity %s: %+v/%v", line, ins, err) + } + observed++ + got[string(ins.Op)+":"+string(ins.Args[1].Reg)]++ + } + if observed != count || !reflect.DeepEqual(got, want) { + t.Fatalf("%s: native masked stores observed %d want %d\n%s", triple, observed, count, output) + } + } +} + +func x86InsertExtractReservedZeroMaskGoObservation(t *testing.T, source string) { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", object, asm) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatalf("reserved zero-mask source observation: %v\n%s", failure, output) + } + if err != nil { + return // actual concrete source rejection; never an unknown tool failure + } + code := []byte{0x62, 0xf3, 0x7d, 0xc8, 0x19, 0xc1, 0} + decode := decodedX86VEXTRACT128Instruction + if strings.Contains(source, "VINSERT") { + code = []byte{0x62, 0xf3, 0xfd, 0xc8, 0x3a, 0xc8, 0} + decode = decodedX86EVEXPackedInsertInstruction + } + var listing strings.Builder + for _, value := range code { + fmt.Fprintf(&listing, "%02x ", value) + } + if !bytes.Contains(output, []byte(strings.TrimSpace(listing.String()))) { + t.Fatalf("accepted old Go reserved bytes changed:\n%s", output) + } + if _, _, recognized, err := decode(code, 64); !recognized || err == nil { + t.Fatal("reserved EVEX.z with aaa=0 acquired valid physical meaning") + } +} + +func TestCrossLinuxRuntimeMatrixX86InsertExtractOriginalGoPSOracle(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + checks.WriteString(`package oracle +import ("encoding/binary";"testing") +func TestOriginalPSBits(t *testing.T) { + a:=[4]uint32{0x7f812345,0x80000000,0xffc45678,0x01234567} + b:=[4]uint32{0x89abcdef,0x7fc11111,0x7f800000,1} +`) + for index, immediate := range []int{0, 16, 32, 48, 64, 128, 192, 1, 2, 4, 8, 15, 255} { + name := fmt.Sprintf("insert%d", index) + fmt.Fprintf(&declarations, "func %s(out,a,b *[4]uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVUPS (BX),X0\nMOVUPS (CX),X1\nINSERTPS $%d,X0,X1\nMOVUPS X1,(AX)\nRET\n", name, immediate) + fmt.Fprintf(&checks, "{var out [4]uint32;want:=b;want[%d]=a[%d];for lane:=0;lane<4;lane++{if %d&(1<>4&3, immediate>>6&3, immediate, name, name+" got=%x want=%x") + } + for lane := 0; lane < 4; lane++ { + name := fmt.Sprintf("extract%d", lane) + fmt.Fprintf(&declarations, "func %s(out *[8]byte,a *[4]uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-16\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVUPS (BX),X0\nEXTRACTPS $%d,X0,CX\nMOVQ CX,(AX)\nRET\n", name, lane) + fmt.Fprintf(&checks, "{var out [8]byte;%s(&out,&a);if binary.LittleEndian.Uint64(out[:])!=uint64(a[%d]){t.Fatalf(%q,out)}}\n", name, lane, name+" zero-extension got=%x") + } + declarations.WriteString("func fpFields(a,b,c,d byte) uint32\nfunc fpStores(a,b,c,d byte) uint32\n") + // ABI0 aligns the result area to the AMD64 stack word independently of + // the four byte parameters; the actual Go wrapper reads ret+8(FP). + assembly.WriteString("TEXT ·fpFields(SB),4,$0-12\nPXOR X0,X0\nINSERTPS $0,a+0(FP),X0\nEXTRACTPS $0,X0,ret+8(FP)\nRET\n" + + "TEXT ·fpStores(SB),4,$0-12\nMOVL $0x87654321,CX\nMOVD CX,X1\nEXTRACTPS $0,X1,a+0(FP)\nMOVBLZX a+0(FP),AX\nMOVBLZX b+1(FP),BX\nSHLL $8,BX\nORL BX,AX\nMOVBLZX c+2(FP),BX\nSHLL $16,BX\nORL BX,AX\nMOVBLZX d+3(FP),BX\nSHLL $24,BX\nORL BX,AX\nMOVL AX,ret+8(FP)\nRET\n") + checks.WriteString(`for i:=0;i<256;i++ { + a,b,c,d:=byte(i),byte(i*37),byte(i*59),byte(i*71) + want:=uint32(a)|uint32(b)<<8|uint32(c)<<16|uint32(d)<<24 + if got:=fpFields(a,b,c,d);got!=want{t.Fatalf("original Go FP fields got=%x want=%x",got,want)} + if got:=fpStores(a,b,c,d);got!=0x87654321{t.Fatalf("original Go FP stores got=%x",got)} +} +`) + checks.WriteString("}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + dir := t.TempDir() + for file, contents := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": checks.String()} { + if err := os.WriteFile(filepath.Join(dir, file), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=asmdecl", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go PS original bit-transfer oracle: %v\n%s", err, output) + } +} diff --git a/amd64_insert_extract_native_runtime_linux_amd64_test.go b/amd64_insert_extract_native_runtime_linux_amd64_test.go new file mode 100644 index 000000000..d73e671db --- /dev/null +++ b/amd64_insert_extract_native_runtime_linux_amd64_test.go @@ -0,0 +1,7 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86InsertExtractMaskedNativeMemory(t *testing.T) { + x86InsertExtractMaskedMemoryRuntime(t) +} diff --git a/amd64_insert_extract_native_runtime_test.go b/amd64_insert_extract_native_runtime_test.go new file mode 100644 index 000000000..433ccb756 --- /dev/null +++ b/amd64_insert_extract_native_runtime_test.go @@ -0,0 +1,303 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// This is exactly the subset deliberately excluded by the portable byte +// oracle: E6NF masked memory destinations execute a physical native opcode. +func x86InsertExtractMaskedMemoryRows() []string { + var rows []string + for _, row := range x86InsertExtractFormats() { + if strings.HasPrefix(row, "VEXTRACT") && strings.Contains(row, "(BX)") && strings.Contains(row, "K1") { + rows = append(rows, row) + } + } + return rows +} + +func x86InsertExtractMaskedMemoryAssembly() (string, map[string]FuncSig) { + var assembly strings.Builder + signatures := map[string]FuncSig{} + for index, row := range x86InsertExtractMaskedMemoryRows() { + name := fmt.Sprintf("masked%d", index) + row = strings.Replace(row, "(BX)", "(AX)", 1) + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),R9\nKMOVQ R9,K1\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nSTC\n%s\nSETCS 64(AX)\nRET\n", name, row) + signatures[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + } + return assembly.String(), signatures +} + +func x86InsertExtractMaskedMemoryShape(row string) (chunk, width, lane int) { + chunk, width, lane = 16, 32, 4 + if strings.Contains(row, "32X8") || strings.Contains(row, "64X4") { + chunk = 32 + } + if strings.Contains(row, ",Z") { + width = 64 + } + if strings.Contains(row, "64X") { + lane = 8 + } + return +} + +func TestX86InsertExtractMaskedMemoryRequiredInventory(t *testing.T) { + rows := x86InsertExtractMaskedMemoryRows() + if len(rows) != 12 || len(x86InsertExtractFormats()) != 148 { + t.Fatalf("changed complete/native fixture denominator: %d/%d", len(x86InsertExtractFormats()), len(rows)) + } + file, err := parser.ParseFile(token.NewFileSet(), "amd64_insert_extract_native_runtime_linux_amd64_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var body *ast.BlockStmt + for _, declaration := range file.Decls { + if fn, ok := declaration.(*ast.FuncDecl); ok && fn.Name.Name == "TestCrossLinuxRuntimeMatrixX86InsertExtractMaskedNativeMemory" { + body = fn.Body + } + } + if body == nil || len(body.List) != 1 { + t.Fatal("all masked-memory extract cases require the native execution consumer") + } + statement, ok := body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native extract counterpart cannot return or skip") + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + t.Fatal("native extract counterpart does not consume its full fixture") + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != "x86InsertExtractMaskedMemoryRuntime" { + t.Fatal("native extract counterpart substituted another consumer") + } + assembly, signatures := x86InsertExtractMaskedMemoryAssembly() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + parsed, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir, err := Translate(parsed, Options{Goarch: "amd64", TargetTriple: triple, Sigs: signatures}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "masked-extract.ll", "masked-extract.o", ir) + } + // The complete original Go oracle must compile even on a portable host; + // only its required native counterpart may execute these AVX512 bytes. + dir := x86InsertExtractMaskedOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-c", "-o", filepath.Join(dir, "original"), ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + x86FloatCompareCaptureCommand(t, command) + t.Log("148 original formats preserved: all 12 excluded native memory-mask cases now have stock Go objects, 3 LLVM target objects and a mandatory Linux AMD64 runtime counterpart (not executed here)") +} + +// Intel Vol2 KMOVQ r64,k requires AVX512BW; the extract forms additionally +// require F/DQ/VL. Checking F alone cannot authorize this existing bridge. +const x86InsertExtractNativePreflightC = ` +#undef main +__attribute__((target("avx512f,avx512dq,avx512bw,avx512vl"))) +static int original_extract(void) { + uint8_t input[64],output[34]; + for (unsigned i=0;i<64;i++) input[i]=(uint8_t)(i*13+0x27); + for (unsigned i=0;i<34;i++) output[i]=0xc3; + uint64_t mask=1; + __asm__ volatile("vmovdqu64 (%0),%%zmm0; kmovq %1,%%k1; " + "vextracti32x8 $1,%%zmm0,(%2)%{%%k1%}" + :: "r"(input),"r"(mask),"r"(output+1) : "zmm0","k1","memory"); + for (unsigned i=0;i<34;i++) { + uint8_t want=i>=1 && i<=4 ? input[31+i] : 0xc3; + if (output[i]!=want) return 0; + } + return 1; +} +static int extract_required_state(unsigned dq,unsigned bw) { return dq && bw; } +int main(void) { + int failure=compare_preflight_main(); + if (failure) return failure; + unsigned a,b,c,d,dq=0,bw=0; + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) { dq=(b>>17)&1; bw=(b>>30)&1; } + printf("masked extract AVX512DQ=%u AVX512BW=%u\n",dq,bw); + fflush(stdout); + if (!extract_required_state(dq,bw)) return 81; + return original_extract() ? 0 : 82; +} +` + +func x86InsertExtractMaskedMemoryRuntime(t *testing.T) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + dir := t.TempDir() + source, binary := filepath.Join(dir, "extract-preflight.c"), filepath.Join(dir, "extract-preflight") + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + x86InsertExtractNativePreflightC + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + t.Logf("original native extract preflight: %s", execution.run(t, binary)) + original := x86InsertExtractMaskedOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-count=1", "-run", "^TestOriginalMaskedExtract$", "-v", ".") + command.Dir = original + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + t.Logf("original Go native masked extract: %s", x86FloatCompareCaptureCommand(t, command)) + assembly, signatures := x86InsertExtractMaskedMemoryAssembly() + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: signatures}) + if err != nil { + t.Fatal(err) + } + var declarations, calls strings.Builder + for index, row := range x86InsertExtractMaskedMemoryRows() { + chunk, width, lane := x86InsertExtractMaskedMemoryShape(row) + fmt.Fprintf(&declarations, "extern void masked%d(void*,const void*,const void*,uint64_t);\n", index) + fmt.Fprintf(&calls, " if (check(masked%d,%d,%d,%d)) return %d;\n", index, chunk, width, lane, index+1) + } + execution.compileAndRunLLVM(t, "masked-native-extract", ir, x86InsertExtractMaskedMemoryC+declarations.String()+"int main(void) {\n"+calls.String()+" return 0;\n}\n", "-O2") + t.Log("all 12 native masked-memory extract programs x40 byte/MXCSR/CF/input-output-canary cases executed by original Go and translated LLVM") +} + +func TestX86InsertExtractNativeOpcodePreflightContract(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, binary := filepath.Join(dir, "extract-capabilities.c"), filepath.Join(dir, "extract-capabilities") + // Compile the actual original-opcode preflight, but execute only its exact + // extra capability predicate here. Synthetic bits never permit ISA execution. + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + + strings.Replace(x86InsertExtractNativePreflightC, "int main(void)", "int actual_extract_main(void)", 1) + ` +int main(void) { + if (!extract_required_state(1,1) || extract_required_state(0,1) || + extract_required_state(1,0) || extract_required_state(0,0)) return 1; + return 0; +} +` + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + execution.run(t, binary) +} + +func x86InsertExtractMaskedOriginalPackage(t *testing.T) string { + t.Helper() + dir := t.TempDir() + assembly, _ := x86InsertExtractMaskedMemoryAssembly() + assembly = strings.ReplaceAll(assembly, "TEXT masked", "TEXT ·masked") + assembly += "TEXT ·setCSR(SB),4,$0-4\nLDMXCSR csr+0(FP)\nRET\nTEXT ·getCSR(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n" + var declarations, cases strings.Builder + for index, row := range x86InsertExtractMaskedMemoryRows() { + chunk, width, lane := x86InsertExtractMaskedMemoryShape(row) + fmt.Fprintf(&declarations, "func masked%d(out *[65]byte,a,b *[64]byte,mask uint64)\n", index) + fmt.Fprintf(&cases, " {masked%d,%d,%d,%d},\n", index, chunk, width, lane) + } + for name, contents := range map[string]string{ + "go.mod": "module nativemaskedextract\n\ngo 1.20\n", "original_amd64.s": assembly, + "original_test.go": "package original\nimport (\"testing\"; \"runtime\")\n" + declarations.String() + "func setCSR(csr uint32)\nfunc getCSR() uint32\n" + strings.Replace(x86InsertExtractMaskedMemoryGo, "REQUIRED_CASES", cases.String(), 1), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + return dir +} + +const x86InsertExtractMaskedMemoryGo = ` +func TestOriginalMaskedExtract(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + defer setCSR(getCSR()) + cases:=[]struct{fn func(*[65]byte,*[64]byte,*[64]byte,uint64); chunk,width,lane int}{ +REQUIRED_CASES + } + masks:=[]uint64{0,1,0xaaaaaaaaaaaaaaaa,0xffffffffffffffff,0xffff000000000000} + for index,fixture:=range cases { + for vector:=0;vector<8;vector++ { + for _,mask:=range masks { + var a,b,out [67]byte + for i:=range a { + a[i]=0xa7 + b[i]=0xb9 + out[i]=0xc3 + } + for i:=0;i<64;i++ { + a[i+1]=byte((i*31+vector*17)^0x81) + b[i+1]=byte((i*13+vector*29)^0x27) + } + aCopy,bCopy:=a,b + want:=out + offset:=(255&(fixture.width/fixture.chunk-1))*fixture.chunk + for i:=0;i +#include +#include +typedef void(*transfer)(void*,const void*,const void*,uint64_t); +static int check(transfer fn,unsigned chunk,unsigned width,unsigned laneBytes) { + const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX,UINT64_C(0xffff000000000000)}; + for (unsigned vector=0;vector<8;vector++) { + for (unsigned m=0;m<5;m++) { + uint8_t as[67],bs[67],out[67],want[67],aCopy[67],bCopy[67]; + uint8_t *a=as+1,*b=bs+1; + memset(as,0xa7,67); memset(bs,0xb9,67); memset(out,0xc3,67); + for (unsigned i=0;i<64;i++) { + a[i]=(i*31+vector*17)^0x81; + b[i]=(i*13+vector*29)^0x27; + } + memcpy(aCopy,as,67); memcpy(bCopy,bs,67); memcpy(want,out,67); + unsigned offset=(255&(width/chunk-1))*chunk; + for (unsigned i=0;i +#include +extern uint32_t fields(uint8_t,uint8_t,uint8_t,uint8_t); +extern uint32_t stores(uint8_t,uint8_t,uint8_t,uint8_t); +int main(void) { + for(unsigned i=0;i<256;i++) { + uint8_t a=i,b=i*37,c=i*59,d=i*71; + uint32_t want=(uint32_t)a|((uint32_t)b<<8)|((uint32_t)c<<16)|((uint32_t)d<<24); + uint32_t got=fields(a,b,c,d); + if(got!=want){fprintf(stderr,"independent FP fields got=%x want=%x\n",got,want);return 1;} + got=stores(a,b,c,d); + if(got!=UINT32_C(0x87654321)){fprintf(stderr,"independent FP stores got=%x\n",got);return 2;} + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "insert-extract-independent-fp", triple, ir, reference, prefix) +} + +func x86InsertExtractNumericalRows() []string { + rows := x86InsertExtractFormats() + // Actual source/destination aliases plus every imm8 selector bit for the + // PS forms. Chunk immediates ignore their high bits, independently tested. + for _, row := range x86InsertExtractFormats() { + if strings.Contains(row, "(BX)") || strings.Contains(row, "K1") || strings.Contains(row, "PS ") { + continue + } + if strings.HasPrefix(row, "VINSERT") { + rows = append(rows, strings.TrimSuffix(row, "2")+"1", strings.TrimSuffix(row, "2")+"0") + } else { + rows = append(rows, strings.TrimSuffix(row, "0")+"1") + } + rows = append(rows, strings.Replace(row, "$255", "$0", 1)) + } + for _, imm := range []int{0, 16, 32, 48, 64, 128, 192, 1, 2, 4, 8, 15, 255} { + rows = append(rows, fmt.Sprintf("VINSERTPS $%d,X0,X1,X2", imm), fmt.Sprintf("VINSERTPS $%d,(BX),X1,X1", imm), fmt.Sprintf("VEXTRACTPS $%d,X1,R8", imm)) + rows = append(rows, fmt.Sprintf("INSERTPS $%d,X0,X1", imm), fmt.Sprintf("INSERTPS $%d,(BX),X1", imm)) + } + for lane := 0; lane < 4; lane++ { + rows = append(rows, fmt.Sprintf("EXTRACTPS $%d,X1,R8", lane)) + } + rows = append(rows, "INSERTPS $128,X1,X1", "INSERTPS $192,X1,X0", "INSERTPS $64,X0,X0") + return rows +} + +func x86InsertExtractRuntimeCheck(name string, parameters ...int) string { + var call strings.Builder + fmt.Fprintf(&call, "if(check(%s", name) + for _, parameter := range parameters { + fmt.Fprintf(&call, ",%d", parameter) + } + // Preserve the case identity in text; a process exit status must never + // wrap a large generated index to zero on Unix. + fmt.Fprintf(&call, ")) { fprintf(stderr, %q); return 1; }\n", name+" numerical/canary failure\n") + return call.String() +} + +func TestCrossLinuxRuntimeMatrixX86InsertExtractByteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + rows := x86InsertExtractNumericalRows() + for index, row := range rows { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + op := strings.Fields(row)[0] + insert, scalar, memory, masked := strings.Contains(op, "INSERT"), strings.HasSuffix(op, "PS"), strings.Contains(row, "(BX)"), strings.Contains(row, "K1") + if !insert && memory && masked { + continue // physical native E6NF objects are checked separately; AVX512 execution is not available here + } + chunk, width, laneBytes := 16, 32, 4 + if strings.Contains(op, "X8") || strings.Contains(op, "64X4") { + chunk = 32 + } + if strings.Contains(op, "64X") { + laneBytes = 8 + } + if strings.Contains(row, ",Z") { + width = 64 + } + if scalar { + chunk, width = 4, 16 + } + destination := ins.Args[len(ins.Args)-1] + store := "" + if destination.Kind == OpReg { + if scalar && !insert { + // AX is the output address, so choose a non-address GP result. + row = strings.TrimSuffix(row, string(destination.Reg)) + "R8" + store = "MOVQ R8,(AX)\n" + } else { + store = fmt.Sprintf("VMOVUPS %s,(AX)\n", x86CallerVectorRoot(destination.Reg)) + } + } else { + row = strings.Replace(row, "(BX)", "(AX)", 1) + } + name := fmt.Sprintf("transfer%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),R9\nKMOVQ R9,K1\nVMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nSTC\n%s\n%sSETCS 64(AX)\nRET\n", name, row, store) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t);\n", name) + checks.WriteString(x86InsertExtractRuntimeCheck(name, + x86FloatAddSubBoolInt(insert), x86FloatAddSubBoolInt(scalar), chunk, width, laneBytes, int(ins.Args[0].Imm), + x86FloatAddSubBoolInt(memory), x86FloatAddSubBoolInt(masked), x86FloatAddSubBoolInt(strings.Contains(op, ".Z")), x86FloatAddSubBoolInt(destination.Kind == OpReg), x86FloatAddSubBoolInt(!strings.HasPrefix(op, "V")), + x86FloatAddSubBoolInt(ins.Args[1].Reg == "X1"), x86FloatAddSubBoolInt(!strings.HasPrefix(op, "V") && destination.Reg == "X0"))) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const reference = `#include +#include +#include +typedef void(*transfer)(void*,const void*,const void*,uint64_t); +__attribute__((noinline)) static int check(transfer fn,int insert,int scalar,int chunk,int width,int laneBytes,unsigned imm,int memory,int masked,int zero,int reg,int legacy,int sourceB,int baseA) { + const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_C(0xffffffffffffffff),UINT64_C(0xffff000000000000)}; + for(unsigned vector=0;vector<8;vector++)for(unsigned m=0;m<5;m++) { + uint8_t as[67],bs[67],out[67],want[64],aCopy[67],bCopy[67];uint8_t *a=as+1,*b=bs+1; + memset(as,0xa7,67);memset(bs,0xb9,67);memset(out,0xc3,67); + for(unsigned i=0;i<64;i++){a[i]=(i*31+vector*17)^0x81;b[i]=(i*13+vector*29)^0x27;} + memcpy(aCopy,as,67);memcpy(bCopy,bs,67);memset(want,reg&&!(scalar&&!insert)?0:0xc3,64); + const uint8_t *base=baseA?a:b,*insertSource=sourceB?b:a; + if(legacy&&insert)memcpy(want,base,64); + unsigned offset=(imm&(width/chunk-1))*chunk; + if(insert) { + memcpy(want,base,width); + if(scalar){offset=((imm>>4)&3)*4;memcpy(want+offset,insertSource+(memory?0:((imm>>6)&3)*4),4); + for(unsigned lane=0;lane<4;lane++)if(imm&(1u< +#include +#include +#include +#include +#include +#include +typedef void(*transfer)(void*,const void*); +static int fault(transfer fn,const void *input,unsigned width){ + int statuses[2]; + for(unsigned oracle=0;oracle<2;oracle++){ + pid_t child=fork();if(child<0)return 1; + if(child==0){ + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + + uint8_t out[64]; + if(oracle){ + volatile const uint8_t *p=input; + volatile uint8_t sink=0; + for(unsigned i=0;i +static void transfer255(void) {} +static int check(void (*fn)(void), ...) { return 1; } +int main(void) { +` + x86InsertExtractRuntimeCheck("transfer255", make([]int, 13)...) + "return 0;\n}\n" + requireX86GeneratedOracleFailure(t, program, "transfer255 numerical/canary failure") +} diff --git a/amd64_insert_extract_sdk_source_test.go b/amd64_insert_extract_sdk_source_test.go new file mode 100644 index 000000000..c37f87a87 --- /dev/null +++ b/amd64_insert_extract_sdk_source_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + "time" +) + +func TestX86InsertExtractActualSDKTablesAreMandatory(t *testing.T) { + const rootName = "TestX86InsertExtractEveryGoOpcodeAttribute" + for _, name := range []string{"avx_optabs.go", "asm6.go"} { + t.Run(name, func(t *testing.T) { + // Execute the actual attribute root from this SDK's test binary. + // A partial fixture GOROOT supplies no replacement encoder evidence. + fixture := t.TempDir() + directory := filepath.Join(fixture, "src", "cmd", "internal", "obj", "x86") + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + if name == "asm6.go" { + original, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86", "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(directory, "avx_optabs.go"), original, 0600); err != nil { + t.Fatal(err) + } + } + missing := filepath.Join(directory, name) + if _, err := os.ReadFile(missing); !os.IsNotExist(err) { + t.Fatalf("missing-table fixture is not absent: %v", err) + } + + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, 30*time.Second) + defer cancel() + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^"+rootName+"$", "-test.count=1", "-test.timeout=20s") + for _, item := range os.Environ() { + if !strings.HasPrefix(item, "GOROOT=") { + command.Env = append(command.Env, item) + } + } + command.Env = append(command.Env, "GOROOT="+fixture) + output, err := runtimeOracleCombinedOutput(ctx, command) + exit, plainExit := err.(*exec.ExitError) + if ctx.Err() != nil || !plainExit || exit.ProcessState == nil || + !exit.ProcessState.Exited() || exit.ExitCode() != 1 { + t.Fatalf("actual SDK-table root did not fail with plain exit 1: %v (context %v)\n%s", err, ctx.Err(), output) + } + + // Bind the complete failure to this one root, its real source file, + // and the exact missing table. Panics, cleanup errors, extra failures, + // unrelated diagnostics and a PASS from version-return are not evidence. + _, missingErr := os.ReadFile(missing) + if !os.IsNotExist(missingErr) { + t.Fatalf("missing-table fixture changed during the child: %v", missingErr) + } + if !x86InsertExtractMissingTableOutput(output, rootName, missingErr.Error(), testing.CoverMode() != "") { + t.Fatalf("unbound actual SDK-table failure (want %s):\n%s", missing, output) + } + t.Logf("actual %s child: plain exit 1, exact %s diagnostic\n%s", runtime.Version(), name, output) + }) + } +} + +func x86InsertExtractMissingTableOutput(output []byte, root, diagnostic string, covered bool) bool { + pattern := fmt.Sprintf(`\A--- FAIL: %s \([0-9]+\.[0-9]+s\)\n amd64_insert_extract_forms_test\.go:[0-9]+: %s\nFAIL\n`, + regexp.QuoteMeta(root), regexp.QuoteMeta(diagnostic)) + if covered { + // The child is this same instrumented test binary. Require Go's one + // coverage trailer only in that role; never strip arbitrary output. + // Go 1.20's coverage writer also prints this exact package prefix. + pattern += `(?:\tgithub\.com/xgo-dev/plan9asm\t)?coverage: (?:(?:[0-9]|[1-9][0-9])\.[0-9]|100\.0)% of statements\n` + } + return regexp.MustCompile(pattern + `\z`).Match(output) +} + +func TestX86InsertExtractMissingTableOutputGrammar(t *testing.T) { + const root, diagnostic = "TestActualTable", "open exact/asm6.go: missing table" + base := "--- FAIL: " + root + " (0.01s)\n amd64_insert_extract_forms_test.go:66: " + diagnostic + "\nFAIL\n" + const trailer = "coverage: 0.2% of statements\n" + for _, covered := range []bool{false, true} { + valid := base + if covered { + valid += trailer + } + if !x86InsertExtractMissingTableOutput([]byte(valid), root, diagnostic, covered) { + t.Fatalf("exact original failure rejected (coverage=%v)", covered) + } + for _, invalid := range []string{ + valid + "PASS\n", "panic: unrelated\n" + valid, valid + trailer, + strings.Replace(valid, "exact/asm6.go", "other/asm6.go", 1), + strings.Replace(valid, root, "TestAnotherRoot", 1), + } { + if x86InsertExtractMissingTableOutput([]byte(invalid), root, diagnostic, covered) { + t.Fatalf("unbound child failure accepted (coverage=%v): %q", covered, invalid) + } + } + } + if x86InsertExtractMissingTableOutput([]byte(base), root, diagnostic, true) || + x86InsertExtractMissingTableOutput([]byte(base+trailer), root, diagnostic, false) { + t.Fatal("child borrowed the wrong coverage instrumentation role") + } + if !x86InsertExtractMissingTableOutput([]byte(base+"\tgithub.com/xgo-dev/plan9asm\t"+trailer), root, diagnostic, true) || + x86InsertExtractMissingTableOutput([]byte(base+"\tother/package\t"+trailer), root, diagnostic, true) { + t.Fatal("historical coverage trailer lost its exact package identity") + } +} diff --git a/amd64_lea_dispatch_test.go b/amd64_lea_dispatch_test.go new file mode 100644 index 000000000..8c160d7e8 --- /dev/null +++ b/amd64_lea_dispatch_test.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86LEADispatchBindsCompleteWidths(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range []Op{"LEAW", "LEAL", "LEAQ"} { + for _, instructionOp := range []Op{"", op, Op(strings.ToLower(string(op)))} { + t.Run(fmt.Sprintf("%s/%s/op=%q", arch, op, instructionOp), func(t *testing.T) { + instruction := Instr{Op: instructionOp, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: BX, Index: CX, Scale: 2, Off: 8}}, + {Kind: OpReg, Reg: AX}, + }} + registered := instruction + registered.Op = op + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{registered}}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.String() + ok, terminated, err := ctx.lowerArith(op, instruction) + if arch == "386" && op == "LEAQ" { + if !ok || terminated || err == nil || output.String() != before { + t.Fatalf("386 LEAQ must reject before emission: (%v, %v, %v)", ok, terminated, err) + } + return + } + if !ok || terminated || err != nil { + t.Fatalf("LEA dispatch = (%v, %v, %v)", ok, terminated, err) + } + if instruction.Op != instructionOp { + t.Fatal("dispatch mutated the caller's original instruction identity") + } + if op == "LEAW" && !strings.Contains(output.String()[len(before):], "to i16") || + op == "LEAL" && !strings.Contains(output.String()[len(before):], "to i32") { + t.Fatal("LEA used a different destination width than the dispatch") + } + }) + } + } + } +} + +func TestX86LEADispatchRejectsContradictoryIdentityBeforeEmission(t *testing.T) { + for _, dispatch := range []Op{"LEAW", "LEAL", "LEAQ"} { + for _, source := range []Op{"LEAW", "LEAL", "LEAQ", "LEAL.Z", "MOVQ", "UNKNOWN"} { + if source == dispatch { + continue + } + t.Run(fmt.Sprintf("%s/source=%s", dispatch, source), func(t *testing.T) { + instruction := Instr{Op: source, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, + {Kind: OpReg, Reg: AX}, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + before := output.String() + ok, terminated, err := ctx.lowerArith(dispatch, instruction) + if !ok || terminated || err == nil { + t.Fatalf("accepted an unknown/contradictory LEA identity: (%v, %v, %v)", ok, terminated, err) + } + if output.String() != before { + t.Fatal("contradictory LEA emitted address loads/truncation/stores before rejection") + } + }) + } + } +} + +func TestX86LEADispatchUnknownOpcodeCannotConsumeKnownInstruction(t *testing.T) { + for _, dispatch := range []Op{"UNKNOWN", "LEAL.Z", "leal"} { + for _, source := range []Op{"", "LEAW", "LEAL", "LEAQ", "UNKNOWN"} { + t.Run(fmt.Sprintf("%s/source=%q", dispatch, source), func(t *testing.T) { + instruction := Instr{Op: source, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, + {Kind: OpReg, Reg: AX}, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + before := output.String() + ok, terminated, _ := ctx.lowerArith(dispatch, instruction) + if ok || terminated || output.String() != before { + t.Fatal("unknown arithmetic dispatch was treated as an emitted LEA") + } + }) + } + } +} diff --git a/amd64_lea_effects.go b/amd64_lea_effects.go new file mode 100644 index 000000000..55e484c12 --- /dev/null +++ b/amd64_lea_effects.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's ALEAW/ALEAL/ALEAQ rows all use ym_rl (Ym, Yrl). Address +// calculation and destination width are independent: W preserves the upper +// register, L zero-extends, and Q is available only in 64-bit mode. +func x86LEAWidth(op Op, arch string) int64 { + switch op { + case "LEAW": + return 2 + case "LEAL": + return 4 + case "LEAQ": + if arch != "386" { + return 8 + } + } + return 0 +} + +func x86LEAOperandWidth(ins Instr, arch string) (int64, error) { + width := x86LEAWidth(ins.Op, arch) + if width == 0 { + return 0, fmt.Errorf("%s %s has no Go LEA width: %q", arch, ins.Op, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[1].Reg, arch) { + return 0, fmt.Errorf("%s %s expects Ym address and Yrl destination: %q", arch, ins.Op, ins.Raw) + } + if bits := ins.x86AddressBits; bits != 0 && (ins.Args[0].Kind != OpMem || + !(arch == "386" && bits == 16 || arch != "386" && bits == 32)) { + return 0, fmt.Errorf("%w: %s %s has an unbound physical address size %d", ErrProbeNeedsContext, arch, ins.Op, bits) + } + switch ins.Args[0].Kind { + case OpMem: + memory := ins.Args[0].Mem + if err := x86SymbolIndexedMemoryError(memory, arch); err != nil { + return 0, err + } + if arch == "386" && strings.HasSuffix(memory.Sym, "(SB)") { + memory.Base = "" // Go's named-memory encoder discards Reg. + } + if !x86MemoryRegistersValidForArch(memory, arch) { + return 0, fmt.Errorf("%s %s source uses an out-of-range address register: %q", arch, ins.Op, ins.Raw) + } + if memory.Index != "" && (memory.Index == SP || + memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8) { + return 0, fmt.Errorf("%s %s source has an unencodable SIB index/scale: %q", arch, ins.Op, ins.Raw) + } + // asm6.go:asmandsz truncates an AMD64 LEAL displacement to int32. + // Other AMD64 widths require signed int32; 386 also accepts uint32. + signed32 := memory.Off == int64(int32(memory.Off)) + unsigned32 := memory.Off == int64(uint32(memory.Off)) + if !(arch != "386" && width == 4) && !signed32 && !(arch == "386" && unsigned32) { + return 0, fmt.Errorf("%s %s displacement is outside Go's encoded address range: %q", arch, ins.Op, ins.Raw) + } + case OpSym: + if !strings.HasSuffix(strings.TrimSpace(ins.Args[0].Sym), "(SB)") || strings.HasPrefix(strings.TrimSpace(ins.Args[0].Sym), "$") { + return 0, fmt.Errorf("%s %s source is not a Go SB memory expression: %q", arch, ins.Op, ins.Raw) + } + fallthrough + case OpFP: + // Bound FP storage and symbol identity remain the actual lowerer's + // responsibility; this row does not establish pointer provenance. + default: + return 0, fmt.Errorf("%s %s expects Ym address and Yrl destination: %q", arch, ins.Op, ins.Raw) + } + return width, nil +} + +func x86CallerLEAStep(ins *Instr, state *x86SourceStackState) bool { + width, err := x86LEAOperandWidth(*ins, "amd64") + if err != nil || ins.Args[0].Kind != OpMem || ins.x86AddressBits != 0 { + return false + } + memory := ins.Args[0].Mem + destination, full := x86CallerGP(ins.Args[1].Reg) + if !full || destination != ins.Args[1].Reg || destination == SP || destination == BP || + memory.Sym != "" || memory.Segment != "" { + return false + } + base, _ := x86CallerGP(memory.Base) + if pointer, present := state.callers[base]; present { + // Only the full, independently bound caller address survives. A + // narrowed address must not become numerical initialization; a + // variable offset cannot borrow this constant-offset certificate. + if width != 8 || memory.Base != base || memory.Index != "" || + memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit { + return false + } + var valid bool + pointer, valid = x86CallerPointerShift(pointer, memory.Off) + if !valid { + return false + } + state.callers[destination] = pointer + delete(state.known, destination) + return true + } + // Arithmetic modulo the result width depends only on those low input + // bytes. Multiplication/addition carry cannot invent any higher input + // dependency. NumericRead still rejects *all* pointer-derived inputs, + // including an index or a source/destination alias. + for _, register := range []Reg{memory.Base, memory.Index} { + if register != "" && !x86CallerNumericRead(Operand{Kind: OpReg, Reg: register}, width, state) { + return false + } + } + return x86CallerNumericWrite(destination, width, state) +} diff --git a/amd64_lea_go_oracle_receipt_test.go b/amd64_lea_go_oracle_receipt_test.go new file mode 100644 index 000000000..37e9b9ae9 --- /dev/null +++ b/amd64_lea_go_oracle_receipt_test.go @@ -0,0 +1,204 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// A test-only observation of this invocation, not a frame or caller capability. +type x86LEAGoObjectReceipt struct { + Target, GoVersion string + Source, Driver, Object [sha256.Size]byte +} + +// These existing LEAW/L/Q source forms also belong to the historical SDK. +// The generic current-table helper remains unchanged for other families. +func requireLEANativeGoObject(t *testing.T, arch, source string, wantSuccess bool) { + t.Helper() + if wantSuccess { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + x86LEAActualTargetGoObject(t, arch, triple, source) + } else { + x86LEAActualSDKDriver(t) + goAssemblerOracleOutput(t, arch, source, "", false) + } +} + +func x86LEAActualSDKDriver(t *testing.T) (string, [sha256.Size]byte) { + t.Helper() + driver, err := exec.LookPath("go") + if err != nil { + t.Fatal(err) + } + driver, err = filepath.EvalSymlinks(driver) + if err != nil { + t.Fatal(err) + } + root, err := filepath.EvalSymlinks(runtime.GOROOT()) + if err != nil || driver != x86LEASDKDriverPath(root, runtime.GOOS) { + t.Fatalf("LEA oracle must invoke this SDK's driver: %s: %v", driver, err) + } + driverBytes, err := os.ReadFile(driver) + if err != nil { + t.Fatal(err) + } + return driver, sha256.Sum256(driverBytes) +} + +func x86LEASDKDriverPath(root, hostGOOS string) string { + filename := "go" + if hostGOOS == "windows" { + filename = "go.exe" + } + return filepath.Join(root, "bin", filename) +} + +func TestX86LEAActualSDKDriverPathUsesHostToolSuffix(t *testing.T) { + root := filepath.Join("selected-sdk", "root") + for _, test := range []struct { + goos, filename string + }{ + {"windows", "go.exe"}, + {"darwin", "go"}, + {"linux", "go"}, + } { + t.Run(test.goos, func(t *testing.T) { + got := x86LEASDKDriverPath(root, test.goos) + want := filepath.Join(root, "bin", test.filename) + if got != want { + t.Fatalf("actual SDK driver for host %s = %q, want %q", test.goos, got, want) + } + }) + } +} + +func x86LEAReadObjectReceipt(t *testing.T, arch, goos, source, input, driver string, driverHash [sha256.Size]byte, object string) *x86LEAGoObjectReceipt { + t.Helper() + info, err := os.Lstat(object) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatalf("actual LEA oracle lacks a regular nonempty object: %v", err) + } + data, err := os.ReadFile(object) + if err != nil { + t.Fatal(err) + } + header := strings.Fields(string(bytes.SplitN(data, []byte("\n"), 2)[0])) + if len(header) < 5 || header[0] != "go" || header[1] != "object" || + header[2] != goos || header[3] != arch || header[4] != runtime.Version() { + t.Fatalf("LEA object does not bind the actual target/SDK: %q", header) + } + inputBytes, err := os.ReadFile(input) + if err != nil || !bytes.Equal(inputBytes, []byte(source)) { + t.Fatalf("actual LEA source changed during the tool invocation: %v", err) + } + driverBytes, err := os.ReadFile(driver) + if err != nil || sha256.Sum256(driverBytes) != driverHash { + t.Fatalf("actual LEA driver changed during the tool invocation: %v", err) + } + receipt := &x86LEAGoObjectReceipt{ + Target: goos + "/" + arch, GoVersion: header[4], + Source: sha256.Sum256(inputBytes), Driver: driverHash, + Object: sha256.Sum256(data), + } + t.Logf("actual LEA source/object receipt: SDK=%s target=%s source=%x driver=%x object=%x", + receipt.GoVersion, receipt.Target, receipt.Source, receipt.Driver, receipt.Object) + return receipt +} + +func TestX86LEAActualGoOracleCannotSilentlyReturnBeforeTool(t *testing.T) { + for name, filename := range map[string]string{ + "x86LEAActualTargetGoObject": "amd64_go_caller_lea_test.go", + "requireLEANativeGoObject": "amd64_lea_go_oracle_receipt_test.go", + } { + file, err := parser.ParseFile(token.NewFileSet(), filename, nil, 0) + if err != nil { + t.Fatal(err) + } + ast.Inspect(file, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if ok { + if identifier, ok := call.Fun.(*ast.Ident); ok && identifier.Name == "requireX86GoAssemblerResult" { + t.Errorf("%s uses a current-table helper instead of its actual applicable LEA tool oracle", filename) + } + } + return true + }) + found := false + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok || function.Name.Name != name { + continue + } + found = true + ast.Inspect(function.Body, func(node ast.Node) bool { + if statement, ok := node.(*ast.ReturnStmt); ok { + missing := len(statement.Results) == 0 + if len(statement.Results) == 1 { + value, ok := statement.Results[0].(*ast.Ident) + missing = ok && value.Name == "nil" + } + if missing { + t.Errorf("%s has a silent return instead of its actual tool receipt", name) + } + } + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if identifier, ok := call.Fun.(*ast.Ident); ok && + (identifier.Name == "goToolchainAtLeast" || identifier.Name == "currentGoMinorAtLeast") { + t.Errorf("%s silently gates its actual tool invocation by build version", name) + } + if selector, ok := call.Fun.(*ast.SelectorExpr); ok && + (selector.Sel.Name == "Skip" || selector.Sel.Name == "Skipf" || selector.Sel.Name == "SkipNow") { + t.Errorf("%s cannot skip an applicable LEA source oracle", name) + } + return true + }) + } + if !found { + t.Fatalf("actual LEA oracle %s is missing", name) + } + } +} + +func TestX86LEAActualGoOracleRequiresToolAndObjectReceipt(t *testing.T) { + targets := map[string]string{ + "x86_64-apple-darwin": "darwin/amd64", + "x86_64-unknown-linux-gnu": "linux/amd64", + "x86_64-pc-windows-msvc": "windows/amd64", + "i386-unknown-linux-gnu": "linux/386", + "i686-pc-windows-msvc": "windows/386", + } + for _, target := range x86RawReturnTargets { + for _, suffix := range []string{"W", "L", "Q"} { + if x86LEAWidth(Op("LEA"+suffix), target.arch) == 0 { + continue // No LEAQ physical encoding in 386 mode. + } + t.Run(target.triple+"/"+suffix, func(t *testing.T) { + source := "TEXT lea(SB),4,$0-0\nLEA" + suffix + " 19(AX)(BX*4),AX\nRET\n" + receipt := x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + if receipt == nil { + t.Fatal("actual LEA oracle omitted its selected SDK tool/object invocation") + } + zero := [sha256.Size]byte{} + if receipt.GoVersion != runtime.Version() || receipt.Source != sha256.Sum256([]byte(source)) || + receipt.Driver == zero || receipt.Object == zero || receipt.Target != targets[target.triple] { + t.Fatalf("actual LEA receipt is incomplete or misbound: %+v", receipt) + } + }) + } + } +} diff --git a/amd64_legacy_float_memory_context_test.go b/amd64_legacy_float_memory_context_test.go new file mode 100644 index 000000000..493948449 --- /dev/null +++ b/amd64_legacy_float_memory_context_test.go @@ -0,0 +1,212 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86LegacyPackedFloatingNativeMemoryUnboundSourcesBeforeEmission(t *testing.T) { + for _, op := range []Op{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} { + for _, operand := range []Operand{ + {Kind: OpMem, Mem: MemRef{Base: SP, Off: 16}}, + {Kind: OpMem, Mem: MemRef{Base: BP, Off: -32}}, + {Kind: OpMem, Mem: MemRef{Base: AX, Index: BP, Scale: 1}}, + {Kind: OpFP, FPOffset: 0}, + } { + t.Run(string(op)+"/"+operand.String(), func(t *testing.T) { + ins := Instr{Op: op, Args: []Operand{operand, {Kind: OpReg, Reg: "X0"}}} + c, ir := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := ir.String() + form := x86LegacyPackedFloatContextForm(t, ins) + if err := c.lowerLegacyPackedFloatingMemory(ins, form); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unbound native memory source accepted: %v", err) + } + if ir.String() != before { + t.Fatal("unbound source rejection emitted IR side effects") + } + }) + } + for _, addressBits := range []int{16, 32} { + ins := Instr{ + Op: op, Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: "X0"}}, + x86AddressBits: addressBits, + } + c, ir := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := ir.String() + if err := c.lowerLegacyPackedFloatingMemory(ins, x86LegacyPackedFloatContextForm(t, ins)); !errors.Is(err, ErrProbeNeedsContext) || ir.String() != before { + t.Fatalf("address-size %d bypassed native source binding or emitted IR: %v", addressBits, err) + } + } + } +} + +func x86LegacyPackedFloatContextForm(t *testing.T, ins Instr) x86FloatBinaryForm { + t.Helper() + addsub, known, err := x86FloatAddSubProperties(ins, "amd64") + if known { + if err != nil { + t.Fatal(err) + } + return addsub.x86FloatBinaryForm + } + muldiv, known, err := x86FloatMulDivProperties(ins, "amd64") + if !known || err != nil { + t.Fatalf("missing complete typed legacy form: known=%t err=%v", known, err) + } + return muldiv.x86FloatBinaryForm +} + +func TestX86LegacyPackedFloatingNativeMemoryUnboundSourcesFiveTargets(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, op := range []string{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} { + for _, operand := range []string{"16(SP)", "-32(BP)", "(AX)(BP*1)"} { + t.Run(target.triple+"/"+op+"/"+operand, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, op+" "+operand+",X0\nRET") + x86InsertExtractGoListing(t, target.arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + t.Run("Translate", func(t *testing.T) { + _, err := Translate(file, opt) + x86LegacyFloatMemoryRequireContext(t, err) + }) + t.Run("TranslateModuleInContext", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + x86LegacyFloatMemoryRequireContext(t, err) + }) + t.Run("TranslateGoModule", func(t *testing.T) { + translation, err := TranslateGoModule(pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, + ResolveSym: opt.ResolveSym, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + x86LegacyFloatMemoryRequireContext(t, err) + }) + }) + } + } + } +} + +func TestX86LegacyPackedFloatingNativeMemoryUnboundAddressSizeActualGo(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, row := range []struct { + opcode string + prefix string + machine int + }{ + {"ADDPS", "", 0x58}, {"ADDPD", "BYTE $0x66\n", 0x58}, + {"SUBPS", "", 0x5c}, {"SUBPD", "BYTE $0x66\n", 0x5c}, + {"MULPS", "", 0x59}, {"MULPD", "BYTE $0x66\n", 0x59}, + {"DIVPS", "", 0x5e}, {"DIVPD", "BYTE $0x66\n", 0x5e}, + } { + t.Run(target.triple+"/"+row.opcode, func(t *testing.T) { + body := row.prefix + fmt.Sprintf("BYTE $0x67\nBYTE $0x0f\nBYTE $%d\nBYTE $0x03\nRET", row.machine) + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body) + x86InsertExtractGoListing(t, target.arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + wantBits := 32 + if target.arch == "386" { + wantBits = 16 + } + if len(decoded.Instrs) != 3 || decoded.Instrs[0].Op != "TEXT" || + string(decoded.Instrs[1].Op) != row.opcode || decoded.Instrs[1].x86AddressBits != wantBits || + decoded.Instrs[1].x86SourceRaw == nil || decoded.Instrs[2].Op != "RET" { + t.Fatalf("physical address-size/opcode identity lost: %+v", decoded.Instrs) + } + t.Run("Translate", func(t *testing.T) { + _, err := Translate(file, opt) + x86LegacyFloatMemoryRequireContext(t, err) + }) + t.Run("TranslateModuleInContext", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + x86LegacyFloatMemoryRequireContext(t, err) + }) + t.Run("TranslateGoModule", func(t *testing.T) { + translation, err := TranslateGoModule(pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: opt.ResolveSym, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + x86LegacyFloatMemoryRequireContext(t, err) + }) + }) + } + } +} + +func TestX86LegacyPackedFloatingNativeMemoryUnboundAddressSizeProducerIdentity(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for index, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, input := range []struct { + name, body string + raw bool + }{ + {"native_memory", "BYTE $0x0f\nBYTE $@OP\nBYTE $0x03", true}, + {"register_override", "BYTE $0x67\nBYTE $0x0f\nBYTE $@OP\nBYTE $0xc3", true}, + {"forged_raw_comment", opcode + " (AX),X0 // decoded from BYTE $0x67", false}, + } { + t.Run(arch+"/"+opcode+"/"+input.name, func(t *testing.T) { + body := input.body + if input.raw { + body = strings.ReplaceAll(body, "@OP", fmt.Sprint([]int{0x58, 0x5c, 0x59, 0x5e}[index/2])) + if index%2 != 0 { + body = "BYTE $0x66\n" + body + } + } + source := "TEXT f(SB),4,$0-0\n" + body + "\nRET\n" + x86InsertExtractGoListing(t, arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 3 || string(decoded.Instrs[1].Op) != opcode { + t.Fatalf("lost complete default/register form: %+v", decoded.Instrs) + } + instruction := decoded.Instrs[1] + if instruction.x86AddressBits != 0 || (instruction.x86SourceRaw != nil) != input.raw { + t.Fatalf("default/register/forged-comment form acquired an override or raw origin: %+v", instruction) + } + }) + } + } + } +} + +func x86LegacyFloatMemoryRequireContext(t *testing.T, err error) { + t.Helper() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("actual Go accepted source acquired native virtual-stack permission: %v", err) + } +} diff --git a/amd64_legacy_float_memory_runtime_test.go b/amd64_legacy_float_memory_runtime_test.go new file mode 100644 index 000000000..d742b1e4a --- /dev/null +++ b/amd64_legacy_float_memory_runtime_test.go @@ -0,0 +1,139 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86LegacyFloatMemoryProgram(t *testing.T, triple string, dead bool) string { + t.Helper() + var source strings.Builder + sigs := map[string]FuncSig{} + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + name := fmt.Sprintf("memory%d", i) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (BX),Z0\n%s (CX),X0\n", name, opcode) + if !dead { + source.WriteString("VMOVUPS Z0,(AX)\n") + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + return ir +} + +// The independent reference executes each original m128 opcode, rather than +// MOVDQA or a register-only arithmetic substitute. The original physical +// operation supplies the expected typed status, without guessing a vendor, +// emulator, signal mapping, or machine-control setting. The required native +// CI counterpart has the same test root; Rosetta is not native hardware proof. +func TestCrossLinuxRuntimeMatrixX86LegacyPackedFloatingNativeMemory(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var references, table strings.Builder + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + fmt.Fprintf(&references, `extern void memory%d(void*,const void*,const void*); +static __attribute__((noinline)) void reference%d(void *out,const void *a,const void *b) { + uint8_t low[16]; + __asm__ volatile("movups (%%1),%%%%xmm0; %s (%%2),%%%%xmm0; movups %%%%xmm0,(%%0)"::"r"(low),"r"(a),"r"(b):"xmm0","memory"); + memmove(out,a,64);memcpy(out,low,16); +} +`, i, i, strings.ToLower(opcode)) + if i > 0 { + table.WriteString(",") + } + fmt.Fprintf(&table, "{reference%d,memory%d,%d}", i, i, 32+32*(i%2)) + } + main := ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +` + references.String() + ` +typedef void (*operation)(void*,const void*,const void*); +struct pair {operation original,translated;int bits;}; +static const struct pair pairs[]={ ` + table.String() + ` }; +struct outcome {int kind,exit_code,signal,core;}; +static struct outcome fault(operation fn,const void *b) { + struct outcome result={0,0,0,0};pid_t pid=fork();if(pid<0)return result; + if(!pid){struct rlimit limit={0,0};if(setrlimit(RLIMIT_CORE,&limit))_exit(120); + uint8_t a[64] __attribute__((aligned(16)))={0},out[64];unsigned csr=0x1f80; + __asm__ volatile("ldmxcsr %0"::"m"(csr));fn(out,a,b);_exit(0);} + int status;if(waitpid(pid,&status,0)!=pid)return result; + if(WIFSIGNALED(status)){result.kind=2;result.signal=WTERMSIG(status);result.core=!!WCOREDUMP(status);} + else if(WIFEXITED(status)){result.kind=1;result.exit_code=WEXITSTATUS(status);} + return result; +} +static int same_status(struct outcome original,struct outcome translated) { + if(!original.kind||!translated.kind||original.exit_code||translated.exit_code)return 0; + return original.kind==translated.kind&&original.exit_code==translated.exit_code&&original.signal==translated.signal&&original.core==translated.core; +} +int main(void) { + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0}; + const uint64_t edges32[]={0,0x80000000,1,0x00800000,0x3f800001,0xbf800001,0x7f800000,0xff800000,0x7fc12345,0x7f812345}; + const uint64_t edges64[]={0,0x8000000000000000ULL,1,0x0010000000000000ULL,0x3ff0000000000001ULL,0xbff0000000000001ULL,0x7ff0000000000000ULL,0xfff0000000000000ULL,0x7ff8123456789abcULL,0x7ff0123456789abcULL}; + uint8_t bStorage[96] __attribute__((aligned(16)))={0}; + for(int op=0;op<8;op++) { + for(int offset=0;offset<16;offset++) { + struct outcome original=fault(pairs[op].original,bStorage+offset),translated=fault(pairs[op].translated,bStorage+offset); + if(!same_status(original,translated)){fprintf(stderr,"opcode=%d mod16=%d original=%d/%d/%d/%d translated=%d/%d/%d/%d\n",op,offset,original.kind,original.exit_code,original.signal,original.core,translated.kind,translated.exit_code,translated.signal,translated.core);return 1;} + } +#if !DEAD_RESULT + for(int round=0;round<4;round++)for(int control=0;control<6;control++)for(int edge=0;edge<10;edge++)for(int alias=0;alias<3;alias++) { + uint8_t a[64] __attribute__((aligned(16))),b[64] __attribute__((aligned(16))),original[66],translated[66]; + for(int j=0;j<64;j++){a[j]=(uint8_t)(j*13+edge);b[j]=0xb7;} + const uint64_t *edges=pairs[op].bits==32?edges32:edges64;int bytes=pairs[op].bits/8; + for(int lane=0;lane<16/bytes;lane++){memcpy(a+lane*bytes,&edges[(edge+lane)%10],bytes);memcpy(b+lane*bytes,&edges[(edge+lane+3)%10],bytes);} + memset(original,0xc3,66);memset(translated,0xc3,66); + unsigned csr=controls[control]|(round<<13),expected,actual; + uint8_t originalA[64] __attribute__((aligned(16))),originalB[64] __attribute__((aligned(16))),translatedA[64] __attribute__((aligned(16))),translatedB[64] __attribute__((aligned(16))); + memcpy(originalA,a,64);memcpy(originalB,b,64);memcpy(translatedA,a,64);memcpy(translatedB,b,64); + void *originalOut=alias==1?originalA:alias==2?originalB:original+1,*translatedOut=alias==1?translatedA:alias==2?translatedB:translated+1; + __asm__ volatile("ldmxcsr %0"::"m"(csr));pairs[op].original(originalOut,originalA,originalB);__asm__ volatile("stmxcsr %0":"=m"(expected)); + __asm__ volatile("ldmxcsr %0"::"m"(csr));pairs[op].translated(translatedOut,translatedA,translatedB);__asm__ volatile("stmxcsr %0":"=m"(actual)); + if(alias){memcpy(original+1,originalOut,64);memcpy(translated+1,translatedOut,64);} + if(memcmp(original,translated,66)||expected!=actual){fprintf(stderr,"opcode=%d round=%d control=%d edge=%d CSR=%x/%x\n",op,round,control,edge,expected,actual);return 3;} + if(memcmp(originalA,translatedA,64)||memcmp(originalB,translatedB,64)||(!alias&&(memcmp(translatedA,a,64)||memcmp(translatedB,b,64))))return 8; + } +#endif + } + long page=sysconf(_SC_PAGESIZE);if(page<128)return 4; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(region==MAP_FAILED||mprotect(region+page,page,PROT_NONE))return 5; + for(int op=0;op<8;op++)for(int extent=0;extent<=16;extent++) { + const void *pointer=region+page-extent;struct outcome original=fault(pairs[op].original,pointer),translated=fault(pairs[op].translated,pointer); + if(!same_status(original,translated)){fprintf(stderr,"guard opcode=%d extent=%d original=%d/%d/%d/%d translated=%d/%d/%d/%d\n",op,extent,original.kind,original.exit_code,original.signal,original.core,translated.kind,translated.exit_code,translated.signal,translated.core);return 6;} + } + if(munmap(region,2*page))return 7; + unsigned csr=0x1f80;__asm__ volatile("ldmxcsr %0"::"m"(csr));return 0; +} +` + for _, dead := range []bool{false, true} { + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("dead=%t/O2=%t", dead, optimized), func(t *testing.T) { + ir := x86LegacyFloatMemoryProgram(t, triple, dead) + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy-float-memory", triple, ir, fmt.Sprintf("#define DEAD_RESULT %d\n", x86FloatAddSubBoolInt(dead))+main, prefix) + }) + } + } +} diff --git a/amd64_legacy_float_memory_test.go b/amd64_legacy_float_memory_test.go new file mode 100644 index 000000000..9cf4a876f --- /dev/null +++ b/amd64_legacy_float_memory_test.go @@ -0,0 +1,241 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +func x86LegacyFloatMemoryOpcodes() []string { + return []string{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} +} + +func TestX86LegacyPackedFloatingMemoryActualGoOpcodeBytes(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + listing := x86InsertExtractGoListing(t, arch, "TEXT f(SB),4,$0-0\n"+opcode+" (AX),X1\nRET\n") + var actual []byte + for _, line := range strings.Split(string(listing), "\n") { + actual = append(actual, x86InsertExtractListedBytes(line)...) + } + want := []byte{0x0f, []byte{0x58, 0x5c, 0x59, 0x5e}[i/2], 0x08, 0xc3} + if i%2 != 0 { + want = append([]byte{0x66}, want...) + } + if !bytes.Equal(actual, want) { + t.Fatalf("%s/%s actual Go bytes=%x independent SSE=%x", arch, opcode, actual, want) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryRetainsOriginalOpcode(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, arch := range []string{"386", "amd64"} { + for _, source := range []string{"(AX)", "16(AX)(BX*2)", "data(SB)", "(FS)", "(GS)"} { + t.Run(opcode+"/"+arch+"/"+source, func(t *testing.T) { + assembly := "TEXT f(SB),4,$0-0\n" + opcode + " " + source + ",X1\nRET\n" + requireX86GoAssemblerResult(t, arch, assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "asm sideeffect \""+strings.ToLower(opcode)+" ") || strings.Contains(ir, "movdqa") { + t.Fatal("native m128 source lost its exact original observable opcode") + } + }) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryFiveTargetsKeepNativeOpcodes(t *testing.T) { + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM22 llc and llvm-objdump required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + var source strings.Builder + sigs := map[string]FuncSig{} + for i, opcode := range x86LegacyFloatMemoryOpcodes() { + for j, operand := range []string{"(AX)", "16(AX)(BX*2)", "data(SB)", "(FS)", "(GS)"} { + name := fmt.Sprintf("memory_%d_%d", i, j) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s %s,X1\nRET\n", name, opcode, operand) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/O2=%t", triple, optimized), func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + dir := t.TempDir() + input, object := filepath.Join(dir, "memory.ll"), filepath.Join(dir, "memory.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + level := "-O0" + if optimized { + level = "-O2" + } + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, level, "-mtriple="+triple, "-filetype=obj", input, "-o", object)) + cancel() + if err != nil { + t.Fatalf("native memory object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + output, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, objdump, "-d", object)) + cancel() + if err != nil { + t.Fatalf("native memory disassembly: %v\n%s", err, output) + } + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + pattern := regexp.MustCompile(`(?m)\b` + strings.ToLower(opcode) + `\s+[^\n]*\([^\n]*\),\s*%xmm`) + if count := len(pattern.FindAll(output, -1)); count != 5 { + t.Fatalf("%s count=%d, original memory opcode lost:\n%s", opcode, count, output) + } + } + if !strings.Contains(string(output), "%fs:") || !strings.Contains(string(output), "%gs:") { + t.Fatal("native FS/GS source prefixes lost") + } + }) + } + } +} + +func TestX86LegacyPackedFloatingMemoryCallerPermissionsAndNativeWidth(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, operand := range []string{"(DI)", "16(DI)", "(BX)", "(SP)", "a+0(FP)", "(R14)", "(CX)"} { + body := fmt.Sprintf("VMOVUPS (SI),Z1\n%s %s,X1\nVMOVSS X1,(DX)", opcode, operand) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + want := operand == "(DI)" || operand == "16(DI)" + if want && err != nil || !want && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%s actual source permission=%v want admitted=%t", opcode, operand, err, want) + } + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+opcode+" (DI),X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, extent := range []int64{15, 16} { + ins := file.Funcs[0].Instrs[1] + proof := &x86GoCallerRegionProof{arguments: []x86CallerArgumentRegion{{kind: x86CallerFreshResult, width: extent}}} + state := x86SourceStackState{callers: map[Reg]x86CallerPointerValue{DI: {argument: 0}}, vectors: map[Reg]uint64{"Z1": ^uint64(0)}} + accepted := x86CallerVectorMoveStep(&ins, &state, proof) + if accepted != (extent == 16) { + t.Fatalf("%s certified a partial m128 source extent=%d accepted=%t", opcode, extent, accepted) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryIdentityAndMalformedBeforeEmission(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + valid, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+opcode+" (AX),X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, sourceOp := range []Op{"", Op(opcode), Op(strings.ToLower(opcode)), "UNKNOWN", "ADDPS.Z", "DIVPS"} { + ins := valid.Funcs[0].Instrs[1] + ins.Op = sourceOp + ctx, output := newAMD64CtxWithFuncForTest(t, valid.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + var known, term bool + if strings.HasPrefix(opcode, "ADD") || strings.HasPrefix(opcode, "SUB") { + known, term, err = ctx.lowerFloatAddSub(Op(opcode), ins) + } else { + known, term, err = ctx.lowerFloatMulDiv(Op(opcode), ins) + } + conflict := sourceOp != "" && !strings.EqualFold(string(sourceOp), opcode) + if !known || term || (err != nil) != conflict || conflict && output.Len() != before || ins.Op != sourceOp { + t.Fatalf("%s/%s changed identity or emitted before rejection: %t/%t/%v", opcode, sourceOp, known, term, err) + } + } + for _, row := range []string{opcode + " (AX),Y1", opcode + " (AX),X16", opcode + ".Z (AX),X1", opcode + " $data(SB),X1", opcode + " (AX),X1,X2"} { + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + ins := file.Funcs[0].Instrs[1] + var known bool + var term bool + if strings.HasPrefix(opcode, "ADD") || strings.HasPrefix(opcode, "SUB") { + known, term, err = ctx.lowerFloatAddSub(ins.Op, ins) + } else { + known, term, err = ctx.lowerFloatMulDiv(ins.Op, ins) + } + if !known || term || err == nil || before != output.Len() { + t.Fatalf("malformed %s emitted native IR: %t/%t/%v", row, known, term, err) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryMode32FeaturesAreFormBound(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + for _, row := range []string{opcode + " (AX),X1", opcode + " (AX),X8", opcode + ".Z (AX),X1", opcode + " value+0(FP),X1"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + bits := x86LegacyPackedFloatingNativeMemoryBits(file.Funcs[0].Instrs[1], "386") + want := 0 + if row == opcode+" (AX),X1" { + want = 32 + if strings.HasSuffix(opcode, "PD") { + want = 64 + } + } + if bits != want { + t.Fatalf("unvalidated source form acquired SSE transport: %s %d/%d", row, bits, want) + } + } + } +} + +func TestX86LegacyPackedFloatingMemoryVirtualFPIsNotNativeStorage(t *testing.T) { + for _, opcode := range x86LegacyFloatMemoryOpcodes() { + assembly := fmt.Sprintf("TEXT f(SB),4,$0-16\n%s source+0(FP),X1\nRET\n", opcode) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sig := FuncSig{Name: "f", Args: []LLVMType{I64, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}, + }}} + _, err = Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{"f": sig}}) + if err == nil || !strings.Contains(err.Error(), "contiguous original FP") { + t.Fatalf("%s invented native m128 alignment/backing from independent virtual fields: %v", opcode, err) + } + } +} diff --git a/amd64_legacy_integer_arithmetic_effects.go b/amd64_legacy_integer_arithmetic_effects.go new file mode 100644 index 000000000..e542707c7 --- /dev/null +++ b/amd64_legacy_integer_arithmetic_effects.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86LegacyIntegerArithmeticForm struct { + x86PackedBinaryForm + source, destination Operand +} + +// Reuse the arithmetic specifications and Go operand rows already owned by +// lowering. This descriptor adds no frame, source, caller or initialized bank. +func x86LegacyIntegerArithmeticProperties(op Op, ins Instr, arch string) (x86LegacyIntegerArithmeticForm, bool, error) { + name, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, add := amd64PackedAddSpecs[Op(name)] + _, _, vector, subtract := amd64PackedSubtractProperties(name) + if add && spec.form == amd64PackedAddVEXEVEX || subtract && vector || !add && !subtract { + return x86LegacyIntegerArithmeticForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86LegacyIntegerArithmeticForm{}, true, fmt.Errorf("legacy integer arithmetic dispatch/source mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + local.Args = append([]Operand(nil), ins.Args...) + encoding := x86PackedLogicalLegacyFP + if add && spec.form == amd64PackedAddLegacyYMM { + encoding = x86PackedLogicalLegacyInteger + } + if encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return x86LegacyIntegerArithmeticForm{}, true, err + } + } + form, err := x86PackedBinaryProperties(local, arch, x86PackedBinarySpec{encoding: encoding}) + result := x86LegacyIntegerArithmeticForm{x86PackedBinaryForm: form} + if err == nil { + result.source, result.destination = local.Args[0], local.Args[1] + } + return result, true, err +} + +func x86TypedLegacyIntegerArithmeticByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, recognized, err := x86LegacyIntegerArithmeticProperties(op, ins, arch) + if !recognized || err != nil { + return x86TypedByteEffect{}, recognized, err + } + width := int64(form.byteWidth) + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: form.source, width: width, read: true}, + {index: 1, operand: form.destination, width: width, read: true, write: true}, + }}, true, nil +} + +func x86LegacyIntegerArithmeticEmitterContext(form x86LegacyIntegerArithmeticForm, ins Instr) error { + if form.mmx && ins.Args[0].Kind == OpReg && ins.Args[0].Reg != form.source.Reg { + return fmt.Errorf("%w: legacy integer arithmetic GP spelling encodes an MMX source without its physical bank transport", ErrProbeNeedsContext) + } + return nil +} diff --git a/amd64_legacy_integer_arithmetic_forms_test.go b/amd64_legacy_integer_arithmetic_forms_test.go new file mode 100644 index 000000000..ebde9a746 --- /dev/null +++ b/amd64_legacy_integer_arithmetic_forms_test.go @@ -0,0 +1,276 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86LegacyIntegerArithmeticTestOpcodes = []byte{ + 0xfc, 0xfd, 0xfe, 0xfe, 0xd4, 0xec, 0xed, 0xdc, 0xdd, + 0xf8, 0xf9, 0xfa, 0xfb, 0xe8, 0xe9, 0xd8, 0xd9, +} + +func TestX86LegacyIntegerArithmeticGrammarEffectsAndNativeWidths(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86LegacyIntegerArithmeticTestNames { + for _, operands := range []string{"X0,X1", "X0,X0", "(AX),X1", "8(AX)(CX*4),X1", "a+0(FP),X1", "value-32(SP),X1", "·data(SB),X1", "(FS),X1", "(GS),X1"} { + t.Run(arch+"/"+op+" "+operands, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·Value(SB),4,$64-64\n"+op+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, recognized, err := x86LegacyIntegerArithmeticProperties(ins.Op, ins, arch) + if !recognized || err != nil || form.byteWidth != 16 || form.mmx { + t.Fatal("full accepted XMM row lost", form, err) + } + effect, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !recognized || err != nil || len(effect.operands) != 2 || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatal("legacy arithmetic lost its two inputs or unchanged flags", effect, err) + } + for index, role := range effect.operands { + if role.index != index || role.width != 16 || !role.read || role.write != (index == 1) || !reflect.DeepEqual(role.operand, ins.Args[index]) { + t.Fatalf("wrong ordered input/output bytes: %#v", role) + } + width, write, known := x86InstructionMemoryShape(ins, index, arch) + if known != (form.memory == index) || known && (width != 16 || write) { + t.Fatal("descriptor and native memory reader differ", width, write, known) + } + if ins.Args[index].Kind == OpFP { + width, write, known = X86FPAccessBytes(ins, index, arch) + if !known || width != 16 || write { + t.Fatal("FP observer does not preserve full native read", width, write, known) + } + } + } + if strings.HasPrefix(op, "PADDS") || strings.HasPrefix(op, "PADDUS") { + if x86CallerVectorMoveKnown(ins) || x86SourceStackInstructionKnown(ins, "MOVQ", "Q", arch) { + t.Fatal("pure descriptor granted unproved saturating caller transport") + } + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("shape observer changed original source") + } + }) + } + } + } +} + +func TestX86LegacyIntegerArithmeticActualGoBytesAndMMXBank(t *testing.T) { + for opIndex, op := range x86LegacyIntegerArithmeticTestNames { + for _, arch := range []string{"amd64", "386"} { + for _, operands := range []struct { + text string + modrm byte + }{ + {"X1,X0", 0xc1}, {"X0,X7", 0xf8}, {"(AX),X1", 0x08}, + } { + t.Run(arch+"/"+op+" "+operands.text, func(t *testing.T) { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + operands.text + "\nRET\n" + want := []byte{0x66, 0x0f, x86LegacyIntegerArithmeticTestOpcodes[opIndex], operands.modrm, 0xc3} + if got := x86PackedShiftActualGoBytes(t, arch, source); !bytes.Equal(got, want) { + t.Fatalf("original source width/opcode differs: %x want %x", got, want) + } + }) + } + } + if !strings.HasPrefix(op, "PADD") || op == "PADDQ" { + continue + } + for index, spelling := range []string{"M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + t.Run("mmx/"+op+" "+spelling, func(t *testing.T) { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + spelling + ",M7\nRET\n" + want := []byte{0x0f, x86LegacyIntegerArithmeticTestOpcodes[opIndex], 0xf8 | byte(index&7), 0xc3} + if index >= 16 { + want = append([]byte{0x41}, want...) + } + if got := x86PackedShiftActualGoBytes(t, "amd64", source); !bytes.Equal(got, want) { + t.Fatalf("Go's GP spelling must name the physical MMX bank: %x want %x", got, want) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !recognized || err != nil || effect.operands[0].width != 8 || effect.operands[0].operand.Reg != Reg(fmt.Sprintf("M%d", index&7)) { + t.Fatal("pure effects confused a numeric GP with a physical MMX input", effect, err) + } + state := x86ResultState{flags: x86ByteFlagsAll} + for i := range state.gp { + state.gp[i] = 0xff + } + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("initialized GP bytes fabricated MMX input/output permission") + } + }) + } + } +} + +func TestX86ABI0ResultLegacyIntegerArithmeticPartialBytesAndFlags(t *testing.T) { + for _, op := range x86LegacyIntegerArithmeticTestNames { + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + for source := 0; source < 2; source++ { + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/input%d/byte%d", op, source, missing), func(t *testing.T) { + state := x86ResultState{flags: x86ByteFlagsAll} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + state.xmm[1-source] &^= uint16(1) << uint(missing) + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("read-modify-write borrowed one missing input byte") + } + }) + } + } + for flags := uint8(0); flags <= x86ByteFlagsAll; flags++ { + state := x86ResultState{flags: flags} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + if !x86ResultTransfer(ins, &state, nil, 0, "amd64") || state.flags != flags || state.xmm[0] != 0xffff || state.xmm[1] != 0xffff { + t.Fatal("legacy lane arithmetic altered flags or byte aliases", op, flags, state) + } + } + body := "MOVSS a+0(FP),X1\nMOVL $1,AX\nCMPL AX,$0\nJE partial\nMOVSD a+0(FP),X0\nJMP joined\npartial:\nMOVSS X1,X0\njoined:\n" + op + " X1,X0\nMOVSD X0,ret0+8(FP)\nRET" + file, sig, _ := x86ResultAUTOFixture(t, "amd64", "func Value(a uint64) uint64", 64, body) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("partial destination path became initialized", op, err) + } + } +} + +func TestX86LegacyIntegerArithmeticThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + for _, op := range x86LegacyIntegerArithmeticTestNames { + t.Run(target.triple+"/"+op, func(t *testing.T) { + body := "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\n" + op + " X1,X0\n" + op + " X0,X0\nMOVSD X0,ret0+32(FP)\nRET" + pkg, file, sig, source := x86ResultInitializationFixture(t, target.arch, "func Value(a,b,c,d uint64) uint64", body) + goAssemblerOracleOutput(t, target.arch, source, "", true) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, target.arch); err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Value": sig}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + before := fmt.Sprintf("%#v %#v", file, sig) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "text") + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, module.String(), "module") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, translated.Module.String(), "typed-go") + if before != fmt.Sprintf("%#v %#v", file, sig) { + t.Fatal("API changed original inputs") + } + }) + } + } +} + +func TestX86LegacyIntegerArithmeticMalformedFormsAndIdentity(t *testing.T) { + for _, op := range x86LegacyIntegerArithmeticTestNames { + for _, operands := range []string{"$1,X0", "X0,(AX)", "Y0,Y1", "Z0,Z1", "X16,X0", "(AX),(BX)", "X0,K1,X1"} { + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+op+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if _, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64"); !recognized || err == nil { + t.Fatal("malformed row acquired effects", ins, err) + } + } + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+op+" X0,X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, identity := range []Op{"", Op(op), Op(strings.ToLower(op))} { + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + if _, recognized, err := x86LegacyIntegerArithmeticProperties(Op(op), ins, "amd64"); !recognized || err != nil { + t.Fatal("matching dispatch lost compatibility", identity, err) + } + } + ins := file.Funcs[0].Instrs[1] + ins.Op = "MOVQ" + if _, recognized, err := x86TypedInstructionByteEffect(Op(op), ins, "amd64"); !recognized || err == nil { + t.Fatal("contradictory dispatch obtained effects", err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + var known bool + if strings.HasPrefix(op, "PADD") { + known, _, err = ctx.lowerPackedIntegerAdd(Op(op), ins) + } else { + known, _, err = ctx.lowerPackedIntegerSubtract(Op(op), ins) + } + if !known || err == nil || output.String() != before { + t.Fatal("contradictory dispatch changed IR before rejection", op, known, err) + } + for _, operands := range []string{"X8,X0", "X0,X15"} { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if _, recognized, err := x86TypedInstructionByteEffect(ins.Op, ins, "386"); !recognized || err == nil { + t.Fatal("mode32 acquired an unavailable physical X bank", op, err) + } + } + } +} + +func TestX86ABI0ResultLegacyIntegerArithmeticDoesNotBorrowPermission(t *testing.T) { + for _, op := range x86LegacyIntegerArithmeticTestNames { + for name, body := range map[string]string{ + "source unknown": "MOVSD a+0(FP),X0\n" + op + " X2,X0\nMOVSD X0,ret0+32(FP)\nRET", + "old destination unknown": "MOVSD a+0(FP),X1\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET", + "native pointer": "MOVSD a+0(FP),X0\nMOVQ a+0(FP),AX\n" + op + " (AX),X0\nMOVSD X0,ret0+32(FP)\nRET", + "AUTO partial": "MOVSD a+0(FP),X0\nMOVQ $1,value-32(SP)\n" + op + " value-32(SP),X0\nMOVSD X0,ret0+32(FP)\nRET", + "FP gap or result": "MOVSD a+0(FP),X0\n" + op + " b+28(FP),X0\nMOVSD X0,ret0+32(FP)\nRET", + "call kills facts": "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\nCALL ·Other(SB)\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET", + "dead unknown effect": "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET\nRDTSC\nRET", + "escape logical AUTO": "MOVSD a+0(FP),X0\nMOVSD c+16(FP),X1\nLEAQ value-32(SP),AX\n" + op + " X1,X0\nMOVSD X0,ret0+32(FP)\nRET", + } { + t.Run(op+"/"+name, func(t *testing.T) { + file, sig, _ := x86ResultAUTOFixture(t, "amd64", "func Value(a,b,c,d uint64) uint64", 64, body) + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("pure arithmetic effects fabricated initialization or backing", err) + } + }) + } + } +} diff --git a/amd64_legacy_integer_arithmetic_runtime_test.go b/amd64_legacy_integer_arithmetic_runtime_test.go new file mode 100644 index 000000000..b7cfad205 --- /dev/null +++ b/amd64_legacy_integer_arithmetic_runtime_test.go @@ -0,0 +1,407 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// Independent mathematical attributes, not emitted from the production specs. +var x86LegacyIntegerArithmeticTestBytes = []int{1, 2, 4, 4, 8, 1, 2, 1, 2, 1, 2, 4, 8, 1, 2, 1, 2} +var x86LegacyIntegerArithmeticTestModes = []int{0, 0, 0, 0, 0, 1, 1, 2, 2, 0, 0, 0, 0, 1, 1, 2, 2} + +const x86LegacyIntegerArithmeticGoMath = ` +func lane(value []byte) uint64 { + var result uint64 + for i,b := range value { result |= uint64(b)<max-b { return max } + return a+b + } + sign := int64(1)< sign-1 { result=sign-1 } + return uint64(result) +} +func fill(value []byte,n,sample,side int) { + max := ^uint64(0) + if n<8 { max=(uint64(1)<>1,(max>>1)+1,max-1,2,(max>>1)-1} + for offset:=0;offset<16;offset+=n { + number:=uint64(sample*71+offset*37+side*19)*0x11223344556677 + if sample<64 { number=choices[(sample/(1+side*7)+offset/n)%len(choices)] } + for j:=0;j>uint(j*8)) } + } +} +` + +const x86LegacyIntegerArithmeticCMath = ` +static uint64_t lane(const uint8_t *value,int n) { + uint64_t result=0; + for(int i=0;imax-b?max:a+b; + } + int64_t sign=INT64_C(1)<<(n*8-1),left=(int64_t)a,right=(int64_t)b; + if(left&sign) left-=sign*2; + if(right&sign) right-=sign*2; + int64_t result=subtract?right-left:right+left; + if(result< -sign) result= -sign; + if(result>sign-1) result=sign-1; + return (uint64_t)result; +} +static void fill(uint8_t *value,int n,int sample,int side) { + uint64_t max=n==8?UINT64_MAX:(UINT64_C(1)<<(n*8))-1; + uint64_t choices[]={0,1,max,max>>1,(max>>1)+1,max-1,2,(max>>1)-1}; + for(int offset=0;offset<16;offset+=n) { + uint64_t number=(uint64_t)(sample*71+offset*37+side*19)*UINT64_C(0x11223344556677); + if(sample<64) number=choices[(sample/(1+side*7)+offset/n)%8]; + for(int j=0;j>(j*8)); + } +} +` + +func TestCrossLinuxRuntimeMatrixX86LegacyIntegerArithmeticOriginalMathFlagsAndCanaries(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, goTable, cTable, cDeclarations strings.Builder + declarations.WriteString("package packedarithmetic\nfunc ReadCSR() uint32\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\n") + for index, op := range x86LegacyIntegerArithmeticTestNames { + for _, kind := range []string{"Register", "Alias", "Memory"} { + name := op + kind + source := "X1" + if kind == "Alias" { + source = "X0" + } else if kind == "Memory" { + source = "(AX)" + } + fmt.Fprintf(&declarations, "func %s(out,x,y *[16]byte,flags *uint64,seed uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),DX\nMOVQ x+8(FP),AX\nMOVQ y+16(FP),CX\nMOVOU (AX),X1\nMOVOU (CX),X0\nPUSHQ seed+32(FP)\nPOPFQ\n%s %s,X0\nMOVOU X0,(DX)\nPUSHFQ\nPOPQ BX\nMOVQ flags+24(FP),SI\nMOVQ BX,(SI)\nRET\n", name, op, source) + fmt.Fprintf(&goTable, "{%q,%s,%d,%d,%t,%t},\n", name, name, + x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestModes[index], index >= 9, kind == "Alias") + fmt.Fprintf(&cDeclarations, "extern void %s(uint8_t*,uint8_t*,uint8_t*,uint64_t*,uint64_t);\n", name) + fmt.Fprintf(&cTable, "{%q,%s,%d,%d,%d,%d},\n", name, name, + x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestModes[index], x86FloatAddSubBoolInt(index >= 9), x86FloatAddSubBoolInt(kind == "Alias")) + } + } + goTest := `package packedarithmetic +import ( + "testing" + "unsafe" +) +` + x86LegacyIntegerArithmeticGoMath + ` +func aligned(value *[80]byte) (*[16]byte,int) { + offset:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15) + return (*[16]byte)(unsafe.Pointer(&value[offset])),offset +} + +func TestOriginal(t *testing.T) { + kernels:=[]struct { + name string + call func(*[16]byte,*[16]byte,*[16]byte,*uint64,uint64) + n,mode int + subtract,alias bool + }{ +` + goTable.String() + ` } + for _,kernel:=range kernels { + for sample:=0;sample<257;sample++ { + var x,y,out [80]byte + xp,_:=aligned(&x);yp,_:=aligned(&y);op,offset:=aligned(&out) + for i:=range x { x[i]=0xb9;y[i]=0xa7 } + fill(xp[:],kernel.n,sample,0);fill(yp[:],kernel.n,sample,1) + originalX,originalY:=x,y + for _,seed:=range []uint64{0,0x8d5} { + for i:=range out { out[i]=0xa5 } + box:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=ReadCSR() + kernel.call(op,xp,yp,&box.value,seed) + if box.value&0x8d5!=seed||box.before!=0x1122334455667788||box.after!=0x8877665544332211||ReadCSR()!=csr||x!=originalX||y!=originalY { + t.Fatalf("%s state/canary sample%d flags%x",kernel.name,sample,box.value) + } + var want [16]byte + for i:=0;i<16;i+=kernel.n { + a,b:=lane(xp[i:i+kernel.n]),lane(yp[i:i+kernel.n]) + if kernel.alias { a=b } + value:=math(a,b,kernel.n,kernel.mode,kernel.subtract) + for j:=0;j>uint(j*8)) } + } + for i,value:=range out { + expected:=byte(0xa5) + if i>=offset&&i +#include +#include +#include +struct Aligned { _Alignas(16) uint8_t before[16],value[16],after[16]; }; +` + cDeclarations.String() + x86LegacyIntegerArithmeticCMath + ` +static const struct Kernel { + const char *name; + void (*call)(uint8_t*,uint8_t*,uint8_t*,uint64_t*,uint64_t); + int n,mode,subtract,alias; +} kernels[]={ +` + cTable.String() + `}; +int main(void) { + for(unsigned k=0;kn,sample,0);fill(y.value,kernel->n,sample,1); + struct Aligned originalX=x,originalY=y; + for(int status=0;status<2;status++) { + uint64_t seed=status?UINT64_C(0x8d5):0; + memset(&out,0xa5,sizeof(out)); + struct {uint64_t before,value,after;} flags={UINT64_C(0x1122334455667788),0,UINT64_C(0x8877665544332211)}; + uint32_t csr=_mm_getcsr(); + kernel->call(out.value,x.value,y.value,&flags.value,seed); + if((flags.value&UINT64_C(0x8d5))!=seed||flags.before!=UINT64_C(0x1122334455667788)||flags.after!=UINT64_C(0x8877665544332211)||_mm_getcsr()!=csr||memcmp(&x,&originalX,sizeof(x))||memcmp(&y,&originalY,sizeof(y))) { + fprintf(stderr,"%s state/input/canary sample%d flags%llx\n",kernel->name,sample,(unsigned long long)flags.value);return 1; + } + for(int i=0;i<16;i+=kernel->n) { + uint64_t a=lane(x.value+i,kernel->n),b=lane(y.value+i,kernel->n); + if(kernel->alias) a=b; + uint64_t expected=math(a,b,kernel->n,kernel->mode,kernel->subtract); + for(int j=0;jn;j++) if(out.value[i+j]!=(uint8_t)(expected>>(j*8))) { + fprintf(stderr,"%s sample%d byte%d got%x want%x\n",kernel->name,sample,i+j,out.value[i+j],(unsigned)(uint8_t)(expected>>(j*8)));return 2; + } + } + for(int i=0;i<16;i++) if(out.before[i]!=0xa5||out.after[i]!=0xa5) return 3; + } + } + } + return 0; +} +` + dir := t.TempDir() + for name, contents := range map[string]string{ + "go.mod": "module packedarithmetic\n\ngo 1.20\n", "arithmetic.go": declarations.String(), + "arithmetic_amd64.s": assembly.String(), "arithmetic_test.go": goTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + binary := filepath.Join(dir, "original.test") + command := exec.Command("go", "test", "-p=2", "-vet=off", "-c", "-o", binary, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + x86FloatCompareCaptureCommand(t, command) + output := execution.run(t, binary, "-test.run=^TestOriginal$", "-test.count=1", "-test.v") + if !strings.Contains(string(output), "--- PASS: TestOriginal") || !strings.Contains(string(output), "PASS\n") { + t.Fatalf("original Go legacy arithmetic lacks successful terminal: %s", output) + } + pkg := mustGoPackage(t, "example.invalid/packedarithmetic", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-integer-arithmetic", ir, main, level) + }) + } + if !t.Failed() { + t.Log("51 original-Go/LLVM XMM kernels ×257 initialized boundary/random samples ×2 full-six-flag seeds, complete bytes/inputs/MXCSR/canaries; Darwin/Rosetta is not native AMD64/386 fault proof.") + } +} + +func TestCrossLinuxRuntimeMatrixX86LegacyIntegerArithmeticOriginalMMXBank(t *testing.T) { + // Go's GP-looking source is an MMX ModRM spelling, not an ABI GP input. + // This original-Go-only oracle does not initialize the LLVM/MMX observer. + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package packedmmx\nfunc ReadCSR() uint32\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\n") + spellings := []string{"M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + for opIndex, op := range x86LegacyIntegerArithmeticTestNames { + if !strings.HasPrefix(op, "PADD") || op == "PADDQ" { + continue + } + for index, spelling := range spellings { + name := fmt.Sprintf("%sBank%d", op, index) + fmt.Fprintf(&declarations, "func %s(a,b,c,d,e,f,v,h uint64,flags *uint64,seed uint64) uint64\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-88\n", name) + for bank, parameter := range []string{"a", "b", "c", "d", "e", "f", "v", "h"} { + fmt.Fprintf(&assembly, "MOVQ %s+%d(FP),M%d\n", parameter, bank*8, bank) + } + // Do not change native SP/BP/g. Other GP values deliberately do not + // match any initialized MMX value. + for _, reg := range []string{"AX", "CX", "DX", "BX", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R15"} { + fmt.Fprintf(&assembly, "MOVQ $0x5a5a5a5a,%s\n", reg) + } + fmt.Fprintf(&assembly, "PUSHQ seed+72(FP)\nPOPFQ\n%s %s,M7\nMOVQ M7,ret0+80(FP)\nEMMS\nPUSHFQ\nPOPQ AX\nMOVQ flags+64(FP),DX\nMOVQ AX,(DX)\nRET\n", op, spelling) + fmt.Fprintf(&table, "{%q,%s,%d,%d,%d},\n", name, name, index&7, + x86LegacyIntegerArithmeticTestBytes[opIndex], x86LegacyIntegerArithmeticTestModes[opIndex]) + } + } + goTest := `package packedmmx +import "testing" +` + x86LegacyIntegerArithmeticGoMath + ` +func TestOriginal(t *testing.T) { + kernels:=[]struct { + name string + call func(uint64,uint64,uint64,uint64,uint64,uint64,uint64,uint64,*uint64,uint64) uint64 + bank,n,mode int + }{ +` + table.String() + ` } + for _,kernel:=range kernels { + for sample:=0;sample<33;sample++ { + var values [8]uint64 + for bank:=range values { + var bytes [16]byte + fill(bytes[:],kernel.n,sample,bank) + values[bank]=lane(bytes[:8]) + } + var want uint64 + for offset:=0;offset<8;offset+=kernel.n { + mask:=(uint64(1)<>uint(offset*8))&mask + b:=(values[7]>>uint(offset*8))&mask + value:=math(a,b,kernel.n,kernel.mode,false)&mask + want|=value<\n#include \n#include \n" + x86LegacyIntegerArithmeticCMath) + constant := func(multiplier, addend int) string { + var values []string + for index := 0; index < 64; index++ { + values = append(values, fmt.Sprintf("i8 %d", byte(index*multiplier+addend))) + } + return "<" + strings.Join(values, ",") + ">" + } + for index, op := range x86LegacyIntegerArithmeticTestNames { + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + fn := Func{Instrs: []Instr{{Op: "VMOVUPS", Args: []Operand{{Kind: OpReg, Reg: "Z0"}, {Kind: OpReg, Reg: "Z1"}}}, ins}} + ctx, output := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + if err := ctx.storeZ("Z0", constant(37, 11)); err != nil { + t.Fatal(err) + } + if err := ctx.storeZ("Z1", constant(19, 71)); err != nil { + t.Fatal(err) + } + known, term, err := ctx.lowerPackedIntegerAdd(ins.Op, ins) + if !known { + known, term, err = ctx.lowerPackedIntegerSubtract(ins.Op, ins) + } + if !known || term || err != nil { + t.Fatal("legacy lowerer did not consume typed X bank", known, term, err) + } + first, err := ctx.loadZ("Z0") + if err != nil { + t.Fatal(err) + } + second, err := ctx.loadZ("Z1") + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(&ir, "define void @%sShadow(ptr %%out) {\n%s store <64 x i8> %s, ptr %%out, align 1\n %%next = getelementptr i8, ptr %%out, i64 64\n store <64 x i8> %s, ptr %%next, align 1\n ret void\n}\n", op, output.String(), first, second) + fmt.Fprintf(&reference, "extern void %sShadow(uint8_t*);\n", op) + fmt.Fprintf(&checks, ` { + memset(out,0xa5,sizeof(out)); + %sShadow(out+16); + uint8_t a[64],b[64]; + for(int i=0;i<64;i++){a[i]=(uint8_t)(i*19+71);b[i]=(uint8_t)(i*37+11);} + for(int offset=0;offset<16;offset+=%d){ + uint64_t value=math(lane(a+offset,%d),lane(b+offset,%d),%d,%d,%d); + for(int j=0;j<%d;j++)b[offset+j]=(uint8_t)(value>>(j*8)); + } + if(memcmp(out+16,b,64)||memcmp(out+80,a,64)){fprintf(stderr,"%s alias\n");return 1;} + for(int i=0;i<16;i++)if(out[i]!=0xa5||out[144+i]!=0xa5)return 2; + } +`, op, x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestBytes[index], x86LegacyIntegerArithmeticTestModes[index], x86FloatAddSubBoolInt(index >= 9), x86LegacyIntegerArithmeticTestBytes[index], op) + } + reference.WriteString("int main(void){uint8_t out[160];\n" + checks.String() + "return 0;}\n") + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-integer-arithmetic-alias", ir.String(), reference.String(), level) + }) + } +} diff --git a/amd64_legacy_logical_memory_linux_amd64_test.go b/amd64_legacy_logical_memory_linux_amd64_test.go new file mode 100644 index 000000000..df2003b39 --- /dev/null +++ b/amd64_legacy_logical_memory_linux_amd64_test.go @@ -0,0 +1,9 @@ +package plan9asm + +import "testing" + +// Required by the cross-runtime CI prefix. The native Linux axis additionally +// checks the original aligned-m128 contract, which Rosetta does not reproduce. +func TestCrossLinuxRuntimeMatrixX86LegacyLogicalMemoryNativeFaults(t *testing.T) { + x86LegacyLogicalMemoryRuntime(t, true) +} diff --git a/amd64_legacy_logical_memory_runtime_unix_test.go b/amd64_legacy_logical_memory_runtime_unix_test.go new file mode 100644 index 000000000..581ae33a7 --- /dev/null +++ b/amd64_legacy_logical_memory_runtime_unix_test.go @@ -0,0 +1,143 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Execute only first-party, initialized storage. The original reference uses +// actual stock-Go opcode bytes, not a compiler intrinsic or reconstructed math. +// Rosetta is a diagnostic transport; required Linux/amd64 executes this root. +func TestCrossLinuxRuntimeMatrixX86LegacyLogicalMemoryOriginalFaultsAndBytes(t *testing.T) { + x86LegacyLogicalMemoryRuntime(t, false) +} + +func x86LegacyLogicalMemoryRuntime(t *testing.T, nativeAlignment bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, kernels, checks strings.Builder + signatures := map[string]FuncSig{} + for index, name := range x86LegacyLogicalMemoryNames { + code := x86LegacyLogicalOriginalBytes(t, "amd64", name) + literal := make([]string, len(code)) + for i, value := range code { + literal[i] = fmt.Sprintf("0x%02x", value) + } + fmt.Fprintf(&kernels, ` +static __attribute__((noinline)) void original%d(uint8_t *out,const uint8_t *a,const uint8_t *b) { + __asm__ volatile("movups (%%1),%%%%xmm0\n\tstc\n\t.byte %s\n\tmovups %%%%xmm0,(%%2)\n\tsetc 16(%%2)" + : : "a"(a),"r"(b),"r"(out) : "xmm0","memory","cc"); +} +`, index, strings.Join(literal, ",")) + for _, dead := range []bool{false, true} { + fn := fmt.Sprintf("logical%d_%t", index, dead) + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVUPS (CX),X0\nSTC\n%s (BX),X0\n", fn, name) + if !dead { + assembly.WriteString("MOVUPS X0,(AX)\n") + } + assembly.WriteString("SETCS 16(AX)\nRET\n") + signatures[fn] = FuncSig{Name: fn, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", fn) + } + mode := []int{0, 0, 1, 1, 2, 2, 3, 3, 0, 1, 2, 3}[index] + fmt.Fprintf(&checks, `{ + if(check_values(original%[1]d,logical%[1]d_false,%[2]d)) return %[3]d; + for(int kind=%[5]d;kind<3;++kind) { + int original_fault=observe_fault(original%[1]d,kind); + if(original_fault!=SIGSEGV && original_fault!=SIGBUS) { fprintf(stderr,"%[4]s original fault kind%%d=%%d\n",kind,original_fault); return %[3]d; } + if(observe_fault(logical%[1]d_false,kind)!=original_fault || observe_fault(logical%[1]d_true,kind)!=original_fault) return %[3]d; + } +} +`, index, mode, index+10, name, map[bool]int{false: 1, true: 0}[nativeAlignment]) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: signatures}) + if err != nil { + t.Fatal(err) + } + const helper = ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*kernel)(uint8_t*,const uint8_t*,const uint8_t*); +static int observe_fault(kernel call,int kind) { + _Alignas(16) uint8_t a[48],b[48],out[48]; + memset(a,0x35,sizeof(a)); memset(b,0x71,sizeof(b)); memset(out,0xa7,sizeof(out)); + long page=sysconf(_SC_PAGESIZE); + if(page<=0) return -3; + uint8_t *area=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_ANON|MAP_PRIVATE,-1,0); + if(area==MAP_FAILED) return -4; + if(mprotect(area+page,(size_t)page,PROT_NONE)) { munmap(area,(size_t)page*2); return -5; } + pid_t child=fork(); if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + const uint8_t *source=kind==0?a+1:kind==1?0:area+page; + call(out+1,source,b+1); + _exit(91); + } + int status=0; + if(waitpid(child,&status,0)!=child) return -2; + if(munmap(area,(size_t)page*2)) return -6; + return WIFSIGNALED(status)?WTERMSIG(status):-(WEXITSTATUS(status)+100); +} +static int check_values(kernel original,kernel translated,int mode) { + for(int sample=0;sample<257;++sample) for(int rc=0;rc<4;++rc) { + _Alignas(16) uint8_t a[48],b[48],go[48],llvm[48]; + memset(a,0xa7,sizeof(a)); memset(b,0xb9,sizeof(b)); + memset(go,0xc3,sizeof(go)); memset(llvm,0xc3,sizeof(llvm)); + for(int i=0;i<16;++i) { a[i+16]=(uint8_t)(sample*71+i*19); b[i+1]=(uint8_t)(sample*31-i*59); } + uint8_t original_a[48],original_b[48]; memcpy(original_a,a,48); memcpy(original_b,b,48); + uint32_t saved=_mm_getcsr(),csr=0x1f80|(rc<<13)|(sample&63); + _mm_setcsr(csr); original(go+1,a+16,b+1); uint32_t go_csr=_mm_getcsr(); + _mm_setcsr(csr); translated(llvm+1,a+16,b+1); uint32_t llvm_csr=_mm_getcsr(); _mm_setcsr(saved); + if(go_csr!=csr || llvm_csr!=csr || memcmp(a,original_a,48) || memcmp(b,original_b,48) || memcmp(go,llvm,48)) return 1; + if(go[17]!=1) return 2; + for(int i=0;i<48;++i) if((i<1 || i>17) && go[i]!=0xc3) return 2; + for(int i=0;i<16;++i) { + uint8_t x=a[i+16],y=b[i+1],want; + /* Two PS/PD encodings per operation; the last four are integer aliases. */ + if(mode==0) want=x&y; + else if(mode==1) want=x&(uint8_t)~y; + else if(mode==2) want=x|y; + else if(mode==3) want=x^y; + else return 3; + if(go[i+1]!=want) return 4; + } + } + return 0; +} +` + driver := helper + kernels.String() + declarations.String() + "int main(void) {\n" + checks.String() + "return 0;\n}\n" + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-logical-native", ir, driver, level) + }) + } + t.Logf("12 original memory opcodes: 257×4 initialized byte/CF/MXCSR/canary inputs; live/dead NULL+guard-page faults at O0/O2; native alignment=%t; host=%s/%s", nativeAlignment, runtime.GOOS, runtime.GOARCH) +} + +func TestX86LegacyLogicalMemoryFaultChildDisablesCore(t *testing.T) { + if err := runtimeOracleFaultCorePolicy(runtimeOracleFaultC(t, "amd64_legacy_logical_memory_runtime_unix_test.go")); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_legacy_logical_memory_test.go b/amd64_legacy_logical_memory_test.go new file mode 100644 index 000000000..a7614f887 --- /dev/null +++ b/amd64_legacy_logical_memory_test.go @@ -0,0 +1,267 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func x86LegacyLogicalOriginalBytes(t *testing.T, arch, name string) []byte { + t.Helper() + listing := x86InsertExtractGoListing(t, arch, "TEXT original(SB),4,$0-0\n"+name+" (AX),X0\nRET\n") + var code []byte + for _, line := range strings.Split(string(listing), "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + // Independent Intel encoding, not obtained from the translator's spec. + opcode := map[string]byte{ + "ANDPS": 0x54, "ANDPD": 0x54, "ANDNPS": 0x55, "ANDNPD": 0x55, + "ORPS": 0x56, "ORPD": 0x56, "XORPS": 0x57, "XORPD": 0x57, + "PAND": 0xdb, "PANDN": 0xdf, "POR": 0xeb, "PXOR": 0xef, + }[name] + want := []byte{0x0f, opcode, 0x00, 0xc3} + if strings.HasSuffix(name, "PD") || strings.HasPrefix(name, "P") { + want = append([]byte{0x66}, want...) + } + if !bytes.Equal(code, want) { + t.Fatalf("%s/%s actual Go memory opcode=%x independent encoding=%x", arch, name, code, want) + } + t.Logf("%s/%s original complete opcode+RET=%x", arch, name, code) + return append([]byte(nil), code[:len(code)-1]...) +} + +func TestX86LegacyLogicalMemoryOriginalGoBytes(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, name := range x86LegacyLogicalMemoryNames { + t.Run(arch+"/"+name, func(t *testing.T) { x86LegacyLogicalOriginalBytes(t, arch, name) }) + } + } +} + +func TestX86LegacyLogicalMemoryRequiredNativeSelection(t *testing.T) { + const file = "amd64_legacy_logical_memory_linux_amd64_test.go" + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "arm64"}, {"linux", "386"}, + {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + context := build.Default + context.GOOS, context.GOARCH = target.os, target.arch + selected, err := context.MatchFile(".", file) + if err != nil || selected != (target.os == "linux" && target.arch == "amd64") { + t.Fatalf("stock Go required native selection %s/%s: %t/%v", target.os, target.arch, selected, err) + } + } + syntax, err := parser.ParseFile(token.NewFileSet(), file, nil, 0) + if err != nil { + t.Fatal(err) + } + if len(syntax.Decls) != 2 { + t.Fatal("native wrapper must contain only its import and required root") + } + function, ok := syntax.Decls[1].(*ast.FuncDecl) + if !ok || function.Name.Name != "TestCrossLinuxRuntimeMatrixX86LegacyLogicalMemoryNativeFaults" || len(function.Body.List) != 1 { + t.Fatal("native root registration changed or became optional") + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("native root bypassed its mandatory helper") + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || len(call.Args) != 2 { + t.Fatal("native root must call the full fault helper") + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != "x86LegacyLogicalMemoryRuntime" { + t.Fatal("native root changed helper") + } + argument, ok := call.Args[1].(*ast.Ident) + if !ok || argument.Name != "true" { + t.Fatal("native root omitted its alignment axis") + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatalf("required cross-runtime prefix missing: %v", err) + } +} + +func TestX86LegacyLogicalMemoryUnboundAddressSizeBeforeEmission(t *testing.T) { + for _, name := range x86LegacyLogicalMemoryNames { + file, err := Parse(ArchAMD64, "TEXT view(SB),4,$0-0\n"+name+" (AX),X0\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, bits := range []int{32, 64} { + ins := file.Funcs[0].Instrs[1] + ins.x86AddressBits = bits + var output strings.Builder + context := &amd64Ctx{goarch: "amd64", b: &output} + known, terminal, err := context.lowerPackedLogicalFamily(ins.Op, ins) + if !known || terminal || !errors.Is(err, ErrProbeNeedsContext) || output.Len() != 0 { + t.Fatalf("%s unbound address-size%d emitted IR: %t/%t/%v", name, bits, known, terminal, err) + } + } + } +} + +// Independent legacy namespace: these twelve Go names all have an m128 +// source whose original SSE instruction requires aligned native storage. +var x86LegacyLogicalMemoryNames = []string{ + "ANDPS", "ANDPD", "ANDNPS", "ANDNPD", "ORPS", "ORPD", + "XORPS", "XORPD", "PAND", "PANDN", "POR", "PXOR", +} + +var x86LegacyLogicalMemoryTargets = []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, +} + +func TestX86LegacyLogicalMemoryFPNeedsNativeBacking(t *testing.T) { + for _, target := range x86LegacyLogicalMemoryTargets { + for _, name := range x86LegacyLogicalMemoryNames { + source := "TEXT ·view(SB),4,$0-16\nMOVOU a+0(FP),X0\n" + name + " a+0(FP),X0\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+name+"/"+api, func(t *testing.T) { + _, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b uint64)", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "original native storage") { + t.Fatalf("independent FP bytes fabricated native m128 backing/alignment: %v", err) + } + }) + } + } + } +} + +func TestX86LegacyLogicalMemoryVirtualStackNeedsCertificate(t *testing.T) { + for _, target := range x86LegacyLogicalMemoryTargets { + for _, name := range x86LegacyLogicalMemoryNames { + for _, address := range []string{"16(SP)", "16(BP)", "16(AX)(BP*1)"} { + source := "TEXT ·view(SB),4,$0-0\nPXOR X0,X0\nMOVL $0,AX\n" + name + " " + address + ",X0\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+name+"/"+address+"/"+api, func(t *testing.T) { + _, err := x86ScalarFPBitViewAPI(t, source, "func view()", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("uncertified virtual stack became original m128 storage: %v", err) + } + }) + } + } + } + } +} + +func TestX86LegacyLogicalMemoryNativeOpcodeObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86LegacyLogicalMemoryTargets { + var source, declaration strings.Builder + source.WriteString("GLOBL ·data(SB),8,$16\n") + declaration.WriteString("var data [16]byte\n") + for _, name := range x86LegacyLogicalMemoryNames { + move, size := "MOVQ", 8 + if target.arch == "386" { + move, size = "MOVL", 4 + } + for i, address := range []string{"(AX)", "16(AX)(CX*2)", "·data(SB)", "(FS)", "(GS)"} { + fn := fmt.Sprintf("%s%d", strings.ToLower(name), i) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),AX\n%s index+%d(FP),CX\nPXOR X0,X0\n%s %s,X0\nRET\n", fn, size*2, move, move, size, name, address) + fmt.Fprintf(&declaration, "func %s(p *byte,index uintptr)\n", fn) + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declaration.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + for _, name := range x86LegacyLogicalMemoryNames { + want := "asm sideeffect \"" + strings.ToLower(name) + " ($2), $0\", \"=x,0,r,~{memory}\"" + if !strings.Contains(ir, want) { + t.Fatalf("dead result lost original single native %s memory effect", name) + } + for _, segment := range []string{"fs", "gs"} { + want := "asm sideeffect \"" + strings.ToLower(name) + " %" + segment + ":($2), $0\"" + if !strings.Contains(ir, want) { + t.Fatalf("native %s/%s source prefix disappeared", name, segment) + } + } + } + compileLLVMToObject(t, llc, target.triple, "logical-native.ll", "logical-native.o", ir) + }) + } + } +} + +func TestX86LegacyLogicalMemoryPSRetainsSSE1Carrier(t *testing.T) { + llc, objdump := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || objdump == "" { + t.Fatal("LLVM22 llc and objdump required") + } + var source strings.Builder + for _, name := range []string{"ANDPS", "ANDNPS", "ORPS", "XORPS"} { + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\nMOVUPS (AX),X0\n%s (AX),X0\nMOVUPS X0,(BX)\nRET\n", strings.ToLower(name), name) + } + x86LEAActualTargetGoObject(t, "386", "i386-unknown-linux-gnu", source.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, name := range []string{"andps", "andnps", "orps", "xorps"} { + sigs[name] = FuncSig{Name: name, Ret: Void} + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "call <4 x float> asm sideeffect") || strings.Contains(ir, "+sse2") { + t.Fatal("PS native carrier silently requires SSE2") + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + directory := t.TempDir() + input, object := filepath.Join(directory, "sse1.ll"), filepath.Join(directory, "sse1.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, level, "-filetype=obj", input, "-o", object)) + cancel() + if err != nil { + t.Fatalf("SSE1 object: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + output, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, objdump, "-d", object)) + cancel() + if err != nil { + t.Fatalf("SSE1 disassembly: %v\n%s", err, output) + } + for _, name := range []string{"andps", "andnps", "orps", "xorps"} { + if !strings.Contains(string(output), name) { + t.Fatalf("SSE1 object lost %s", name) + } + } + for _, forbidden := range []string{"movdqa", "movdqu", "pand", "pxor", "por", "movapsd", "movq\t%xmm"} { + if strings.Contains(string(output), forbidden) { + t.Fatalf("SSE1 source introduced %s:\n%s", forbidden, output) + } + } + }) + } +} diff --git a/amd64_legacy_mmx_bank.go b/amd64_legacy_mmx_bank.go new file mode 100644 index 000000000..9a2a7f512 --- /dev/null +++ b/amd64_legacy_mmx_bank.go @@ -0,0 +1,233 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86LegacyMMXTable uint8 + +const ( + x86LegacyMMXBinary x86LegacyMMXTable = iota + x86LegacyMMXHorizontal + x86LegacyMMXShuffle + x86LegacyMMXCount +) + +type x86LegacyMMXReadSpec struct { + table x86LegacyMMXTable + opcode, immediateOpcode, opcodeDigit byte +} + +// This is the complete integer MMX-read class in Go's ymm, ymmxmm0f38, +// ymshuf and yps tables. MOVQ, PINSR, PEXTR and floating conversions have +// different ordered operand roles and must not use this bank selector. +var x86LegacyMMXReadSpecs = map[Op]x86LegacyMMXReadSpec{ + "PACKSSLW": {opcode: 0x6b}, + "PACKSSWB": {opcode: 0x63}, + "PACKUSWB": {opcode: 0x67}, + "PADDB": {opcode: 0xfc}, + "PADDL": {opcode: 0xfe}, + "PADDW": {opcode: 0xfd}, + "PADDSB": {opcode: 0xec}, + "PADDSW": {opcode: 0xed}, + "PADDUSB": {opcode: 0xdc}, + "PADDUSW": {opcode: 0xdd}, + "PAND": {opcode: 0xdb}, + "PANDN": {opcode: 0xdf}, + "POR": {opcode: 0xeb}, + "PXOR": {opcode: 0xef}, + "PAVGB": {opcode: 0xe0}, + "PAVGW": {opcode: 0xe3}, + "PCMPEQB": {opcode: 0x74}, + "PCMPEQW": {opcode: 0x75}, + "PCMPEQL": {opcode: 0x76}, + "PCMPGTB": {opcode: 0x64}, + "PCMPGTW": {opcode: 0x65}, + "PCMPGTL": {opcode: 0x66}, + "PMADDWL": {opcode: 0xf5}, + "PMULLW": {opcode: 0xd5}, + "PMULHW": {opcode: 0xe5}, + "PMULHUW": {opcode: 0xe4}, + "PMULULQ": {opcode: 0xf4}, + "PUNPCKHBW": {opcode: 0x68}, + "PUNPCKHWL": {opcode: 0x69}, + "PUNPCKHLQ": {opcode: 0x6a}, + "PUNPCKLBW": {opcode: 0x60}, + "PUNPCKLWL": {opcode: 0x61}, + "PUNPCKLLQ": {opcode: 0x62}, + "PHADDD": {table: x86LegacyMMXHorizontal, opcode: 0x02}, + "PSHUFW": {table: x86LegacyMMXShuffle, opcode: 0x70}, + "PSLLW": {x86LegacyMMXCount, 0xf1, 0x71, 6}, + "PSLLL": {x86LegacyMMXCount, 0xf2, 0x72, 6}, + "PSLLQ": {x86LegacyMMXCount, 0xf3, 0x73, 6}, + "PSRLW": {x86LegacyMMXCount, 0xd1, 0x71, 2}, + "PSRLL": {x86LegacyMMXCount, 0xd2, 0x72, 2}, + "PSRLQ": {x86LegacyMMXCount, 0xd3, 0x73, 2}, + "PSRAW": {x86LegacyMMXCount, 0xe1, 0x71, 4}, + "PSRAL": {x86LegacyMMXCount, 0xe2, 0x72, 4}, +} + +type x86LegacyMMXEncodingRow struct { + classes []string + zcase string + zoffset, byteOffset int +} + +func (s x86LegacyMMXReadSpec) encoderRows() (string, string, []byte, []x86LegacyMMXEncodingRow) { + switch s.table { + case x86LegacyMMXBinary: + return "ymm", "Py1", []byte{s.opcode, 0x66, s.opcode}, []x86LegacyMMXEncodingRow{ + {[]string{"Ymm", "Ymr"}, "Zm_r_xm", 1, 0}, + {[]string{"Yxm", "Yxr"}, "Zm_r_xm", 2, 1}, + } + case x86LegacyMMXHorizontal: + // Go advances z by zoffset+xo; the initial 0x0f makes xo=1. + return "ymmxmm0f38", "Px", []byte{0x0f, 0x38, s.opcode, 0, 0x66, 0x0f, 0x38, s.opcode, 0}, []x86LegacyMMXEncodingRow{ + {[]string{"Ymm", "Ymr"}, "Zlitm_r", 3, 0}, + {[]string{"Yxm", "Yxr"}, "Zlitm_r", 5, 4}, + } + case x86LegacyMMXShuffle: + return "ymshuf", "Pm", []byte{s.opcode, 0}, []x86LegacyMMXEncodingRow{ + {[]string{"Yi8", "Ymm", "Ymr"}, "Zibm_r", 2, 0}, + } + default: + return "yps", "Py3", []byte{s.opcode, s.immediateOpcode, s.opcodeDigit, + 0x66, s.opcode, 0x66, s.immediateOpcode, s.opcodeDigit}, []x86LegacyMMXEncodingRow{ + {[]string{"Ymm", "Ymr"}, "Zm_r_xm", 1, 0}, + {[]string{"Yi8", "Ymr"}, "Zibo_m_xm", 2, 1}, + {[]string{"Yxm", "Yxr"}, "Zm_r_xm", 2, 3}, + {[]string{"Yi8", "Yxr"}, "Zibo_m_xm", 3, 5}, + } + } +} + +type x86LegacyMMXReadForm struct { + canonical Op + spec x86LegacyMMXReadSpec + row int + sourceIndex, destination int + source Operand + byteWidth, memoryBytes int + mmx, gpSpelling bool +} + +func x86LegacyMMXGPBank(reg Reg, arch string) (Reg, bool) { + for index, register := range [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + if reg == register && (arch == "amd64" || arch == "386" && index < 8) { + return Reg(fmt.Sprintf("M%d", index&7)), true + } + } + return "", false +} + +// The selected Go row records physical source identity only. It does not +// define any M register, authorize a native address, model x87 state or +// supply an incoming/caller contract. Raw physical instructions never turn +// a GP source into MMX through Go's source-only ycover compatibility. +func x86LegacyMMXReadProperties(ins Instr, arch string) (x86LegacyMMXReadForm, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + if name == "PADDD" { + name = "PADDL" + } + spec, known := x86LegacyMMXReadSpecs[Op(name)] + if !known { + return x86LegacyMMXReadForm{}, false, nil + } + form := x86LegacyMMXReadForm{canonical: Op(name), spec: spec, sourceIndex: 0, + destination: len(ins.Args) - 1, byteWidth: 16} + fail := func() (x86LegacyMMXReadForm, bool, error) { + return form, true, fmt.Errorf("%s %s outside complete Go integer Ymm read rows: %q", arch, ins.Op, ins.Raw) + } + want := 2 + if spec.table == x86LegacyMMXShuffle { + want, form.sourceIndex = 3, 1 + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != want || ins.Args[form.destination].Kind != OpReg { + return fail() + } + destination := ins.Args[form.destination] + _, form.mmx = amd64ParseMReg(destination.Reg) + if form.mmx { + form.byteWidth = 8 + if arch == "386" && !ins.x86Encoded && (spec.table == x86LegacyMMXBinary || spec.table == x86LegacyMMXCount) { + return fail() + } + } else { + index, valid := amd64ParseXReg(destination.Reg) + if !valid || index >= 16 || arch == "386" && index >= 8 || spec.table == x86LegacyMMXShuffle { + return fail() + } + } + form.source = ins.Args[form.sourceIndex] + immediate := spec.table == x86LegacyMMXCount && form.source.Kind == OpImm + if spec.table == x86LegacyMMXShuffle || immediate { + operand, integer := x86ImmediateShuffleConstant(ins.Args[0], arch, ins.x86Encoded) + if !integer || !amd64SignedImmediate(operand, 8) { + return fail() + } + } + switch { + case immediate: + // An immediate count has no source-bank read. + case form.source.Kind == OpReg && form.mmx: + if _, valid := amd64ParseMReg(form.source.Reg); !valid { + register, valid := x86LegacyMMXGPBank(form.source.Reg, arch) + if !valid || ins.x86Encoded { + return fail() + } + form.source.Reg, form.gpSpelling = register, true + } + case form.source.Kind == OpReg: + index, valid := amd64ParseXReg(form.source.Reg) + if !valid || index >= 16 || arch == "386" && index >= 8 { + return fail() + } + case isAMD64MemoryOperand(form.source) && x86PackedShiftMemoryValid(form.source, arch): + form.memoryBytes = form.byteWidth + if form.mmx && (name == "PUNPCKLBW" || name == "PUNPCKLWL" || name == "PUNPCKLLQ") { + form.memoryBytes = 4 + } + default: + return fail() + } + if spec.table == x86LegacyMMXCount { + if form.mmx { + if immediate { + form.row = 1 + } + } else { + form.row = 2 + if immediate { + form.row = 3 + } + } + } else if !form.mmx { + form.row = 1 + } + return form, true, nil +} + +func x86LegacyMMXReadInstruction(ins Instr, arch string) (Instr, error) { + form, known, err := x86LegacyMMXReadProperties(ins, arch) + if !known || err != nil { + return ins, err + } + local := ins + local.Args = append([]Operand(nil), ins.Args...) + local.Args[form.sourceIndex] = form.source + return local, nil +} + +func x86LegacyMMXReadEmitterContext(ins Instr, arch string) error { + form, known, err := x86LegacyMMXReadProperties(ins, arch) + if !known || err != nil { + return err + } + if form.gpSpelling { + return fmt.Errorf("%w: %s source %s encodes %s without a proved initialized physical MMX bank transport", + ErrProbeNeedsContext, ins.Op, ins.Args[form.sourceIndex].Reg, form.source.Reg) + } + return nil +} diff --git a/amd64_legacy_mmx_bank_test.go b/amd64_legacy_mmx_bank_test.go new file mode 100644 index 000000000..24bd85440 --- /dev/null +++ b/amd64_legacy_mmx_bank_test.go @@ -0,0 +1,555 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +// This census is independent of the implementation. Ordered Go tables, +// opcode prefixes and all opBytes remain available to each comparison. +type x86YMMOriginalRule struct { + name, table, prefix, opBytes string +} + +func x86YMMOriginalRules(t *testing.T) []x86YMMOriginalRule { + t.Helper() + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + var rules []x86YMMOriginalRule + pattern := regexp.MustCompile(`\{A([A-Z0-9]+), (ymm|ymmxmm0f38|ymshuf|yps), (\w+), opBytes\{([^}]+)\}\}`) + for _, row := range pattern.FindAllStringSubmatch(string(source), -1) { + rules = append(rules, x86YMMOriginalRule{row[1], row[2], row[3], row[4]}) + } + if len(rules) != 43 { + t.Fatalf("complete integer MMX-read rule census changed: %d", len(rules)) + } + return rules +} + +func x86YMMReadSource(t *testing.T, row string) Instr { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + return file.Funcs[0].Instrs[1] +} + +func TestX86YMMExistingDescriptorsOriginalBank(t *testing.T) { + var names []string + for _, rule := range x86YMMOriginalRules(t) { + if strings.HasPrefix(rule.name, "PUNPCK") || rule.name == "PAND" || rule.name == "PANDN" || + rule.name == "POR" || rule.name == "PXOR" || rule.table == "yps" && !strings.HasPrefix(rule.name, "PSRA") { + names = append(names, rule.name) + } + } + sort.Strings(names) + if len(names) != 16 { + t.Fatal("logical/unpack/uniform logical-shift descriptor class changed", names) + } + for _, name := range names { + for index, source := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + t.Run(name+"/"+source, func(t *testing.T) { + ins := x86YMMReadSource(t, name+" "+source+",M1") + before := fmt.Sprintf("%#v", ins) + var known bool + var err error + var selected Operand + switch { + case strings.HasPrefix(name, "PUNPCK"): + var form x86PackedUnpackForm + form, known, err = x86PackedUnpackProperties(ins, "amd64") + selected = form.source + case strings.HasPrefix(name, "PS"): + var form x86PackedShiftForm + form, known, err = x86PackedShiftProperties(ins.Op, ins, "amd64") + selected = form.countOperand + default: + var form x86PackedLogicalForm + form, known, err = x86PackedLogicalProperties(ins, "amd64") + selected = form.source + } + if !known || err != nil { + t.Errorf("actual Go MMX-read spelling is absent from pure descriptor: %t/%v", known, err) + } + if err == nil && (selected.Kind != OpReg || selected.Reg != Reg(fmt.Sprintf("M%d", index&7))) { + t.Errorf("family descriptor retained the source GP bank: %+v", selected) + } + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("pure descriptor mutated original source") + } + }) + } + } +} + +func TestX86YMMOriginalGoIntegerReadBanks(t *testing.T) { + registers := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15", "M0", "M7"} + var rows []string + var expected []x86asm.Reg + for _, rule := range x86YMMOriginalRules(t) { + for index, source := range registers { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + rows = append(rows, rule.name+" "+prefix+source+",M1") + bank := index & 7 + if source == "M0" { + bank = 0 + } else if source == "M7" { + bank = 7 + } + expected = append(expected, x86asm.M0+x86asm.Reg(bank)) + } + } + // Alias has independent original Go evidence, not a prefix-name inference. + rows = append(rows, "PADDD AX,M1") + expected = append(expected, x86asm.M0) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+strings.Join(rows, "\n")+"\nRET\n") + t.Logf("complete original Go bytes=%x SHA256=%x", code, sha256.Sum256(code)) + for index, row := range rows { + physical, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal("original object cannot be decoded", row, err) + } + if physical.Args[0] != x86asm.M1 || physical.Args[1] != expected[index] { + t.Fatalf("original Go selected different physical register bank: %s bytes=%x operands=%v expected M1/%s", + row, code[:physical.Len], physical.Args, expected[index]) + } + decoded, err := decodeX86RawDirectiveGroup(code[:physical.Len], 64, 0, row, map[string]bool{}) + if err != nil || len(decoded) != 1 { + t.Fatal("actual physical Go instruction did not decode as one typed row", row, decoded, err) + } + form, known, err := x86LegacyMMXReadProperties(decoded[0], "amd64") + if !known || err != nil || form.gpSpelling || form.source.Reg != Reg(expected[index].String()) { + t.Fatal("raw decoder changed the actual MMX bank into a source spelling", row, decoded, form, known, err) + } + code = code[physical.Len:] + } + if len(code) == 0 || code[0] != 0xc3 || !bytes.Equal(bytes.Trim(code[1:], "\x00"), nil) { + t.Fatalf("original complete native instruction sequence has an unexpected tail: %x", code) + } + t.Logf("actual Go integer MMX-read rows=%d, physical M input independent of GP spelling", len(rows)) +} + +func TestX86YMMSelectorReusableDescriptorBanks(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + for index, register := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + row := rule.name + " " + register + ",M1" + if rule.table == "ymshuf" { + row = rule.name + " $27," + register + ",M1" + } + ins := x86YMMReadSource(t, row) + form, known, err := x86LegacyMMXReadProperties(ins, "amd64") + if !known || err != nil || !form.gpSpelling || !form.mmx || form.row != 0 || + form.source.Reg != Reg(fmt.Sprintf("M%d", index&7)) { + t.Fatal("complete selector lost physical source bank", row, form, known, err) + } + var effect x86TypedByteEffect + if effect, known, err = x86TypedLegacyIntegerArithmeticByteEffect(ins.Op, ins, "amd64"); !known { + effect, known, err = x86TypedPackedWordMultiplyByteEffect(ins.Op, ins, "amd64") + } + if !known { + effect, known, err = x86TypedPackedMultiplyAddByteEffect(ins.Op, ins, "amd64") + } + if known { + if err != nil || len(effect.operands) != 2 || effect.operands[0].operand.Reg != form.source.Reg || + effect.operands[0].width != 8 || !effect.operands[0].read || effect.operands[0].write { + t.Fatal("existing byte descriptor does not share encoded bank selection", row, effect, err) + } + } + if strings.HasPrefix(rule.name, "PACK") { + pack, known, err := x86PackedNarrowProperties(ins, "amd64") + if !known || err != nil || pack.source.Reg != form.source.Reg { + t.Fatal("pack descriptor does not share encoded bank selection", row, pack, known, err) + } + } + } + } +} + +func TestX86YMMSelectorContextIsBeforeIRAndKeepsInput(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + ins := x86YMMReadSource(t, rule.name+" "+prefix+"AX,M1") + file, err := Parse(ArchAMD64, "TEXT probe(SB),4,$0-0\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before, original := output.String(), fmt.Sprintf("%#v", ins) + terminated, err := ctx.lowerInstr(0, 0, ins, nil, nil) + if terminated || !errors.Is(err, ErrProbeNeedsContext) || before != output.String() || original != fmt.Sprintf("%#v", ins) { + t.Fatal("source-bank Context was late or mutated the source/IR", rule.name, terminated, err) + } + } +} + +func TestX86YMMSelectorCompleteOriginalOrderedRules(t *testing.T) { + bytesSource, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + source := string(bytesSource) + if len(x86LegacyMMXReadSpecs) != 43 { + t.Fatal("selector class does not have exactly 43 canonical names") + } + for _, rule := range x86YMMOriginalRules(t) { + t.Run(rule.name, func(t *testing.T) { + spec, found := x86LegacyMMXReadSpecs[Op(rule.name)] + if !found { + t.Fatal("complete original integer read class lacks a selector") + } + table, prefix, opcodeBytes, rows := spec.encoderRows() + var originalBytes []byte + for _, token := range strings.Split(rule.opBytes, ",") { + token = strings.TrimSpace(token) + if token == "Pe" { + token = "0x66" + } + value, err := strconv.ParseUint(token, 0, 8) + if err != nil { + t.Fatal("unknown original opBytes expression", token, err) + } + originalBytes = append(originalBytes, byte(value)) + } + if table != rule.table || prefix != rule.prefix || !bytes.Equal(opcodeBytes, originalBytes) { + t.Fatalf("selected encoding metadata differs from original Go: %s/%s %x; original=%+v %x", + table, prefix, opcodeBytes, rule, originalBytes) + } + block := regexp.MustCompile(`(?s)var ` + table + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(source) + if len(block) != 2 { + t.Fatal("original ordered table was not found", table) + } + pattern := regexp.MustCompile(`\{(Z\w+), (\d+), argList\{([^}]+)\}\}`) + originalRows := pattern.FindAllStringSubmatch(block[1], -1) + if len(rows) != len(originalRows) { + t.Fatal("original ordered table row count changed") + } + byteOffset, xo := 0, 0 + if originalBytes[0] == 0x0f { + xo = 1 + } + for index, original := range originalRows { + zoffset, _ := strconv.Atoi(original[2]) + classes := strings.Split(strings.ReplaceAll(original[3], " ", ""), ",") + row := rows[index] + if row.zcase != original[1] || row.zoffset != zoffset || row.byteOffset != byteOffset || + !reflect.DeepEqual(row.classes, classes) { + t.Fatalf("ordered row %d differs: %+v original=%v offset=%d", index, row, original, byteOffset) + } + byteOffset += zoffset + xo + } + }) + } +} + +func TestX86YMMSelectorCompleteRolesAndImmutability(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + for destination := 0; destination < 8; destination++ { + for source := 0; source < 8; source++ { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + ins := x86YMMReadSource(t, fmt.Sprintf("%s %sM%d,M%d", rule.name, prefix, source, destination)) + before := fmt.Sprintf("%#v", ins) + form, known, err := x86LegacyMMXReadProperties(ins, "amd64") + if !known || err != nil || !form.mmx || form.gpSpelling || form.row != 0 || + form.byteWidth != 8 || form.memoryBytes != 0 || form.source.Reg != Reg(fmt.Sprintf("M%d", source)) { + t.Fatal("literal MMX source/destination role changed", ins, form, known, err) + } + local, err := x86LegacyMMXReadInstruction(ins, "amd64") + if err != nil || before != fmt.Sprintf("%#v", ins) { + t.Fatal("selector mutated original", err) + } + local.Args[form.sourceIndex].Reg = "M6" + if before != fmt.Sprintf("%#v", ins) { + t.Fatal("normalized operand slice aliases original") + } + } + } + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + for _, bank := range []string{"M", "X"} { + if bank == "X" && rule.table == "ymshuf" { + continue + } + ins := x86YMMReadSource(t, rule.name+" "+prefix+"(AX),"+bank+"1") + form, known, err := x86LegacyMMXReadProperties(ins, "amd64") + want := 16 + if bank == "M" { + want = 8 + if strings.HasPrefix(rule.name, "PUNPCKL") { + want = 4 + } + } + if !known || err != nil || form.memoryBytes != want || form.gpSpelling { + t.Fatal("complete native read shape changed", ins, form, known, err) + } + if rule.table == "yps" { + for _, count := range []int{-128, 0, 127} { + ins = x86YMMReadSource(t, fmt.Sprintf("%s $%d,%s1", rule.name, count, bank)) + form, known, err = x86LegacyMMXReadProperties(ins, "amd64") + if !known || err != nil || form.row != 1 && form.row != 3 || form.memoryBytes != 0 || form.gpSpelling { + t.Fatal("immediate row borrowed a bank read", ins, form, known, err) + } + } + } + } + } +} + +func TestX86YMMSelectorActualGoModeAndRoleBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + row := rule.name + " " + prefix + "AX,M1" + source := "TEXT probe(SB),4,$0-0\n" + row + "\nRET\n" + want := arch == "amd64" || rule.table == "ymshuf" || rule.table == "ymmxmm0f38" + goAssemblerOracleOutput(t, arch, source, "", want) + form, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), arch) + if !known || (err == nil) != want || want && (form.source.Reg != "M0" || !form.gpSpelling) { + t.Fatal("actual named mode admission differs from selector", arch, row, form, known, err) + } + if want { + code := x86PackedShiftActualGoBytes(t, arch, source) + mode := 64 + if arch == "386" { + mode = 32 + } + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Args[0] != x86asm.M1 || physical.Args[1] != x86asm.M0 { + t.Fatal("actual accepted mode did not select physical MMX", arch, row, physical, err) + } + } + } + } + for _, row := range []string{"PAND M0,X1", "PAND X0,M1", "PAND AX,X1", "PAND M0,AX", + "PAND R8,M1", "PSHUFW $128,M0,M1", "PSHUFW $-129,M0,M1", "PSRLW $128,X1"} { + arch := "amd64" + if row == "PAND R8,M1" { + arch = "386" + } + goAssemblerOracleOutput(t, arch, "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n", "", false) + _, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), arch) + if !known || err == nil { + t.Fatal("original Go rejected row obtained a pure bank form", arch, row, known, err) + } + } +} + +func TestX86YMMSelectorActualMode32GPSpellings(t *testing.T) { + for _, name := range []string{"PHADDD", "PSHUFW"} { + for index, register := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + prefix := "" + if name == "PSHUFW" { + prefix = "$27," + } + row := name + " " + prefix + register + ",M1" + source := "TEXT probe(SB),4,$0-0\n" + row + "\nRET\n" + want := index < 8 + goAssemblerOracleOutput(t, "386", source, "", want) + form, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), "386") + if !known || (err == nil) != want || want && form.source.Reg != Reg(fmt.Sprintf("M%d", index)) { + t.Fatal("mode32 original GP class does not match selected MMX bank", row, form, known, err) + } + } + } +} + +func TestX86YMMSelectorExcludesRealGPRolesAndPhysicalForgery(t *testing.T) { + for _, row := range []string{"MOVQ AX,M1", "PINSRB $0,AX,X1", "PINSRD $0,AX,X1", "PINSRQ $0,AX,X1", + "PEXTRB $0,X1,AX", "PEXTRD $0,X1,AX", "PEXTRQ $0,X1,AX", "CVTPL2PS M0,X1", "CVTPL2PD M0,X1"} { + _, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), "amd64") + if known || err != nil { + t.Fatal("different ordered GP/x87 role was claimed by integer read selector", row, known, err) + } + } + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + ins := x86YMMReadSource(t, rule.name+" "+prefix+"AX,M1") + ins.x86Encoded = true + if _, known, err := x86LegacyMMXReadProperties(ins, "amd64"); !known || err == nil { + t.Fatal("physical decoder identity acquired source-only GP compatibility", rule.name) + } + ins.Args[len(ins.Args)-2].Reg = "M0" + if form, known, err := x86LegacyMMXReadProperties(ins, "386"); !known || err != nil || form.gpSpelling { + t.Fatal("actual physical MMX input was changed into a source alias", rule.name, form, known, err) + } + } + for _, row := range []string{"PAND.BAD AX,M1", "PAND AX,M8", "PSHUFW $0,AX,X1"} { + if _, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), "amd64"); !known || err == nil { + t.Fatal("malformed selector identity was accepted", row) + } + } +} + +func TestX86YMMSelectorDoesNotGrantEmitterOrCallerAuthority(t *testing.T) { + for _, rule := range x86YMMOriginalRules(t) { + prefix := "" + if rule.table == "ymshuf" { + prefix = "$27," + } + row := rule.name + " " + prefix + "AX,M1" + ins := x86YMMReadSource(t, row) + if !errors.Is(x86LegacyMMXReadEmitterContext(ins, "amd64"), ErrProbeNeedsContext) || x86CallerVectorMoveKnown(ins) { + t.Fatal("bank identity alone acquired transport or caller permission", row) + } + ins.Args[len(ins.Args)-2].Reg = "M0" + state := x86SourceStackState{vectors: map[Reg]uint64{}, masks: map[Reg]bool{}} + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, &x86GoCallerRegionProof{}) { + t.Fatal("undefined MMX input acquired a caller value", rule.name) + } + pkg := mustGoPackage(t, "probe", "package probe\nfunc probe()\n") + source := "TEXT ·probe(SB),4,$0-0\n" + row + "\nRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{"probe": {Ret: Void}}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("text API granted unproved MMX source transport", row, triple, err) + } + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("module API granted unproved MMX source transport", row, triple, err) + } + result, err := translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go API granted unproved MMX source transport", row, triple, err) + } + }() + } + } +} + +func TestX86YMMSelectorFiveTargetRegisteredXMMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc", + "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT ·probe(SB),4,$0-16\nMOVOU value+0(FP),X0\nMOVOU value+0(FP),X1\n" + count := 0 + for _, rule := range x86YMMOriginalRules(t) { + if rule.table == "ymshuf" { + continue // This original row is MMX-only, not an XMM grant. + } + source += rule.name + " X0,X1\n" + count++ + } + source += "RET\n" + if count != 42 { + t.Fatal("complete original XMM subdomain changed", count) + } + goAssemblerOracleOutput(t, arch, source, "", true) + pkg := mustGoPackage(t, "probe", "package probe\nfunc probe(value [16]byte)\n") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sig, err := DeriveGoFuncABI0Signature("probe", pkg.Types.Scope().Lookup("probe").(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"probe": sig}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(triple, err) + } + x86ResultInitializationObject(t, llc, triple, ir, "ymm-selector-text") + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(triple, err) + } + x86ResultInitializationObject(t, llc, triple, module.String(), "ymm-selector-module") + result, err := translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(triple, err) + } + x86ResultInitializationObject(t, llc, triple, result.Module.String(), "ymm-selector-go") + }() + } +} + +func TestX86YMMOriginalGoOrderedGPControls(t *testing.T) { + for _, row := range []string{"MOVQ AX,M1", "PINSRB $0,AX,X1", "PINSRD $0,AX,X1", "PINSRQ $0,AX,X1", + "PEXTRB $0,X1,AX", "PEXTRD $0,X1,AX", "PEXTRQ $0,X1,AX"} { + t.Run(row, func(t *testing.T) { + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT probe(SB),4,$0-0\n"+row+"\nRET\n") + physical, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + for _, arg := range physical.Args { + if arg == x86asm.M0 { + t.Fatal("actual transfer/insertion/extraction GP role was changed to MMX", physical) + } + } + if reflect.DeepEqual(physical.Args, x86asm.Args{}) { + t.Fatal("control has no actual decoded operands") + } + }) + } +} diff --git a/amd64_legacy_mmx_mode32_test.go b/amd64_legacy_mmx_mode32_test.go new file mode 100644 index 000000000..d57e394d0 --- /dev/null +++ b/amd64_legacy_mmx_mode32_test.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86YMMMode32LegacyShiftActualGoRows(t *testing.T) { + var names []string + for _, rule := range x86YMMOriginalRules(t) { + if rule.table == "yps" { + names = append(names, rule.name) + } + } + if len(names) != 8 { + t.Fatal("complete original yps class changed", names) + } + + for _, arch := range []string{"amd64", "386"} { + for _, name := range names { + for _, bank := range []string{"M", "X"} { + for _, source := range []string{"$3", bank + "0", "8(AX)"} { + row := fmt.Sprintf("%s %s,%s1", name, source, bank) + t.Run(arch+"/"+row, func(t *testing.T) { + text := "TEXT probe(SB),4,$0-0\n" + row + "\nRET\n" + want := arch == "amd64" || bank == "X" + goAssemblerOracleOutput(t, arch, text, "", want) + t.Logf("original source SHA256=%x accepted=%t", sha256.Sum256([]byte(text)), want) + if want { + code := x86PackedShiftActualGoBytes(t, arch, text) + mode := 64 + if arch == "386" { + mode = 32 + } + physical, err := x86asm.Decode(code, mode) + destination := x86asm.M1 + if bank == "X" { + destination = x86asm.X1 + } + if err != nil || physical.Args[0] != destination { + t.Fatal("original Go selected a different physical destination", row, code, physical, err) + } + t.Logf("original Go bytes=%x SHA256=%x physical=%v", code, sha256.Sum256(code), physical) + } + + form, known, err := x86LegacyMMXReadProperties(x86YMMReadSource(t, row), arch) + if !known || (err == nil) != want { + t.Fatal("named Go mode admission differs from pure selector", row, form, known, err) + } + if !want { + file, err := Parse(ArchAMD64, text) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"probe": {Name: "probe", Ret: Void}}, + }); err == nil { + t.Fatal("Go-rejected named mode32 MMX row produced LLVM IR", row) + } + } + }) + } + } + } + } +} diff --git a/amd64_legacy_move_native_linux_amd64_test.go b/amd64_legacy_move_native_linux_amd64_test.go new file mode 100644 index 000000000..2a593e373 --- /dev/null +++ b/amd64_legacy_move_native_linux_amd64_test.go @@ -0,0 +1,9 @@ +package plan9asm + +import "testing" + +// Unlike Rosetta diagnostics, this required native CI axis also compares the +// original aligned-SSE faults for every nonzero misalignment. +func TestCrossLinuxRuntimeMatrixX86LegacyMoveNativeAlignment(t *testing.T) { + x86LegacyMoveNativeRuntime(t, true) +} diff --git a/amd64_legacy_move_native_runtime_unix_test.go b/amd64_legacy_move_native_runtime_unix_test.go new file mode 100644 index 000000000..ed8a32d3e --- /dev/null +++ b/amd64_legacy_move_native_runtime_unix_test.go @@ -0,0 +1,201 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86LegacyMoveNativeBytesAndFaults(t *testing.T) { + x86LegacyMoveNativeRuntime(t, false) +} + +func x86LegacyMoveNativeRuntime(t *testing.T, nativeAlignment bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + boolInt := func(value bool) int { + if value { + return 1 + } + return 0 + } + var source, declarations, originals, pairs strings.Builder + sigs := map[string]FuncSig{} + for index, row := range x86LegacyMoveNativeOracle { + for _, store := range []bool{false, true} { + for _, dead := range []bool{false, true} { + name := fmt.Sprintf("move%d_%t_%t", index, store, dead) + source.WriteString(x86LegacyMoveNativeSource("amd64", name, row.name, store, dead)) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,uint8_t*);\n", name) + operands := "(DI),X1" + before, after := "stc\\n\\t", "movups %%xmm1,(%1)\\n\\t" + if store { + operands = "X1,(DI)" + before, after = "movups (%1),%%xmm1\\n\\tstc\\n\\t", "" + } else if dead { + after = "" + } + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row.name+" "+operands+"\nRET\n") + if len(code) < 4 || code[len(code)-1] != 0xc3 { + t.Fatal("missing original Go opcode and ordinary RET", code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&originals, `static void original_%s(uint8_t *p,uint8_t *out) { + __asm__ volatile("%s.byte %s\n\t%ssetc 16(%%1)" + : : "D"(p),"r"(out) : "xmm1","memory","cc"); +} +`, name, before, strings.Join(bytes, ","), after) + fmt.Fprintf(&pairs, "{original_%s,%s,%d,%d,%d,16,0},\n", name, name, + boolInt(row.aligned), boolInt(store), boolInt(dead)) + } + } + for lane := 0; lane < 4; lane++ { + name := fmt.Sprintf("move_result%d_%d", index, lane) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-12\nMOVQ p+0(FP),AX\n%s (AX),X1\nPEXTRD $%d,X1,CX\nMOVL CX,ret+8(FP)\nRET\n", name, row.name, lane) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: I32, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I32, Index: 0, Field: -1}}, + }} + fmt.Fprintf(&declarations, `extern uint32_t %s(uint8_t*); +static void translated_%s(uint8_t *p,uint8_t *out) { + uint32_t value=%s(p); memcpy(out,&value,4); out[16]=1; +} +`, name, name, name) + code := x86PackedShiftActualGoBytes(t, "amd64", fmt.Sprintf("TEXT f(SB),4,$0-0\n%s (DI),X1\nPEXTRD $%d,X1,AX\nRET\n", row.name, lane)) + if len(code) < 8 || code[len(code)-1] != 0xc3 { + t.Fatal("missing original Go load/extract/RET", code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + fmt.Fprintf(&originals, `static void original_%s(uint8_t *p,uint8_t *out) { + uint32_t value; + __asm__ volatile(".byte %s" : "=a"(value) : "D"(p) : "xmm1","memory"); + memcpy(out,&value,4); out[16]=1; +} +`, name, strings.Join(bytes, ",")) + fmt.Fprintf(&pairs, "{original_%s,translated_%s,%d,0,0,4,%d},\n", name, name, boolInt(row.aligned), lane*4) + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const helper = ` +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*operation)(uint8_t*,uint8_t*); +struct pair { operation original,translated; int aligned,store,dead,width,offset; }; +static int outcome(operation fn,uint8_t *p) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + uint8_t data[32]; memset(data,0x53,sizeof(data)); + fn(p,data); _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + return WIFSIGNALED(status)?WTERMSIG(status):-2; +} +static int check_bytes(struct pair pair,int offset) { + for(int sample=0;sample<257;++sample) for(int rc=0;rc<4;++rc) { + _Alignas(16) uint8_t a[64],b[64],x[64],y[64]; + for(int i=0;i<64;++i) { + a[i]=b[i]=(uint8_t)(sample*71+i*19); + x[i]=y[i]=(uint8_t)(sample*31-i*59); + } + uint8_t before_a[64],before_x[64]; + memcpy(before_a,a,64); memcpy(before_x,x,64); + uint32_t saved=_mm_getcsr(),csr=0x1f80|(rc<<13)|(sample&63); + _mm_setcsr(csr); pair.original(a+offset,x+1); uint32_t original_csr=_mm_getcsr(); + _mm_setcsr(csr); pair.translated(b+offset,y+1); uint32_t translated_csr=_mm_getcsr(); + _mm_setcsr(saved); + if(original_csr!=csr || translated_csr!=csr || memcmp(a,b,64) || memcmp(x,y,64)) return 1; + for(int i=0;i<64;++i) { + uint8_t want_a=pair.store && i>=offset && i=1 && i<1+pair.width?before_a[offset+pair.offset+i-1]:before_x[i]; + if(i==17) want_x=1; + if(a[i]!=want_a || x[i]!=want_x) return 2; + } + } + return 0; +} +` + main := fmt.Sprintf("#define CHECK_ALIGNMENT %d\n", boolInt(nativeAlignment)) + helper + declarations.String() + originals.String() + "static const struct pair pairs[]={\n" + pairs.String() + `}; +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<128) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op +#include +extern void observe(uint32_t*); +int main(void) { + uint32_t out[6] = {77,0,0,0,0,88}; + observe(out+1); + for(unsigned i=1;i<=4;i++) if(out[i]!=i) { + fprintf(stderr,"result field %u: got %u, want %u\n",i-1,out[i],i); + return 1; + } + return out[0]!=77 || out[5]!=88; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "packed-fp-results", ir, main, level) + }) + } + }) + } + }) + } +} diff --git a/amd64_legacy_packed_move_runtime_test.go b/amd64_legacy_packed_move_runtime_test.go new file mode 100644 index 000000000..8dad27d8f --- /dev/null +++ b/amd64_legacy_packed_move_runtime_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86LegacyPackedMoveOriginalBytesAndCanaries(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, goTest, reference, checks strings.Builder + declarations.WriteString("package packedmove\nfunc ReadCSR() uint32\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\n") + goTest.WriteString(`package packedmove +import ( + "testing" + "unsafe" +) +func aligned(value *[80]byte) (*[16]byte,int) { + offset:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15) + return (*[16]byte)(unsafe.Pointer(&value[offset])),offset +} +func TestOriginal(t *testing.T) { + for sample:=0;sample<257;sample++ { for _,seed:=range []uint64{0,0x8d5} { + var x,y,out [80]byte + xp,_:=aligned(&x);yp,_:=aligned(&y);op,offset:=aligned(&out) + for i:=range x { x[i]=byte(sample*71+i*37);y[i]=byte(sample*19+i*11);out[i]=0xa5 } + originalX,originalY:=x,y +`) + reference.WriteString(`#include +#include +#include +#include +struct Aligned { _Alignas(16) uint8_t before[16],value[16],after[16]; }; +`) + for _, op := range x86LegacyPackedMoveTestNames { + for _, kind := range []string{"Register", "Alias", "Load", "Store"} { + name := op + kind + operation, input := op+" X1,X0\nMOVOU X0,(DX)", "xp" + switch kind { + case "Alias": + operation, input = op+" X0,X0\nMOVOU X0,(DX)", "yp" + case "Load": + operation = op + " (AX),X0\nMOVOU X0,(DX)" + case "Store": + operation = op + " X1,(DX)" + } + fmt.Fprintf(&declarations, "func %s(out,x,y *[16]byte,flags *uint64,seed uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),DX\nMOVQ x+8(FP),AX\nMOVQ y+16(FP),CX\nMOVOU (AX),X1\nMOVOU (CX),X0\nPUSHQ seed+32(FP)\nPOPFQ\n%s\nPUSHFQ\nPOPQ BX\nMOVQ flags+24(FP),SI\nMOVQ BX,(SI)\nRET\n", name, operation) + fmt.Fprintf(&goTest, `{ + for i:=range out { out[i]=0xa5 } + box:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=ReadCSR() + %s(op,xp,yp,&box.value,seed) + if box.value&0x8d5!=seed||box.before!=0x1122334455667788||box.after!=0x8877665544332211||ReadCSR()!=csr||x!=originalX||y!=originalY { t.Fatalf("%s state/canary sample%%d flags%%x",sample,box.value) } + for i,value:=range out { + want:=byte(0xa5) + if i>=offset&&i\n#include \n#include \n") + constant := func(multiplier, addend int) string { + var values []string + for index := 0; index < 64; index++ { + values = append(values, fmt.Sprintf("i8 %d", byte(index*multiplier+addend))) + } + return "<" + strings.Join(values, ",") + ">" + } + for _, op := range x86LegacyPackedMoveTestNames { + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + fn := Func{Instrs: []Instr{ + {Op: "VMOVUPS", Args: []Operand{{Kind: OpReg, Reg: "Z0"}, {Kind: OpReg, Reg: "Z1"}}}, ins, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, fn, FuncSig{Ret: Void}, nil) + if err := ctx.storeZ("Z0", constant(37, 11)); err != nil { + t.Fatal(err) + } + if err := ctx.storeZ("Z1", constant(19, 71)); err != nil { + t.Fatal(err) + } + known, term, err := ctx.lowerFP(ins.Op, ins) + if !known { + known, term, err = ctx.lowerVec(ins.Op, ins) + } + if !known || term || err != nil { + t.Fatal("legacy lowerer did not consume its typed X-bank shape", known, term, err) + } + first, err := ctx.loadZ("Z0") + if err != nil { + t.Fatal(err) + } + second, err := ctx.loadZ("Z1") + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(&ir, "define void @%sShadow(ptr %%out) {\n%s store <64 x i8> %s, ptr %%out, align 1\n %%next = getelementptr i8, ptr %%out, i64 64\n store <64 x i8> %s, ptr %%next, align 1\n ret void\n}\n", op, output.String(), first, second) + fmt.Fprintf(&reference, "extern void %sShadow(uint8_t*);\n", op) + fmt.Fprintf(&checks, "{memset(out,0xa5,sizeof(out));%sShadow(out+16);for(int i=0;i<64;i++){uint8_t want=(uint8_t)(i<16?i*19+71:i*37+11);if(out[16+i]!=want||out[80+i]!=(uint8_t)(i*19+71)){fprintf(stderr,\"%s shadow byte%%d\\n\",i);return 1;}}for(int i=0;i<16;i++)if(out[i]!=0xa5||out[144+i]!=0xa5)return 2;}\n", op, op) + } + reference.WriteString("int main(void){uint8_t out[160];\n" + checks.String() + "return 0;}\n") + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "legacy-packed-move-alias", ir.String(), reference.String(), level) + }) + } +} diff --git a/amd64_legacy_ps_insert_extract_test.go b/amd64_legacy_ps_insert_extract_test.go new file mode 100644 index 000000000..fba8a59f5 --- /dev/null +++ b/amd64_legacy_ps_insert_extract_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86LegacyPSIndependentFPFields(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + const source = "TEXT fields(SB),4,$0-12\nPXOR X0,X0\nINSERTPS $0,a+0(FP),X0\nEXTRACTPS $0,X0,ret+8(FP)\nRET\n" + + "TEXT stores(SB),4,$0-12\nMOVL $0x87654321,CX\nMOVD CX,X1\nEXTRACTPS $0,X1,a+0(FP)\nMOVBLZX a+0(FP),AX\nMOVBLZX b+1(FP),BX\nSHLL $8,BX\nORL BX,AX\nMOVBLZX c+2(FP),BX\nSHLL $16,BX\nORL BX,AX\nMOVBLZX d+3(FP),BX\nSHLL $24,BX\nORL BX,AX\nMOVL AX,ret+8(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + pkg := mustGoPackage(t, "oracle", "package oracle\nfunc fields(a,b,c,d byte) uint32\nfunc stores(a,b,c,d byte) uint32\n") + sigs := map[string]FuncSig{} + for _, name := range []string{"fields", "stores"} { + decl := pkg.Types.Scope().Lookup(name).(*types.Func) + sig, err := DeriveGoFuncABI0Signature(name, decl, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + if len(sig.Frame.Params) != 4 || len(sig.Frame.Results) != 1 || sig.Frame.Results[0].Offset != 8 { + t.Fatalf("actual AMD64 ABI0 four-byte fields/result layout changed: %+v", sig.Frame) + } + sigs[name] = sig + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const reference = `#include +#include +extern uint32_t fields(uint8_t,uint8_t,uint8_t,uint8_t); +extern uint32_t stores(uint8_t,uint8_t,uint8_t,uint8_t); +int main(void) { + unsigned failures=0; + for(unsigned i=0;i<256;i++) { + uint8_t a=i,b=i*37,c=i*59,d=i*71; + uint32_t want=(uint32_t)a|((uint32_t)b<<8)|((uint32_t)c<<16)|((uint32_t)d<<24); + uint32_t got=fields(a,b,c,d); + if(got!=want){if(!(failures&1))fprintf(stderr,"legacy FP fields got=%x want=%x\n",got,want);failures|=1;} + got=stores(a,b,c,d); + if(got!=UINT32_C(0x87654321)){if(!(failures&2))fprintf(stderr,"legacy FP stores got=%x\n",got);failures|=2;} + } + return failures; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy-ps-independent-fp", triple, ir, reference, prefix) +} + +func TestX86LegacyPSInsertExtractSourceCallerReachesObjects(t *testing.T) { + for _, body := range []string{ + "VMOVUPS (DI),Z0\nVMOVUPS (SI),X1\nINSERTPS $64,X1,X0\nVMOVSS X0,(DX)", + "VMOVUPS (DI),X0\nEXTRACTPS $3,X0,AX\nMOVL AX,(DX)", + } { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatalf("%s: %v", body, err) + } + } +} + +func TestX86LegacyPSInsertedByteKnowledgePreservesUpper(t *testing.T) { + for _, missing := range []int{-1, 0, 12, 16, 31, 63} { + const row = "INSERTPS $0,X0,X1" + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + f, recognized, err := x86InsertExtractProperties(ins, "amd64") + if !recognized || err != nil { + t.Fatalf("legacy shared grammar missing: %t/%v", recognized, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 15, "Z1": ^uint64(0)}, masks: map[Reg]bool{}} + if missing >= 0 { + state.vectors["Z1"] &^= uint64(1) << uint(missing) + } + if !x86CallerInsertExtractStep(&ins, f, &state, nil) { + t.Fatal("defined inserted bytes rejected") + } + want := ^uint64(0) + if missing >= 4 { + want &^= uint64(1) << uint(missing) + } + if got := state.vectors["Z1"]; got != want { + t.Fatal(fmt.Sprintf("legacy upper knowledge %d: got%x want%x", missing, got, want)) + } + } +} + +func TestX86LegacyPSCallerUnknownInputsAndPermissions(t *testing.T) { + for name, body := range map[string]string{ + "unknown inserted lane": "VMOVUPS (DI),X1\nINSERTPS $64,X0,X1\nVMOVSS X1,(DX)", + "unknown retained base": "VMOVUPS (DI),X0\nINSERTPS $16,X0,X1\nVMOVSS X1,(DX)", + "unknown extract lane": "EXTRACTPS $0,X0,AX\nMOVL AX,(DX)", + "unknown memory even all clear": "INSERTPS $15,(BX),X0\nVMOVSS X0,(DX)", + "pointer FP source": "VMOVUPS (DI),X1\nINSERTPS $0,a+0(FP),X1\nVMOVSS X1,(DX)", + "source SP": "VMOVUPS (DI),X1\nINSERTPS $0,(SP),X1\nVMOVSS X1,(DX)", + "input write": "VMOVUPS (DI),X0\nEXTRACTPS $0,X0,(DI)", + "g write": "VMOVUPS (DI),X0\nEXTRACTPS $0,X0,R14", + "missing CFG source": "CMPQ AX,$0\nJE join\nVMOVUPS (DI),X0\njoin:\nEXTRACTPS $0,X0,AX\nMOVL AX,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("legacy unknown source acquired caller permission: %v", err) + } + }) + } +} diff --git a/amd64_legacy_scalar_move_alias_runtime_test.go b/amd64_legacy_scalar_move_alias_runtime_test.go new file mode 100644 index 000000000..52044a034 --- /dev/null +++ b/amd64_legacy_scalar_move_alias_runtime_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +type x86LegacyScalarAliasForm struct { + name, move string + width int + register bool + self bool +} + +func x86LegacyScalarAliasForms() []x86LegacyScalarAliasForm { + return []x86LegacyScalarAliasForm{ + {name: "loadSS", move: "MOVSS", width: 4}, + {name: "copySS", move: "MOVSS", width: 4, register: true}, + {name: "selfSS", move: "MOVSS", width: 4, register: true, self: true}, + {name: "loadSD", move: "MOVSD", width: 8}, + {name: "copySD", move: "MOVSD", width: 8, register: true}, + {name: "selfSD", move: "MOVSD", width: 8, register: true, self: true}, + } +} + +func x86LegacyScalarAliasAssembly(wide int, goarch string) string { + var source strings.Builder + word, pointerMove := 8, "MOVQ" + if goarch == "386" { + word, pointerMove = 4, "MOVL" + } + register := "Y0" + if wide == 64 { + register = "Z0" + } + for _, form := range x86LegacyScalarAliasForms() { + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s input+0(FP),AX\n%s output+%d(FP),BX\nVMOVUPS (AX),%s\n", + form.name, word*2, pointerMove, pointerMove, word, register) + switch { + case form.self: + fmt.Fprintf(&source, "%s X0,X0\n", form.move) + case form.register: + fmt.Fprintf(&source, "%s 64(AX),X1\n%s X1,X0\n", form.move, form.move) + default: + fmt.Fprintf(&source, "%s 64(AX),X0\n", form.move) + } + // A store must not alter the register or any overlapping view. + fmt.Fprintf(&source, "%s X0,128(BX)\nMOVUPS X0,(BX)\nVMOVUPS Y0,16(BX)\n", form.move) + if wide == 64 { + source.WriteString("VMOVUPS Z0,48(BX)\n") + } + source.WriteString("RET\n") + } + return source.String() +} + +func x86LegacyScalarAliasIR(t *testing.T, wide int, goarch, triple string) string { + t.Helper() + assembly := x86LegacyScalarAliasAssembly(wide, goarch) + requireX86GoAssemblerResult(t, goarch, assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + word := int64(8) + if goarch == "386" { + word = 4 + } + for _, form := range x86LegacyScalarAliasForms() { + symbol := "scalar." + form.name + sigs[symbol] = FuncSig{ + Name: symbol, Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: word, Type: Ptr, Index: 1, Field: -1}, + }}, + } + } + ir, err := Translate(file, Options{ + Goarch: goarch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "scalar.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + return ir +} + +func TestX86LegacyScalarMoveAliasFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, wide := range []int{32, 64} { + for _, target := range []struct{ goarch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i386-pc-windows-msvc"}, + } { + t.Run(fmt.Sprintf("%d/%s", wide, target.triple), func(t *testing.T) { + ir := x86LegacyScalarAliasIR(t, wide, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "legacy-alias.ll", "legacy-alias.o", ir) + }) + } + } +} + +func x86LegacyScalarAliasActualGoOracle(t *testing.T, wide int) { + t.Helper() + var declarations, tests strings.Builder + declarations.WriteString("package scalar\nimport \"unsafe\"\n") + tests.WriteString("package scalar\nimport (\"testing\"; \"unsafe\")\nfunc TestFullViews(t *testing.T) {\n") + for _, form := range x86LegacyScalarAliasForms() { + fmt.Fprintf(&declarations, "func %s(input, output unsafe.Pointer)\n", form.name) + fmt.Fprintf(&tests, ` + for iteration := 0; iteration < 64; iteration++ { + var input [80]byte + var output [160]byte + for index := range input { + input[index] = byte(index*17 + iteration) + } + for index := range output { + output[index] = byte(index*7 + iteration) + } + beforeInput, want := input, output + for _, view := range [][2]int{{0,16},{16,32},{48,%[4]d}} { + if view[0] == 48 && %[4]d != 64 { + continue + } + for lane := 0; lane < view[1]; lane++ { + value := input[lane] + if !%[3]t && lane < %[2]d { + value = input[64+lane] + } else if !%[3]t && !%[5]t && lane < 16 { + value = 0 + } + want[16+view[0]+lane] = value + } + } + for lane := 0; lane < %[2]d; lane++ { + value := input[64+lane] + if %[3]t { + value = input[lane] + } + want[144+lane] = value + } + %[1]s(unsafe.Pointer(&input[0]), unsafe.Pointer(&output[16])) + if output != want || input != beforeInput { + t.Fatalf("%[1]s iteration=%%d got=%%x want=%%x inputChanged=%%t", iteration, output, want, input != beforeInput) + } + } +`, form.name, form.width, form.self, wide, form.register) + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module scalar\n\ngo 1.20\n", "scalar.go": declarations.String(), + "scalar_amd64.s": x86LegacyScalarAliasAssembly(wide, "amd64"), "scalar_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + command.WaitDelay = 2 * time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go %d-byte legacy alias oracle: %v\n%s", wide, err, output) + } +} + +func TestCrossLinuxRuntimeMatrixX86LegacyScalarMoveAliasViews(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + // This original Go counterpart requires AVX, not AVX512. ZMM encodings + // below are independently Go-assembled and portable LLVM/C checked; that + // does not claim that the host executes those original AVX512 instructions. + x86LegacyScalarAliasActualGoOracle(t, 32) + for _, wide := range []int{32, 64} { + var source strings.Builder + source.WriteString(`#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86LegacyScalarAliasForms() { + fmt.Fprintf(&source, "extern void %s(void*,void*) __asm__(LABEL(\"scalar.%s\"));\n", form.name, form.name) + } + source.WriteString("int main(void) {\n") + for _, form := range x86LegacyScalarAliasForms() { + register, self := 0, 0 + if form.register { + register = 1 + } + if form.self { + self = 1 + } + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 64; iteration++) { + uint8_t input[80], output[160], before_input[80], want[160]; + for (unsigned index = 0; index < sizeof(input); index++) { + input[index] = index*17 + iteration; + } + for (unsigned index = 0; index < sizeof(output); index++) { + output[index] = index*7 + iteration; + } + memcpy(before_input, input, sizeof(input)); + memcpy(want, output, sizeof(output)); + const unsigned offsets[] = {0,16,48}, widths[] = {16,32,%[4]d}; + for (unsigned view = 0; view < (%[4]d == 64 ? 3 : 2); view++) { + for (unsigned lane = 0; lane < widths[view]; lane++) { + uint8_t value = input[lane]; + if (!%[3]d && lane < %[2]d) { + value = input[64+lane]; + } else if (!%[3]d && !%[5]d && lane < 16) { + value = 0; + } + want[16+offsets[view]+lane] = value; + } + } + for (unsigned lane = 0; lane < %[2]d; lane++) { + want[144+lane] = input[(%[3]d ? 0 : 64)+lane]; + } + %[1]s(input, output + 16); + for (unsigned index = 0; index < sizeof(output); index++) { + if (output[index] != want[index]) { + fprintf(stderr, "%[1]s wide=%[4]d iteration=%%u byte=%%u got=%%02x want=%%02x\n", iteration, index, output[index], want[index]); + return 1; + } + } + if (memcmp(input, before_input, sizeof(input))) { + return 2; + } + } +`, form.name, form.width, self, wide, register) + } + source.WriteString("\treturn 0;\n}\n") + triple := testTargetTriple(runtime.GOOS, "amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, fmt.Sprintf("legacy_alias_%d", wide), triple, + x86LegacyScalarAliasIR(t, wide, "amd64", triple), source.String(), runner) + } +} diff --git a/amd64_legacy_scalar_move_upper_runtime_test.go b/amd64_legacy_scalar_move_upper_runtime_test.go new file mode 100644 index 000000000..c4fc91fad --- /dev/null +++ b/amd64_legacy_scalar_move_upper_runtime_test.go @@ -0,0 +1,228 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +type x86LegacyScalarMoveUpperForm struct { + name, move string + width int + register bool +} + +func x86LegacyScalarMoveUpperForms() []x86LegacyScalarMoveUpperForm { + return []x86LegacyScalarMoveUpperForm{ + {name: "loadSS", move: "MOVSS", width: 4}, + {name: "copySS", move: "MOVSS", width: 4, register: true}, + {name: "loadSD", move: "MOVSD", width: 8}, + {name: "copySD", move: "MOVSD", width: 8, register: true}, + } +} + +func x86LegacyScalarMoveUpperAssembly(goarch string) string { + word, pointerMove := 8, "MOVQ" + if goarch == "386" { + word, pointerMove = 4, "MOVL" + } + var source strings.Builder + for _, form := range x86LegacyScalarMoveUpperForms() { + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s input+0(FP),AX\n%s output+%d(FP),BX\nMOVUPS (AX),X0\n", + form.name, word*2, pointerMove, pointerMove, word) + if form.register { + fmt.Fprintf(&source, "%s 16(AX),X1\n%s X1,X0\n", form.move, form.move) + } else { + fmt.Fprintf(&source, "%s 16(AX),X0\n", form.move) + } + source.WriteString("MOVUPS X0,(BX)\nRET\n") + } + return source.String() +} + +func x86LegacyScalarMoveUpperIR(t *testing.T, goarch, triple string) string { + t.Helper() + assembly := x86LegacyScalarMoveUpperAssembly(goarch) + requireX86GoAssemblerResult(t, goarch, assembly, true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + word := int64(8) + if goarch == "386" { + word = 4 + } + sigs := make(map[string]FuncSig) + for _, form := range x86LegacyScalarMoveUpperForms() { + symbol := "scalar." + form.name + sigs[symbol] = FuncSig{ + Name: symbol, Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: word, Type: Ptr, Index: 1, Field: -1}, + }}, + } + } + ir, err := Translate(file, Options{ + Goarch: goarch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "scalar.", 1) }, + }) + if err != nil { + t.Fatal(err) + } + return ir +} + +func TestX86LegacyScalarMoveUpperFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ goarch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i386-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + ir := x86LegacyScalarMoveUpperIR(t, target.goarch, target.triple) + compileLLVMToObject(t, llc, target.triple, "scalar-upper.ll", "scalar-upper.o", ir) + }) + } +} + +func x86LegacyScalarMoveActualGoUpperOracle(t *testing.T) { + t.Helper() + var declarations, tests strings.Builder + declarations.WriteString("package scalar\nimport \"unsafe\"\n") + tests.WriteString("package scalar\nimport (\"testing\"; \"unsafe\")\nfunc TestFullBytes(t *testing.T) {\n") + for _, form := range x86LegacyScalarMoveUpperForms() { + fmt.Fprintf(&declarations, "func %s(input, output unsafe.Pointer)\n", form.name) + fmt.Fprintf(&tests, ` + for iteration := 0; iteration < 128; iteration++ { + var input [32]byte + var output [48]byte + for index := range input { + input[index] = byte(index*17 + iteration) + } + for index := range output { + output[index] = byte(index*7 + iteration) + } + beforeInput, beforeOutput := input, output + %[1]s(unsafe.Pointer(&input[0]), unsafe.Pointer(&output[16])) + for index := range output { + want := beforeOutput[index] + if index >= 16 && index < 32 { + lane := index - 16 + want = 0 + if lane < %[2]d { + want = input[16+lane] + } else if %[3]t { + want = input[lane] + } + } + if output[index] != want { + t.Fatalf("%[1]s iteration=%%d byte=%%d got=%%#x want=%%#x", iteration, index, output[index], want) + } + } + if input != beforeInput { + t.Fatal("%[1]s changed input") + } + } +`, form.name, form.width, form.register) + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module scalar\n\ngo 1.20\n", "scalar.go": declarations.String(), + "scalar_amd64.s": x86LegacyScalarMoveUpperAssembly("amd64"), "scalar_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + command.WaitDelay = 2 * time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go full-byte scalar move oracle: %v\n%s", err, output) + } +} + +func TestCrossLinuxRuntimeMatrixX86LegacyScalarMoveUpperBytes(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the Linux/amd64 runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + x86LegacyScalarMoveActualGoUpperOracle(t) + var source strings.Builder + source.WriteString(`#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + for _, form := range x86LegacyScalarMoveUpperForms() { + fmt.Fprintf(&source, "extern void %s(void*,void*) __asm__(LABEL(\"scalar.%s\"));\n", form.name, form.name) + } + source.WriteString("int main(void) {\n") + for _, form := range x86LegacyScalarMoveUpperForms() { + register := 0 + if form.register { + register = 1 + } + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 128; iteration++) { + uint8_t input[32], output[48], before_input[32], before_output[48]; + for (unsigned index = 0; index < sizeof(input); index++) { + input[index] = index*17 + iteration; + } + for (unsigned index = 0; index < sizeof(output); index++) { + output[index] = index*7 + iteration; + } + memcpy(before_input, input, sizeof(input)); + memcpy(before_output, output, sizeof(output)); + %[1]s(input, output + 16); + for (unsigned index = 0; index < sizeof(output); index++) { + uint8_t want = before_output[index]; + if (index >= 16 && index < 32) { + unsigned lane = index - 16; + want = lane < %[2]d ? input[16+lane] : (%[3]d ? input[lane] : 0); + } + if (output[index] != want) { + fprintf(stderr, "%[1]s iteration=%%u byte=%%u got=%%02x want=%%02x\n", iteration, index, output[index], want); + return 1; + } + } + if (memcmp(input, before_input, sizeof(input))) { + return 2; + } + } +`, form.name, form.width, register) + } + source.WriteString("\treturn 0;\n}\n") + triple := testTargetTriple(runtime.GOOS, "amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_upper_bytes", triple, + x86LegacyScalarMoveUpperIR(t, "amd64", triple), source.String(), runner) +} diff --git a/amd64_local_label_collision_test.go b/amd64_local_label_collision_test.go new file mode 100644 index 000000000..b27dff95a --- /dev/null +++ b/amd64_local_label_collision_test.go @@ -0,0 +1,54 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestX86LocalLabelsDoNotCollideWithIRValues(t *testing.T) { + for _, target := range []struct { + arch, triple, mov string + }{ + {"amd64", "x86_64-apple-darwin", "MOVQ"}, + {"amd64", "x86_64-unknown-linux-gnu", "MOVQ"}, + {"amd64", "x86_64-pc-windows-msvc", "MOVQ"}, + {"386", "i386-unknown-linux-gnu", "MOVL"}, + {"386", "i686-pc-windows-msvc", "MOVL"}, + } { + t.Run(target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT collision(SB),4,$0-0 +PXOR X1,X1 +JMP x1 +x1: +%s X1,AX +JMP t1 +t1: +JMP reg_AX +reg_AX: +JMP bb_foo +bb_foo: +RET +`, target.mov) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + ir, err := Translate(file, Options{ + Goarch: target.arch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"collision": { + Name: "collision", Ret: Void, + }}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "local-label.ll", "local-label.o", ir) + }) + } +} diff --git a/amd64_lock_scalar_rmw_bank_test.go b/amd64_lock_scalar_rmw_bank_test.go new file mode 100644 index 000000000..0b1e437cc --- /dev/null +++ b/amd64_lock_scalar_rmw_bank_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86LockScalarRMWByteSourcePhysicalBank(t *testing.T) { + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR"} { + for _, source := range []string{"AH", "CH", "DH", "BH"} { + for _, address := range []struct { + name, setup, memory string + rex bool + }{ + {"SI", "", "(SI)", false}, + {"R8", "MOVQ SI,R8\n", "(R8)", true}, + {"R15", "MOVQ SI,R15\n", "(R15)", true}, + {"R8-index", "MOVQ $0,R8\n", "(SI)(R8*1)", true}, + {"R15-index", "MOVQ $0,R15\n", "(SI)(R15*1)", true}, + } { + t.Run(stem+"/"+source+"/"+address.name, func(t *testing.T) { + assembly := x86NamedLockOracleSource("amd64", address.setup+ + "CLC\nLOCK\n"+stem+"B "+source+","+address.memory) + code := x86LockScalarOriginalInstruction(t, "amd64", assembly) + physical, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go bytes=%x operands=%s", code, x86asm.GoSyntax(physical, 0, nil)) + want := map[string]x86asm.Reg{"AH": x86asm.AH, "CH": x86asm.CH, + "DH": x86asm.DH, "BH": x86asm.BH}[source] + if address.rex { + want = map[string]x86asm.Reg{"AH": x86asm.SPB, "CH": x86asm.BPB, + "DH": x86asm.SIB, "BH": x86asm.DIB}[source] + } + if physical.Args[1] != want { + t.Fatalf("actual source physical bank=%v, want %v", physical.Args[1], want) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, assembly, + "package ordering\nfunc locked(p *[16]byte)\n", "amd64", "") + if !address.rex { + if err != nil || !strings.Contains(ir, "lock; "+strings.ToLower(stem)+"b") { + t.Fatal("native high byte without REX lost its actual source bank", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("source high-byte spelling borrowed a different REX physical bank", err) + } + }) + } + }) + } + } + } +} diff --git a/amd64_lock_scalar_rmw_boundaries_test.go b/amd64_lock_scalar_rmw_boundaries_test.go new file mode 100644 index 000000000..5dbcc2047 --- /dev/null +++ b/amd64_lock_scalar_rmw_boundaries_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86LockScalarRMWTypedRolesAgreeWithoutMemoryPermission(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LockScalarOracleForms(arch) { + t.Run(arch+"/"+row.text, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·f(SB),4,$0-0\n"+row.text+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if len(file.Funcs[0].Instrs) != 3 || file.Funcs[0].Instrs[0].Op != OpTEXT { + t.Fatal("complete fixture lost its TEXT/RMW/RET roles") + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if err != nil || !known { + t.Fatal("complete accepted grammar row lost its typed form", err) + } + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if err != nil || !known || len(effect.operands) != len(ins.Args) { + t.Fatal("accepted lowering row lacks its complete ordered byte effect", err) + } + for index, operand := range effect.operands { + if operand.index != index || operand.width != int64(form.spec.bits/8) || !operand.read || + operand.write != (index == len(ins.Args)-1) { + t.Fatal("read/write/native width role changed", effect) + } + } + stem := row.op[:len(row.op)-1] + defined, killed, read := x86ByteFlagsAll, uint8(0), uint8(0) + switch stem { + case "ADC", "SBB": + read = x86ByteFlagCF + case "AND", "OR", "XOR": + defined &^= x86ByteFlagAF + killed = x86ByteFlagAF + case "INC", "DEC": + defined &^= x86ByteFlagCF + case "NOT": + defined = 0 + case "BTC", "BTR", "BTS": + defined = x86ByteFlagCF + killed = x86ByteFlagOF | x86ByteFlagSF | x86ByteFlagAF | x86ByteFlagPF + } + bitRegister := strings.HasPrefix(stem, "BT") && ins.Args[0].Kind == OpReg + if effect.flagsRead != read || effect.flagsDefined != defined || effect.flagsKilled != killed || + effect.dynamicMemory != bitRegister { + t.Fatal("defined/preserved/killed flags or full bit-string memory role changed", effect) + } + for _, address := range []string{"p+0(FP)", "local-16(SP)", "0(SP)", "0(BP)", "1(SI)(BP*1)"} { + body := strings.Replace(row.text, "1(SI)", address, 1) + source := strings.Replace(x86NamedLockOracleSource(arch, "CLC\nLOCK\n"+body), "$0-", "$32-", 1) + x86LockScalarOriginalInstruction(t, arch, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("pure effect borrowed virtual FP/AUTO native backing", api, address, err) + } + } + } + }) + } + } +} + +func TestX86LockScalarRMWNativeBoundaryCannotBorrowVirtualOrMixedProof(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LockScalarOracleForms(arch) { + t.Run(arch+"/"+row.text, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT ·f(SB),4,$32-0\n"+row.text+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if err != nil || !known { + t.Fatal(err) + } + // These are consumer-input mutations, not additional Go syntax + // acceptance claims. In particular, SP as an index is invalid. + for _, memory := range []MemRef{ + {Base: SP}, {Base: BP}, {Base: SI, Index: SP, Scale: 1}, + {Base: SI, Index: BP, Scale: 1}, {Base: SP, Sym: "local", Off: -16}, + } { + changed := form + changed.destination = Operand{Kind: OpMem, Mem: memory} + if err := x86LockScalarNativeMemoryBound(changed, ins, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native form boundary borrowed virtual frame backing", memory, err) + } + } + ins.x86UnboundAddressOverride = true + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch, ctx.namedAtomicLock, ctx.lockScalarNS = arch, true, "__bound_" + before, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerLockScalarRMW(ins.Op, ins) + if !known || terminated || !errors.Is(err, ErrProbeNeedsContext) || + before != output.String() || temporary != ctx.tmp { + t.Fatal("mixed unbound native marker borrowed another form's bridge", err) + } + }) + } + } +} + +func TestX86LockScalarRMWMalformedSourceAndDispatchRejectBeforeIR(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, body := range []string{ + "ADDL CX", "ADDL (SI),(DI)", "ADDL X0,(SI)", "ADDL CX,$1", + "ADDL $1.0,(SI)", "ADDL.BAD CX,(SI)", "BTCL $128,(SI)", + "BTCL $-129,(SI)", "INCL $1", "NOTL (SI),(DI)", + "ADDL CX,(SI)(SP*1)", "ADDL CX,(SI)(DI*3)", + } { + t.Run(arch+"/"+body, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CLC\nLOCK\n"+body) + goAssemblerOracleOutput(t, arch, source, "", false) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if err == nil || ir != "" { + t.Fatal("malformed complete source emitted IR", api, err) + } + } + }) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.namedAtomicLock, ctx.lockScalarNS = true, "__bound_" + before, temporary := output.String(), ctx.tmp + ins := Instr{Op: "SUBL", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: SI}}}} + known, terminated, err := ctx.lowerLockScalarRMW("ADDL", ins) + if !known || terminated || err == nil || before != output.String() || temporary != ctx.tmp { + t.Fatal("contradictory dispatch mutated IR or temporary state", known, terminated, err) + } +} + +func TestX86LockScalarRMWCarryDoesNotAuthorizeFrameReads(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, address := range []string{"0(SP)", "0(BP)", "missing+32(FP)"} { + t.Run(arch+"/"+address, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CMPL "+address+",AX\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("CF observation borrowed an unproved stack/frame read", api, err) + } + } + }) + } + } +} diff --git a/amd64_lock_scalar_rmw_carry.go b/amd64_lock_scalar_rmw_carry.go new file mode 100644 index 000000000..530aba683 --- /dev/null +++ b/amd64_lock_scalar_rmw_carry.go @@ -0,0 +1,346 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +// This domain proves only the source of the CF value consumed by a named +// locked ADC/SBB. It grants no register, memory, frame or caller permission. +// GP byte facts prevent a default-zero alloca or an empty flags stack from +// becoming an incoming carry. The ordinary consumers still check operands. +type x86LockCarryState struct { + carry bool + gp [16]uint8 + stack []x86LockCarryCell + stackUnbound bool +} + +type x86LockCarryCell struct { + bits int + valueKnown bool + carryKnown bool +} + +func (state x86LockCarryState) clone() x86LockCarryState { + state.stack = append([]x86LockCarryCell(nil), state.stack...) + return state +} + +func (state x86LockCarryState) intersect(other x86LockCarryState) x86LockCarryState { + state = state.clone() + state.carry = state.carry && other.carry + state.stackUnbound = state.stackUnbound || other.stackUnbound + for index := range state.gp { + state.gp[index] &= other.gp[index] + } + if len(state.stack) != len(other.stack) { + state.stack = nil + state.stackUnbound = true + } else { + for index := range state.stack { + if state.stack[index].bits != other.stack[index].bits { + state.stack = nil + state.stackUnbound = true + break + } + state.stack[index].valueKnown = state.stack[index].valueKnown && other.stack[index].valueKnown + state.stack[index].carryKnown = state.stack[index].carryKnown && other.stack[index].carryKnown + } + } + if state.stackUnbound { + state.stack = nil + } + return state +} + +func (c *amd64Ctx) lockCarryRead(operand Operand, width int64, state x86LockCarryState) bool { + if operand.Kind == OpFP { + for _, slot := range c.sig.Frame.Params { + bytes := x86FrameScalarBytes(slot.Type, c.goarch) + if bytes > 0 && operand.FPOffset >= slot.Offset && + operand.FPOffset-slot.Offset <= bytes-width && width > 0 { + return true // Actual typed argument bits, not pointee/backing facts. + } + } + return false + } + if operand.Kind == OpMem || operand.Kind == OpSym { + // A native instruction reads actual memory bits; this does not prove + // its address or authorize the read. The final memory guards do that. + return operand.Kind == OpSym && !strings.HasPrefix(operand.Sym, "$") || + operand.Kind == OpMem && operand.Mem.Sym == "" && operand.Mem.Base != SP && operand.Mem.Base != BP + } + return x86ResultRead(operand, width, &x86ResultState{gp: state.gp}, nil, c.goarch) +} + +func (c *amd64Ctx) lockCarryWrite(operand Operand, width int64, known bool, state *x86LockCarryState) { + if operand.Kind == OpMem || operand.Kind == OpSym || operand.Kind == OpFP { + // No numerical/address fact here establishes nonaliasing with an + // outstanding source PUSH. Even an independently allowed native store + // cannot preserve a stack cell without a separate exact alias proof. + state.stack = nil + return + } + if operand.Kind != OpReg { + return + } + if base, valid := x86CallerGP(operand.Reg); valid && base == SP { + // Explicit SP replacement/adjustment is not an ordinary balanced + // PUSH/POP. This domain cannot recover its original physical identity, + // even if a later instruction defines the pushed GP value again. + state.stack = nil + state.stackUnbound = true + } + index, mask, valid := x86ResultGP(operand.Reg, width, c.goarch) + if !valid { + state.gp = [16]uint8{} + return + } + state.gp[index] &^= mask + if known { + state.gp[index] |= mask + } + if width == 4 || width == 8 { + state.gp[index] = 0 + if known { + state.gp[index] = 0xff + } + } +} + +func (c *amd64Ctx) lockCarryTransfer(ins Instr, state x86LockCarryState) x86LockCarryState { + op := Op(strings.ToUpper(string(ins.Op))) + if op == OpTEXT || op == OpRET || op == "LOCK" && len(ins.Args) == 0 || + op == OpJMP || isAMD64ConditionalBranch(op) || op == "LOOP" { + return state + } + if (op == "CLC" || op == "STC" || op == "CMC") && len(ins.Args) == 0 { + state.carry = op != "CMC" || state.carry + return state + } + if width := x86RawReturnScalarMoveBytes(op); width != 0 && + op == ins.Op && len(ins.Args) == 2 && !(width == 8 && c.goarch == "386") { + known := c.lockCarryRead(ins.Args[0], width, state) + c.lockCarryWrite(ins.Args[1], width, known, &state) + return state + } + if spec, found := amd64StackWidthSpecs[op]; found { + validMode := spec.bits == 16 || spec.bits == 32 && c.goarch == "386" || spec.bits == 64 && c.goarch == "amd64" + validCount := len(ins.Args) == 0 + if spec.mode == amd64StackPushValue || spec.mode == amd64StackPopValue { + validCount = len(ins.Args) == 1 + } + if !validMode || !validCount { + return x86LockCarryState{stackUnbound: true} + } + if spec.mode == amd64StackPushValue || spec.mode == amd64StackPushFlags { + if len(state.stack) >= 64 { + return x86LockCarryState{stackUnbound: true} + } + cell := x86LockCarryCell{bits: spec.bits, carryKnown: state.carry} + if spec.mode == amd64StackPushValue { + cell.valueKnown = c.lockCarryRead(ins.Args[0], int64(spec.bits/8), state) + cell.carryKnown = cell.valueKnown + } + if state.stackUnbound { + cell.valueKnown, cell.carryKnown = false, false + } + // PUSHF captures a flags word, but this domain proves only its CF. + // POP into GP must not promote unknown OF or other bits to numeric + // byte facts. A direct POPF may independently consume known CF. + state.stack = append(state.stack, cell) + return state + } + cell := x86LockCarryCell{} + if len(state.stack) != 0 { + cell = state.stack[len(state.stack)-1] + state.stack = state.stack[:len(state.stack)-1] + if cell.bits != spec.bits { + cell = x86LockCarryCell{} + } + } + if spec.mode == amd64StackPopFlags { + state.carry = cell.carryKnown + } else { + c.lockCarryWrite(ins.Args[0], int64(spec.bits/8), cell.valueKnown, &state) + } + return state + } + if effect, recognized, err := x86TypedInstructionByteEffect(op, ins, c.goarch); recognized { + if err != nil { + return x86LockCarryState{stackUnbound: true} + } + known := effect.flagsRead&x86ByteFlagCF == 0 || state.carry + for _, operand := range effect.operands { + if operand.read { + known = known && c.lockCarryRead(operand.operand, operand.width, state) + } + } + for _, operand := range effect.operands { + if operand.write { + c.lockCarryWrite(operand.operand, operand.width, known, &state) + } + } + if effect.flagsKilled&x86ByteFlagCF != 0 { + state.carry = false + } + if effect.flagsDefined&x86ByteFlagCF != 0 { + state.carry = known + } + return state + } + if (op == "CMPB" || op == "CMPW" || op == "CMPL" || op == "CMPQ" || + op == "TESTB" || op == "TESTW" || op == "TESTL" || op == "TESTQ") && len(ins.Args) == 2 { + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[string(op)[len(op)-1]] + valid := validateAMD64ScalarAddSubOperands(c.goarch, string(op), bits, ins.Args[0], ins.Args[1], ins) == nil + if strings.HasPrefix(string(op), "CMP") { + valid = valid || validateAMD64ScalarAddSubOperands(c.goarch, string(op), bits, ins.Args[1], ins.Args[0], ins) == nil + } + if !(bits == 64 && c.goarch == "386") && valid { + state.carry = c.lockCarryRead(ins.Args[0], int64(bits/8), state) && c.lockCarryRead(ins.Args[1], int64(bits/8), state) + return state + } + } + if op == "STMXCSR" && len(ins.Args) == 1 && isAMD64MemoryOperand(ins.Args[0]) { + c.lockCarryWrite(ins.Args[0], 4, false, &state) + return state + } + if op == "NOP" && len(ins.Args) == 0 { + return state + } + // Calls, native/raw effects and unregistered forms have no incoming flags + // preservation contract. A later explicit definition may start a new fact. + return x86LockCarryState{stackUnbound: true} +} + +func (c *amd64Ctx) verifyNamedLockCarry() error { + var required [][2]int + for block, body := range c.blocks { + for index := 1; index < len(body.instrs); index++ { + previous, ins := body.instrs[index-1], body.instrs[index] + if previous.Op != "LOCK" || len(previous.Args) != 0 { + continue + } + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, c.goarch) + if known && err == nil && (form.spec.stem == "ADC" || form.spec.stem == "SBB") { + required = append(required, [2]int{block, index}) + } + } + } + if len(required) == 0 { + return nil // Unrelated translations retain their existing contracts. + } + fail := func(reason string) error { + return fmt.Errorf("%w: named LOCK incoming carry is not source-defined: %s", ErrProbeNeedsContext, reason) + } + if len(c.blocks) > 4096 { + return fail("source CFG exceeds the bounded proof domain") + } + steps := 0 + states := map[int]x86LockCarryState{0: {}} + queue := []int{0} + inputs := make(map[[2]int]bool) + for visits := 0; len(queue) != 0; visits++ { + if visits >= 32768 { + return fail("source CFG fixed-point budget exhausted") + } + block := queue[0] + queue = queue[1:] + state := states[block].clone() + for index, ins := range c.blocks[block].instrs { + steps++ + if steps > 262144 { + return fail("combined source instruction work budget exhausted") + } + inputs[[2]int{block, index}] = state.carry + state = c.lockCarryTransfer(ins, state) + } + successors, valid := c.lockCarrySuccessors(block) + if !valid { + return fail("indirect, external or unresolved source control flow") + } + for _, target := range successors { + if previous, found := states[target]; found { + stateAtTarget := previous.intersect(state) + if reflect.DeepEqual(previous, stateAtTarget) { + continue + } + states[target] = stateAtTarget + } else { + states[target] = state.clone() + } + queue = append(queue, target) + } + } + for _, location := range required { + if !inputs[location] { + return fail("a complete ADC/SBB has an unknown predecessor CF value") + } + } + return nil +} + +func (c *amd64Ctx) lockCarrySuccessors(block int) ([]int, bool) { + body := c.blocks[block].instrs + if len(body) == 0 { + if block+1 < len(c.blocks) { + return []int{block + 1}, true + } + return nil, true + } + ins := body[len(body)-1] + op := Op(strings.ToUpper(string(ins.Op))) + if op == OpRET { + return nil, len(ins.Args) == 0 + } + if op != OpJMP && op != "LOOP" && !isAMD64ConditionalBranch(op) { + if block+1 < len(c.blocks) { + return []int{block + 1}, true + } + return nil, false + } + args := ins.Args + if op != OpJMP && op != "LOOP" && !isAMD64CounterZeroBranch(op) && + len(args) == 2 && args[0].Kind == OpImm && (args[0].Imm == 0 || args[0].Imm == 1) { + args = args[1:] + } + if len(args) != 1 { + return nil, false + } + target, found := -1, false + if args[0].Kind == OpMem && args[0].Mem.Base == PC && args[0].Mem.OffRaw == "" { + base := int64(c.blockBase[block] + len(body) - 1) + if args[0].Mem.Off > 1<<63-1-base { + return nil, false + } + pc := base + args[0].Mem.Off + if pc >= 0 && pc <= int64(^uint(0)>>1) { + target, found = c.blockByIdx[int(pc)] + } + } else { + label := "" + switch args[0].Kind { + case OpLabel, OpSym: + label = args[0].Sym + case OpIdent: + label = args[0].Ident + case OpReg: + label = string(args[0].Reg) + } + for index, candidate := range c.blocks { + if candidate.name == label { + target, found = index, true + break + } + } + } + if !found || target < 0 || target >= len(c.blocks) { + return nil, false + } + if op == OpJMP { + return []int{target}, true + } + return []int{target, block + 1}, block+1 < len(c.blocks) +} diff --git a/amd64_lock_scalar_rmw_carry_alias_test.go b/amd64_lock_scalar_rmw_carry_alias_test.go new file mode 100644 index 000000000..d0133e81b --- /dev/null +++ b/amd64_lock_scalar_rmw_carry_alias_test.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86LockScalarRMWCarryStackAliasesAndIdentityStayContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word := "Q" + if arch == "386" { + word = "L" + } + for _, width := range []string{"W", word} { + push, pop := "PUSH"+width+" DX\n", "\nPOPF"+width + for _, control := range []struct{ name, body string }{ + {"explicit stack store", push + "MOV" + width + " BP,0(SP)" + pop}, + {"derived stack store", push + "MOV" + word + " SP,BX\nMOV" + width + " BP,(BX)" + pop}, + {"unknown vector store", push + "MOVOU X0,0(SP)" + pop}, + {"CSR stack store", push + "STMXCSR 0(SP)" + pop}, + {"SP replacement before push", "MOV" + word + " $0,SP\nMOVL $5,DX\n" + push + "POPF" + width}, + {"POP SP before new push", "PUSH" + word + " BP\nPOP" + word + " SP\nMOVL $5,DX\n" + push + "POPF" + width}, + {"LEA SP before push", "LEA" + word + " 0(SP),SP\nMOVL $5,DX\n" + push + "POPF" + width}, + {"balanced SP adjustment", "SUB" + word + " $16,SP\nMOVL $5,DX\n" + push + "POPF" + width + "\nADD" + word + " $16,SP"}, + } { + t.Run(arch+"/"+width+"/"+control.name, func(t *testing.T) { + // These accepted objects are unsafe/unknown-input tripwires. + // They are never executed as numerical or caller evidence. + source := x86NamedLockOracleSource(arch, control.body+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + if err := ctx.verifyNamedLockCarry(); !errors.Is(err, ErrProbeNeedsContext) || + output.String() != before || ctx.tmp != temporary { + t.Error("pure CF domain retained stale stack bits or regained replaced SP identity", err) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("a stale stack value or replaced SP authorized carry", err) + } + }) + } + }) + } + } + } +} + +func TestX86LockScalarRMWCarryInvalidStackDoesNotInvalidateExplicitCF(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word := "Q" + if arch == "386" { + word = "L" + } + for _, seed := range []string{"CLC", "STC"} { + source := x86NamedLockOracleSource(arch, "MOV"+word+" $0,SP\n"+seed+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + if err := ctx.verifyNamedLockCarry(); err != nil || output.String() != before || ctx.tmp != temporary { + t.Fatal("explicit CF fact was lost or granted an IR/memory permission", err) + } + if strings.Contains(output.String(), "lock;") { + t.Fatal("a CF-only observation emitted an atomic object") + } + } + } +} diff --git a/amd64_lock_scalar_rmw_carry_stack_test.go b/amd64_lock_scalar_rmw_carry_stack_test.go new file mode 100644 index 000000000..8a99012c4 --- /dev/null +++ b/amd64_lock_scalar_rmw_carry_stack_test.go @@ -0,0 +1,47 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86LockScalarRMWCarryStackDoesNotInventWholeFlagsWord(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, counter := "Q", "JCXZQ" + if arch == "386" { + word, counter = "L", "JCXZL" + } + for _, suffix := range []string{"W", word} { + compare := "AND" + suffix + " $2048,AX\nCMP" + suffix + " AX,$1" + for _, control := range []struct { + name, body string + known bool + }{ + {"unknown OF through flags word", "CLC\nPUSHF" + suffix + "\nPOP" + suffix + " AX\n" + compare, false}, + {"CF-only flags roundtrip", "CLC\nPUSHF" + suffix + "\nPOPF" + suffix, true}, + {"defined whole numerical word", "PUSH" + suffix + " DX\nPOP" + suffix + " AX\n" + compare, true}, + {"mixed flags and numerical join", "CLC\n" + counter + " flags\nPUSH" + suffix + " DX\nPOP" + suffix + " AX\nJMP compare\nflags:\nPUSHF" + suffix + "\nPOP" + suffix + " AX\ncompare:\n" + compare, false}, + {"partial flags word through GP", "CLC\nPUSHF" + suffix + "\nPOP" + suffix + " AX\nPUSH" + suffix + " AX\nPOPF" + suffix, false}, + } { + t.Run(arch+"/"+suffix+"/"+control.name, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, control.body+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if control.known { + if err != nil || !strings.Contains(ir, "lock; adcl") { + t.Fatal("known numerical word or CF roundtrip was lost", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || ir != "" || !strings.Contains(err.Error(), "carry") { + t.Fatal("CF-only PUSHF falsely defined unknown GP/OF bits", err) + } + }) + } + }) + } + } + } +} diff --git a/amd64_lock_scalar_rmw_carry_test.go b/amd64_lock_scalar_rmw_carry_test.go new file mode 100644 index 000000000..9d9396142 --- /dev/null +++ b/amd64_lock_scalar_rmw_carry_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86LockScalarRMWUnknownIncomingCarryNeedsContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + widths := "BWL" + if arch == "amd64" { + widths += "Q" + } + for _, stem := range []string{"ADC", "SBB"} { + for _, width := range widths { + register := "CX" + if width == 'B' { + register = "CL" + } + body := "LOCK\n\t" + stem + string(width) + " " + register + ",(SI)" + t.Run(arch+"/"+stem+string(width), func(t *testing.T) { + source := x86NamedLockOracleSource(arch, body) + machine := x86LockScalarOriginalInstruction(t, arch, source) + t.Logf("actual Go accepts original instruction=%x; incoming CF remains undefined by the source", machine) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" || + !strings.Contains(err.Error(), "carry") { + t.Fatalf("native carry-dependent memory effect borrowed an undefined entry flag: %v", err) + } + }) + } + }) + } + } + } +} + +func TestX86LockScalarRMWCarryCFGAndStackSources(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + suffix, counter := "Q", "JCXZQ" + if arch == "386" { + suffix, counter = "L", "JCXZL" + } + for _, control := range []struct { + name, body string + known bool + }{ + {"CLC", "CLC", true}, + {"STC CMC", "STC\nCMC", true}, + {"initialized CMP", "CMPL AX,$0", true}, + {"undefined CMC", "CMC", false}, + {"undefined compare", "CMPL BP,$0", false}, + {"explicit stack seed", "PUSH" + suffix + " DX\nPOPF" + suffix, true}, + {"unknown stack seed", "PUSH" + suffix + " BP\nPOPF" + suffix, false}, + {"unseeded join", counter + " join\nCLC\njoin:", false}, + {"seeded join", "STC\n" + counter + " join\nCLC\njoin:", true}, + {"call clobber", "STC\nCALL ·other(SB)", false}, + {"undefined divide flags", "STC\nDIVL CX", false}, + {"loop retains seed", "STC\nloop:\nNOP\nLOOP loop", true}, + } { + t.Run(arch+"/"+control.name, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, + control.body+"\nLOCK\nADCL CX,(SI)") + x86LockScalarOriginalInstruction(t, arch, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\nfunc other()\n", arch, "") + if control.known { + if err != nil || !strings.Contains(ir, "lock; adcl") { + t.Fatal("independent defined carry was not retained", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || ir != "" || !strings.Contains(err.Error(), "carry") { + t.Fatal("unclosed carry acquired a numerical value", err) + } + }) + } + }) + } + } +} + +func TestX86LockScalarRMWCarryBudgetAndInputsRemainUnchanged(t *testing.T) { + file, err := Parse(ArchAMD64, x86NamedLockOracleSource("amd64", "CLC\nLOCK\nADCL CX,(SI)")) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before, temporary := output.String(), ctx.tmp + source := fmt.Sprintf("%#v", file) + for count := 0; count != 2; count++ { + if err := ctx.verifyNamedLockCarry(); err != nil || before != output.String() || temporary != ctx.tmp || source != fmt.Sprintf("%#v", file) { + t.Fatal("CF proof modified IR, temporary state or input source", err) + } + } + ctx.blocks = make([]amd64Block, 4097) + ctx.blocks[0].instrs = []Instr{{Op: "LOCK"}, {Op: "ADCL", Args: []Operand{{Kind: OpReg, Reg: CX}, {Kind: OpMem, Mem: MemRef{Base: SI}}}}} + if err := ctx.verifyNamedLockCarry(); !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "bounded") { + t.Fatal("oversized CF domain did not fail closed", err) + } + ctx.blocks = nil + // Synthetic empty/mismatched cells are consumer tripwires, not Go source + // acceptance evidence: stock Go rejects a bare POPF as an unbalanced pop. + if state := ctx.lockCarryTransfer(Instr{Op: "POPFQ"}, x86LockCarryState{carry: true}); state.carry { + t.Fatal("synthetic empty stack supplied carry") + } + state := x86LockCarryState{carry: true, stack: []x86LockCarryCell{{bits: 16, valueKnown: true, carryKnown: true}}} + if state = ctx.lockCarryTransfer(Instr{Op: "POPFQ"}, state); state.carry { + t.Fatal("synthetic mismatched-width stack cell supplied carry") + } +} diff --git a/amd64_lock_scalar_rmw_encoder_test.go b/amd64_lock_scalar_rmw_encoder_test.go new file mode 100644 index 000000000..7d0b989d6 --- /dev/null +++ b/amd64_lock_scalar_rmw_encoder_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +// This independent carrier binds every ordered row, operand class, zcase, +// offset, prefix and opcode byte. It does not establish an address permission +// or assert which overlapping row an actual source instruction selects. +func x86LockScalarEncoderCarrier(source []byte) error { + file, err := parser.ParseFile(token.NewFileSet(), "asm6.go", source, 0) + if err != nil { + return err + } + variables := make(map[string]*ast.CompositeLit) + for _, declaration := range file.Decls { + general, ok := declaration.(*ast.GenDecl) + if !ok || general.Tok != token.VAR { + continue + } + for _, spec := range general.Specs { + value := spec.(*ast.ValueSpec) + for index, name := range value.Names { + if index < len(value.Values) { + if literal, ok := value.Values[index].(*ast.CompositeLit); ok { + variables[name.Name] = literal + } + } + } + } + } + canonical := func(node ast.Node) string { + var output bytes.Buffer + if err := format.Node(&output, token.NewFileSet(), node); err != nil { + return "" + } + return output.String() + } + parse := func(source string) (ast.Expr, error) { + return parser.ParseExpr(source) + } + tables := map[string]string{ + "yxorb": `[]ytab{ + {Zib_, 1, argList{Yi32, Yal}}, + {Zibo_m, 2, argList{Yi32, Ymb}}, + {Zr_m, 1, argList{Yrb, Ymb}}, + {Zm_r, 1, argList{Ymb, Yrb}}, + }`, + "yaddl": `[]ytab{ + {Zibo_m, 2, argList{Yi8, Yml}}, + {Zil_, 1, argList{Yi32, Yax}}, + {Zilo_m, 2, argList{Yi32, Yml}}, + {Zr_m, 1, argList{Yrl, Yml}}, + {Zm_r, 1, argList{Yml, Yrl}}, + }`, + "yincl": `[]ytab{{Z_rp, 1, argList{Yrl}}, {Zo_m, 2, argList{Yml}}}`, + "yincq": `[]ytab{{Zo_m, 2, argList{Yml}}}`, + "yscond": `[]ytab{{Zo_m, 2, argList{Ymb}}}`, + "ybtl": `[]ytab{{Zibo_m, 2, argList{Yi8, Yml}}, {Zr_m, 1, argList{Yrl, Yml}}}`, + } + for name, text := range tables { + want, err := parse(text) + if err != nil { + return err + } + actual := variables[name] + if actual == nil || canonical(actual) != canonical(want) { + return fmt.Errorf("complete ordered Go encoder table changed: %s", name) + } + } + optab := variables["optab"] + if optab == nil { + return fmt.Errorf("Go encoder has no complete optab") + } + pattern := regexp.MustCompile(`^A(?:ADC|ADD|AND|OR|SBB|SUB|XOR|INC|DEC|NEG|NOT|BTC|BTR|BTS)[BWLQ]$`) + rows := make(map[string]string) + for _, entry := range optab.Elts { + row, ok := entry.(*ast.CompositeLit) + if !ok || len(row.Elts) == 0 { + continue + } + name := canonical(row.Elts[0]) + if !pattern.MatchString(name) { + continue + } + if _, exists := rows[name]; exists { + return fmt.Errorf("duplicate encoder row: %s", name) + } + rows[name] = canonical(row) + } + if len(rows) != 53 { + return fmt.Errorf("complete opcode carrier has %d rows, want 53", len(rows)) + } + // The native byte formulas below are independent of the typed production + // spec. Group numbers are written in the actual Go table's octal spelling. + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR", "INC", "DEC", "NEG", "NOT", "BTC", "BTR", "BTS"} { + for _, width := range "BWLQ" { + if strings.HasPrefix(stem, "BT") && width == 'B' { + continue + } + name := "A" + stem + string(width) + table, prefix := "yaddl", map[rune]string{'B': "Pb", 'W': "Pe", 'L': "Px", 'Q': "Pw"}[width] + encoding := "" + switch stem { + case "INC", "DEC": + group, short := 0, 0x40 + if stem == "DEC" { + group, short = 1, 0x48 + } + table = "yincq" + encoding = fmt.Sprintf("0xff, %02o", group) + if width == 'B' { + table, encoding = "yscond", fmt.Sprintf("0xfe, %02o", group) + } else if width == 'L' { + table, prefix, encoding = "yincl", "Px1", fmt.Sprintf("0x%02x, 0xff, %02o", short, group) + } + case "NEG", "NOT": + group, opcode := 3, 0xf7 + if stem == "NOT" { + group = 2 + } + if width == 'B' { + opcode = 0xf6 + } + table, encoding = "yscond", fmt.Sprintf("0x%02x, %02o", opcode, group) + case "BTC", "BTR", "BTS": + group, opcode := map[string]int{"BTC": 7, "BTR": 6, "BTS": 5}[stem], map[string]int{"BTC": 0xbb, "BTR": 0xb3, "BTS": 0xab}[stem] + table, encoding = "ybtl", fmt.Sprintf("0xba, %02o, 0x%02x", group, opcode) + prefix = "Pm" + if width == 'W' { + prefix = "Pq" + } else if width == 'Q' { + prefix, encoding = "Pw", fmt.Sprintf("0x0f, 0xba, %02o, 0x0f, 0x%02x", group, opcode) + } + default: + group := map[string]int{"ADD": 0, "OR": 1, "ADC": 2, "SBB": 3, "AND": 4, "SUB": 5, "XOR": 6}[stem] + base := group * 8 + encoding = fmt.Sprintf("0x83, %02o, 0x%02x, 0x81, %02o, 0x%02x, 0x%02x", group, base+5, group, base+1, base+3) + if width == 'B' { + table, encoding = "yxorb", fmt.Sprintf("0x%02x, 0x80, %02o, 0x%02x, 0x%02x", base+4, group, base, base+2) + } + } + want, err := parse("[]Optab{{" + name + ", " + table + ", " + prefix + ", opBytes{" + encoding + "}}}") + if err != nil { + return err + } + if rows[name] != canonical(want.(*ast.CompositeLit).Elts[0]) { + return fmt.Errorf("opcode-specific complete encoder attributes changed: %s", name) + } + } + } + return nil +} + +func TestX86LockScalarRMWEncoderCarrierRejectsRuleMutation(t *testing.T) { + source, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if err := x86LockScalarEncoderCarrier(source); err != nil { + t.Fatal(err) + } + for _, mutation := range []struct{ name, from, to string }{ + {"class", "{Zib_, 1, argList{Yi32, Yal}}", "{Zib_, 1, argList{Yi8, Yal}}"}, + {"operand-count", "{Zib_, 1, argList{Yi32, Yal}}", "{Zib_, 1, argList{Yi32, Yal, Ymb}}"}, + {"zcase", "{Zib_, 1, argList{Yi32, Yal}}", "{Zil_, 1, argList{Yi32, Yal}}"}, + {"zoffset", "{Zib_, 1, argList{Yi32, Yal}}", "{Zib_, 2, argList{Yi32, Yal}}"}, + {"row-order", "{Zib_, 1, argList{Yi32, Yal}},\n\t{Zibo_m, 2, argList{Yi32, Ymb}}", "{Zibo_m, 2, argList{Yi32, Ymb}},\n\t{Zib_, 1, argList{Yi32, Yal}}"}, + {"extra-row", "var yincq = []ytab{", "var yincq = []ytab{\n{Zo_m, 2, argList{Ymb}},"}, + {"prefix", "{AADCB, yxorb, Pb,", "{AADCB, yxorb, Px,"}, + {"opcode-bytes", "{AADCB, yxorb, Pb, opBytes{0x14,", "{AADCB, yxorb, Pb, opBytes{0x15,"}, + {"table-binding", "{AADCB, yxorb, Pb,", "{AADCB, yaddl, Pb,"}, + } { + t.Run(mutation.name, func(t *testing.T) { + if !strings.Contains(string(source), mutation.from) { + t.Fatal("mutation no longer binds its actual Go source") + } + changed := strings.Replace(string(source), mutation.from, mutation.to, 1) + if err := x86LockScalarEncoderCarrier([]byte(changed)); err == nil { + t.Fatal("full Go encoder carrier admitted a changed rule") + } + }) + } +} diff --git a/amd64_lock_scalar_rmw_forms.go b/amd64_lock_scalar_rmw_forms.go new file mode 100644 index 000000000..b1482e416 --- /dev/null +++ b/amd64_lock_scalar_rmw_forms.go @@ -0,0 +1,186 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// This is an operand/effect record, not an atomic, pointer or frame capability. +// LOCK requires the independent complete source-adjacency consumer. +type x86LockScalarRMWSpec struct { + stem string + bits int + unary bool + bitString bool + carryInput bool + flagsDefined uint8 + flagsKilled uint8 +} + +type x86LockScalarRMWForm struct { + spec x86LockScalarRMWSpec + source Operand + destination Operand +} + +var x86LockScalarRMWSpecs = func() map[Op]x86LockScalarRMWSpec { + result := make(map[Op]x86LockScalarRMWSpec) + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR", "INC", "DEC", "NEG", "NOT", "BTC", "BTR", "BTS"} { + for index, suffix := range "BWLQ" { + spec := x86LockScalarRMWSpec{stem: stem, bits: []int{8, 16, 32, 64}[index], flagsDefined: x86ByteFlagsAll} + switch stem { + case "ADC", "SBB": + spec.carryInput = true + case "AND", "OR", "XOR": + spec.flagsDefined &^= x86ByteFlagAF + spec.flagsKilled = x86ByteFlagAF + case "INC", "DEC": + spec.unary, spec.carryInput = true, true + spec.flagsDefined &^= x86ByteFlagCF + case "NEG": + spec.unary = true + case "NOT": + spec.unary, spec.flagsDefined = true, 0 + case "BTC", "BTR", "BTS": + if suffix == 'B' { + continue + } + spec.bitString = true + spec.flagsDefined = x86ByteFlagCF + spec.flagsKilled = x86ByteFlagOF | x86ByteFlagSF | x86ByteFlagAF | x86ByteFlagPF + } + result[Op(stem+string(suffix))] = spec + } + } + return result +}() + +func x86LockScalarRMWProperties(op Op, ins Instr, arch string) (x86LockScalarRMWForm, bool, error) { + name := strings.ToUpper(string(op)) + base := strings.SplitN(name, ".", 2)[0] + spec, recognized := x86LockScalarRMWSpecs[Op(base)] + if !recognized { + return x86LockScalarRMWForm{}, false, nil + } + if spec.bitString { + form, _, err := x86BitTestProperties(op, ins, arch) + return x86LockScalarRMWForm{spec: spec, source: form.index, destination: form.destination}, true, err + } + fail := func(reason string) (x86LockScalarRMWForm, bool, error) { + return x86LockScalarRMWForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if name != base || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) || + arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 { + return fail("has an invalid suffix, dispatch identity or machine mode") + } + count := 2 + if spec.unary { + count = 1 + } + if len(ins.Args) != count { + return fail("has an incomplete operand list") + } + form := x86LockScalarRMWForm{spec: spec, destination: ins.Args[count-1]} + if form.destination.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(form.destination.Sym), "$") { + return fail("requires writable memory, not a symbol address literal") + } + if !spec.unary { + form.source = ins.Args[0] + } + if spec.unary { + if form.destination.Kind == OpReg { + allowed := isX86YrlRegisterForArch(form.destination.Reg, arch) + if spec.stem == "NEG" || spec.stem == "NOT" || spec.bits == 8 { + allowed = isGoYmbRegisterForArch(form.destination.Reg, arch) + } + if !allowed { + return fail("destination is outside its complete Go register class") + } + if spec.stem == "NEG" || spec.stem == "NOT" { + form.destination.Reg = amd64YmbEffectiveRegister(form.destination.Reg, spec.bits) + } + } else if !isAMD64MemoryOperand(form.destination) { + return fail("destination is outside the Go memory class") + } + } else { + if err := validateAMD64ScalarAddSubOperands(arch, base, spec.bits, form.source, form.destination, ins); err != nil { + return x86LockScalarRMWForm{}, true, err + } + if form.source.Kind == OpImm { + form.source.Imm = amd64ScalarAddSubImmediateInt64(form.source.Imm, spec.bits) + } + } + needsTLS := false + for _, operand := range ins.Args { + if operand.Kind == OpImm && (operand.ImmRaw != "" || operand.ImmIsFloat) { + return fail("requires a resolved integer immediate") + } + if operand.Kind != OpMem { + continue + } + if x86SourceTLSMemoryRole(operand, arch) { + needsTLS = true + continue + } + memory := operand.Mem + if err := x86SymbolIndexedMemoryError(memory, arch); err != nil { + return x86LockScalarRMWForm{}, true, err + } + if arch == "386" && strings.HasSuffix(memory.Sym, "(SB)") { + memory.Base = "" // Only the actual Go ignored annotation is discarded. + } + if !x86MemoryRegistersValidForArch(memory, arch) || memory.OffRaw != "" || + memory.Index == SP || memory.Index != "" && memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 || + memory.Off != int64(int32(memory.Off)) && !(arch == "386" && memory.Off == int64(uint32(memory.Off))) { + return fail("has an unencodable or unresolved native memory operand") + } + } + if needsTLS { + // Validate the complete operand row before preserving TLS's source + // role. It is not an ordinary SIB address or a native memory grant. + return form, true, fmt.Errorf("%w: %s %s TLS memory needs its actual Go OS/arch/shared address contract: %q", + ErrProbeNeedsContext, arch, op, ins.Raw) + } + if arch == "amd64" && spec.bits == 8 && x86CompareExchangeHasREXByteView(x86CompareExchangeForm{ + source: form.source, destination: form.destination, + }) { + // Go emits these original Ymb spellings with their raw ModRM number. + // A REX prefix selects SP/BP/SI/DI's low byte instead of AH/CH/DH/BH. + // Recording the physical view does not bind a source/native input. + for _, operand := range []*Operand{&form.source, &form.destination} { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == CH || operand.Reg == DH || operand.Reg == BH) { + operand.Reg = amd64YmbEffectiveRegister(operand.Reg, 16) + } + } + } + return form, true, nil +} + +func x86LockScalarNativeMemoryBound(form x86LockScalarRMWForm, ins Instr, arch string) error { + if form.destination.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(form.destination.Sym), "$") { + return fmt.Errorf("%s LOCK RMW address literal is not writable native memory", arch) + } + if form.destination.Kind != OpMem && form.destination.Kind != OpSym || ins.x86UnboundAddressOverride { + return fmt.Errorf("%w: %s LOCK RMW needs its bound native memory destination, not a virtual FP/AUTO span", ErrProbeNeedsContext, arch) + } + if form.destination.Kind == OpMem { + memory := form.destination.Mem + if memory.Base == SP || memory.Base == BP || memory.Index == SP || memory.Index == BP { + // Numeric source slots and LLVM's private SP allocation do not + // establish the original instruction's physical native address. + return fmt.Errorf("%w: %s LOCK RMW needs independent native SP/BP backing", ErrProbeNeedsContext, arch) + } + } + if arch == "amd64" && form.spec.bits == 8 && form.source.Kind == OpReg && + len(ins.Args) == 2 && form.source.Reg != ins.Args[0].Reg { + return fmt.Errorf("%w: amd64 LOCK byte source needs its original Go REX physical register view", ErrProbeNeedsContext) + } + if arch == "386" && form.spec.bits == 8 && form.source.Kind == OpReg { + switch form.source.Reg { + case AX, BX, CX, DX, AL, AH, BL, BH, CL, CH, DL, DH: + default: + return fmt.Errorf("%w: 386 LOCK byte source requires Go's original scratch-register expansion", ErrProbeNeedsContext) + } + } + return nil +} diff --git a/amd64_lock_scalar_rmw_forms_test.go b/amd64_lock_scalar_rmw_forms_test.go new file mode 100644 index 000000000..0efdca435 --- /dev/null +++ b/amd64_lock_scalar_rmw_forms_test.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +type x86LockScalarOracleForm struct { + op string + text string +} + +// This inventory is independent of the production dispatch. Intel's complete +// LOCK list contains these fourteen stems plus the five existing exchange and +// compare-exchange stems. Go's yxorb/yaddl/yscond/yincl/yincq/ybtl ordered tables +// determine their immediate, register and memory operand forms. +func x86LockScalarOracleForms(arch string) []x86LockScalarOracleForm { + widths := "BWL" + if arch == "amd64" { + widths += "Q" + } + var forms []x86LockScalarOracleForm + for _, stem := range []string{"ADC", "ADD", "AND", "OR", "SBB", "SUB", "XOR"} { + for _, width := range widths { + op, register := stem+string(width), "CX" + if width == 'B' { + register = "CL" + } + for _, source := range []string{register, "$1"} { + forms = append(forms, x86LockScalarOracleForm{op, op + " " + source + ",1(SI)"}) + } + } + } + for _, stem := range []string{"INC", "DEC", "NEG", "NOT"} { + for _, width := range widths { + op := stem + string(width) + forms = append(forms, x86LockScalarOracleForm{op, op + " 1(SI)"}) + } + } + for _, stem := range []string{"BTC", "BTR", "BTS"} { + for _, width := range strings.TrimPrefix(widths, "B") { + op := stem + string(width) + for _, source := range []string{"CX", "$1"} { + forms = append(forms, x86LockScalarOracleForm{op, op + " " + source + ",1(SI)"}) + } + } + } + return forms +} + +func TestX86LockScalarRMWCompleteGoTables(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if err := x86LockScalarEncoderCarrier(data); err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`(?m)^\s*\{A((?:ADC|ADD|AND|OR|SBB|SUB|XOR|INC|DEC|NEG|NOT|BTC|BTR|BTS)[BWLQ]),\s*(\w+),`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 53 { + t.Fatalf("complete scalar LOCK opcode inventory changed: %d", len(rows)) + } + seen := make(map[string]bool) + for _, row := range rows { + op, table := row[1], row[2] + if seen[op] { + t.Fatal("duplicate Go opcode registration", op) + } + seen[op] = true + width, stem := op[len(op)-1], op[:len(op)-1] + want := "yaddl" + switch stem { + case "BTC", "BTR", "BTS": + want = "ybtl" + case "INC", "DEC": + want = "yincq" + if width == 'B' { + want = "yscond" + } else if width == 'L' { + want = "yincl" + } + case "NEG", "NOT": + want = "yscond" + default: + if width == 'B' { + want = "yxorb" + } + } + if table != want { + t.Fatalf("Go ordered-table role changed: %s/%s, want %s", op, table, want) + } + } + for _, form := range x86LockScalarOracleForms("amd64") { + if !seen[form.op] { + t.Fatal("independent syntax inventory is absent from actual Go tables", form.op) + } + } +} + +func x86LockScalarOriginalInstruction(t *testing.T, arch, source string) []byte { + t.Helper() + code := x86OrderingHintGoBytes(t, arch, source) + mode := 64 + if arch == "386" { + mode = 32 + } + for offset := 0; offset < len(code); { + instruction, err := x86asm.Decode(code[offset:], mode) + if err != nil || instruction.Len <= 0 { + t.Fatalf("actual original Go bytes cannot be decoded at %d: %v", offset, err) + } + if code[offset] == 0xf0 { + return append([]byte(nil), code[offset:offset+instruction.Len]...) + } + offset += instruction.Len + } + t.Fatalf("actual complete Go object lacks its LOCK-prefixed instruction: %x", code) + return nil +} + +func TestX86LockScalarRMWWholeSourceAdmission(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + forms := x86LockScalarOracleForms(arch) + want := 90 + if arch == "386" { + want = 66 + } + if len(forms) != want { + t.Fatal("complete accepted operand-form denominator changed", arch, len(forms)) + } + for _, form := range forms { + t.Run(arch+"/"+form.text, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CMPL AX,AX\n\tLOCK\n\t"+form.text) + machine := x86LockScalarOriginalInstruction(t, arch, source) + t.Logf("actual original LOCK object bytes=%x", machine) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if err != nil || !strings.Contains(ir, "lock; "+strings.ToLower(form.op)) { + t.Fatalf("actual Go accepted whole scalar LOCK RMW lacks its ordered atomic effect: %v", err) + } + }) + } + }) + } + } +} + +func TestX86LockScalarRMWRawWholeInstructionRetainsContext(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, form := range x86LockScalarOracleForms(arch) { + t.Run(arch+"/"+form.text, func(t *testing.T) { + source := x86NamedLockOracleSource(arch, "CMPL AX,AX\n\tLOCK\n\t"+form.text) + machine := x86LockScalarOriginalInstruction(t, arch, source) + var body strings.Builder + for _, value := range machine { + fmt.Fprintf(&body, "BYTE $%d\n", value) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, + x86NamedLockOracleSource(arch, "CMPL AX,AX\n\t"+body.String()), + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("raw LOCK lost its unresolved whole physical-prefix contract", err) + } + }) + } + }) + } + } +} diff --git a/amd64_lock_scalar_rmw_linux_amd64_test.go b/amd64_lock_scalar_rmw_linux_amd64_test.go new file mode 100644 index 000000000..8767104de --- /dev/null +++ b/amd64_lock_scalar_rmw_linux_amd64_test.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// These are required native execution axes. Darwin/Rosetta and QEMU numerical +// observations do not establish native Linux arithmetic or atomic semantics. +func TestCrossLinuxRuntimeMatrixX86LockScalarRMWNativeAMD64(t *testing.T) { + x87IA32RequireNativeHost(t) + TestCrossLinuxRuntimeMatrixX86LockScalarRMWOriginalAndLLVM(t) + x87IA32RequireNativeHost(t) +} + +func TestCrossLinuxRuntimeMatrixX86LockScalarRMWNative386(t *testing.T) { + x87IA32RequireNativeHost(t) + llc := findLLVM22Tool("llc") + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if llc == "" || err != nil { + t.Fatal("required LLVM22 and IA32 static compiler unavailable", err) + } + declarations, assembly, driver, reference, count := x86LockScalarRuntimeFixture("386") + if count != 129 { + t.Fatal("complete IA32 kernel denominator changed", count) + } + want := fmt.Sprintf("lock observations %d\n", count*292*2*8) + dir := t.TempDir() + original := x86OrderingHintBuildOriginal(t, dir, "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != want { + t.Fatalf("actual original Go native IA32 full arithmetic/status/canary oracle: %q", output) + } + mode := "i386-unknown-linux-gnu" + ir, err := x86OrderingHintTranslateAPI(t, "Go", assembly, declarations, "386", mode) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := filepath.Join(dir, "lock"+level+".ll") + object := filepath.Join(dir, "lock"+level+".o") + main := filepath.Join(dir, "main.c") + executable := filepath.Join(dir, "lock"+level) + for name, contents := range map[string]string{input: ir, main: reference} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, + "-mtriple="+mode, "-filetype=obj", input, "-o", object)) + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", + object, main, "-O2", "-o", executable)) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != want { + t.Fatalf("native LLVM IA32 full-width/flags/MXCSR/canary oracle: %q", output) + } + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_lock_scalar_rmw_namespace_test.go b/amd64_lock_scalar_rmw_namespace_test.go new file mode 100644 index 000000000..fe217310d --- /dev/null +++ b/amd64_lock_scalar_rmw_namespace_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86LockScalarRMWNoCandidateDoesNotResolveLocals(t *testing.T) { + for _, body := range []string{ + "MOVQ value+0(FP),AX\nloop:\nRET", + "LOCK\nADDL AX,BX\nRET", + } { + file, err := Parse(ArchAMD64, "TEXT ·f(SB),4,$0-8\n"+body+"\n") + if err != nil { + t.Fatal(err) + } + var calls []string + namespace := x86LockScalarNamespace(file, "amd64", func(symbol string) string { + calls = append(calls, symbol) + return strings.TrimPrefix(symbol, "·") + }, map[string]FuncSig{"f": {Name: "f", Ret: Void}}) + if namespace != "" || len(calls) != 0 { + t.Fatalf("no legal helper must not allocate names or call the user resolver: namespace=%q calls=%v", namespace, calls) + } + } +} + +func TestX86LockScalarRMWNamespaceUsesFinalGlobalsOnly(t *testing.T) { + file, err := Parse(ArchAMD64, `TEXT ·f(SB),4,$0-8 + MOVQ value+0(FP),SI + MOVL $1,AX + LOCK + ADDL AX,(SI) +done: + RET +DATA ·table+0(SB)/8,$·imported(SB) +GLOBL ·table(SB),8,$8 +`) + if err != nil { + t.Fatal(err) + } + final := map[string]string{ + "·f": "__plan9asm_lock_rmw_0_amd64_add32_native", + "·table": "__plan9asm_lock_rmw_1_amd64_add32_native", + "·imported": "imported", + } + var unknown []string + resolve := func(symbol string) string { + if name, ok := final[symbol]; ok { + return name + } + unknown = append(unknown, symbol) + return symbol + } + sigs := map[string]FuncSig{ + final["·f"]: {Name: final["·f"], Ret: Void}, + "imported": {Name: "__plan9asm_lock_rmw_2_amd64_add32_native", Ret: Void}, + } + if namespace := x86LockScalarNamespace(file, "amd64", resolve, sigs); namespace != "__plan9asm_lock_rmw_3_" || len(unknown) != 0 { + t.Fatalf("helper namespace must bind final function/data/import names, not FP or local labels: namespace=%q unknown=%v", namespace, unknown) + } +} + +func TestX86LockScalarRMWUnrelatedTranslationKeepsResolverDomain(t *testing.T) { + file, err := Parse(ArchAMD64, `TEXT ·f(SB),4,$0-8 + MOVQ value+0(FP),AX +done: + RET +`) + if err != nil { + t.Fatal(err) + } + var calls, unknown []string + resolve := func(symbol string) string { + calls = append(calls, symbol) + if symbol != "·f" { + unknown = append(unknown, symbol) + } + return strings.TrimPrefix(symbol, "·") + } + sig := FuncSig{Name: "f", Args: []LLVMType{I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}} + ir, err := Translate(file, Options{Goarch: "amd64", ResolveSym: resolve, Sigs: map[string]FuncSig{"f": sig}}) + if err != nil || len(unknown) != 0 || len(calls) != 6 || strings.Contains(ir, "__plan9asm_lock_rmw_") { + t.Fatalf("a no-LOCK translation changed its existing resolver domain/count: error=%v calls=%v", err, calls) + } +} diff --git a/amd64_lock_scalar_rmw_native.go b/amd64_lock_scalar_rmw_native.go new file mode 100644 index 000000000..d85569dd7 --- /dev/null +++ b/amd64_lock_scalar_rmw_native.go @@ -0,0 +1,248 @@ +package plan9asm + +import ( + "fmt" + "sort" + "strings" +) + +// A namespace is a compiler name allocation, never a source certificate. +func x86LockScalarNamespace(file *File, arch string, resolve func(string) string, sigs map[string]FuncSig) string { + if len(x86LockScalarRMWHelperForms(file, arch)) == 0 { + return "" + } + var names []string + reserve := func(symbol string) { + names = append(names, symbol) + } + for name, sig := range sigs { + // Signature keys and names are already final LLVM symbols. + reserve(name) + reserve(sig.Name) + } + for _, fn := range file.Funcs { + name := resolve(fn.Sym) + reserve(funcSigSymbol(name, sigs[name])) + for _, ins := range fn.Instrs { + for _, operand := range ins.Args { + symbol := "" + if operand.Kind == OpSym { + symbol = strings.TrimSpace(strings.TrimPrefix(operand.Sym, "*")) + } else if operand.Kind == OpMem { + symbol = operand.Mem.Sym + } + if base, _, ok := parseSBRef(symbol); ok && base != "" { + // Only SB identities can become LLVM globals; local branch + // labels and FP/AUTO component names never reach ResolveSym. + base = strings.TrimPrefix(base, "$") + reserve(resolveDataAddressSymbol(file, base, resolve, sigs)) + } + } + } + } + for _, data := range file.Data { + reserve(resolveDataSymbol(data.Sym, resolve)) + if data.Addr != "" { + if symbol, _, err := dataAddress(data); err == nil { + reserve(resolveDataAddressSymbol(file, symbol, resolve, sigs)) + } + } + } + for _, global := range file.Globl { + reserve(resolveDataSymbol(global.Sym, resolve)) + } + for number := 0; ; number++ { + prefix := fmt.Sprintf("__plan9asm_lock_rmw_%d_", number) + collision := false + for _, name := range names { + if strings.Contains(name, prefix) { + collision = true + break + } + } + if !collision { + return prefix + } + } +} + +func x86LockScalarHelperName(namespace string, form x86LockScalarRMWForm, arch string) string { + segment := "native" + if form.destination.Kind == OpMem && form.destination.Mem.Segment != "" { + segment = strings.ToLower(string(form.destination.Mem.Segment)) + } + name := namespace + arch + "_" + strings.ToLower(form.spec.stem) + fmt.Sprint(form.spec.bits) + "_" + segment + if form.source.Kind == OpImm { + name += fmt.Sprintf("_imm_%016x", uint64(form.source.Imm)) + } + return name +} + +func x86LockScalarPointerType(form x86LockScalarRMWForm) string { + if form.destination.Kind == OpMem { + switch form.destination.Mem.Segment { + case FS: + return "ptr addrspace(257)" + case GS: + return "ptr addrspace(256)" + } + } + return "ptr" +} + +func x86LockScalarRMWHelperForms(file *File, arch string) map[string]x86LockScalarRMWForm { + forms := make(map[string]x86LockScalarRMWForm) + if file == nil || file.Arch != ArchAMD64 || arch != "amd64" && arch != "386" { + return forms + } + for _, fn := range file.Funcs { + for _, block := range amd64SplitBlocks(fn) { + for index := 1; index < len(block.instrs); index++ { + prefix, body := block.instrs[index-1], block.instrs[index] + if !strings.EqualFold(string(prefix.Op), "LOCK") || len(prefix.Args) != 0 { + continue + } + form, recognized, err := x86LockScalarRMWProperties(body.Op, body, arch) + if !recognized || err != nil || x86LockScalarNativeMemoryBound(form, body, arch) != nil { + continue // The ordinary final consumer retains the actual error. + } + forms[x86LockScalarHelperName("", form, arch)] = form + } + } + } + return forms +} + +func emitX86LockScalarRMWHelpers(b *strings.Builder, file *File, arch, namespace string) { + if namespace == "" { + return + } + forms := x86LockScalarRMWHelperForms(file, arch) + names := make([]string, 0, len(forms)) + for name := range forms { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + emitX86LockScalarRMWHelper(b, namespace+name, forms[name], arch) + } +} + +func emitX86LockScalarRMWHelper(b *strings.Builder, name string, form x86LockScalarRMWForm, arch string) { + spec := form.spec + typ, word := amd64IntegerTypeForBits(spec.bits), I64 + push, pop, carryTest := "pushfq", "popq", "btq" + if arch == "386" { + word, push, pop, carryTest = I32, "pushfl", "popl", "btl" + } + pointer := x86LockScalarPointerType(form) + fmt.Fprintf(b, "define internal %s @%s(%s %%memory, %s %%source, %s %%carry) noinline noredzone {\nentry:\n", + word, name, pointer, typ, word) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[spec.bits] + segment := "" + if form.destination.Kind == OpMem && form.destination.Mem.Segment != "" { + segment = "%" + strings.ToLower(string(form.destination.Mem.Segment)) + ":" + } + // The only native stack use is balanced within this compiler-owned helper. + // noinline+noredzone prevents a later inliner/red-zone allocation from + // invalidating it. No source SP/FP allocation or extra LAHF ISA is borrowed. + if spec.stem == "NOT" { + fmt.Fprintf(b, " call void asm sideeffect \"lock; not%s %s$0\", \"=*m,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %%memory, %s elementtype(%s) %%memory)\n", + width, segment, pointer, typ, pointer, typ) + fmt.Fprintf(b, " ret %s 0\n}\n\n", word) + return + } + opcode := "lock; " + strings.ToLower(spec.stem) + width + " " + if spec.unary { + opcode += segment + "$1" + } else { + opcode += "$2, " + segment + "$1" + } + if spec.carryInput { + opcode = carryTest + " $$0, $3; " + opcode + } + opcode += "; " + push + "; " + pop + " $0" + register := "r" + if spec.bits == 8 { + register = "q" + } + source := "%source" + if form.source.Kind == OpImm { + register, source = "i", fmt.Sprint(form.source.Imm) + } + // Keep fixed operand numbering even for unary forms. The unused source is + // a compile-time zero, not an invented machine input. Output flags are + // written after every input is consumed; indirect *m is read/write. + fmt.Fprintf(b, " %%flags = call %s asm sideeffect \"%s\", \"=r,=*m,%s,r,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %%memory, %s %s, %s %%carry, %s elementtype(%s) %%memory)\n", + word, opcode, register, pointer, typ, typ, source, word, pointer, typ) + fmt.Fprintf(b, " ret %s %%flags\n}\n\n", word) +} + +func (c *amd64Ctx) lowerLockScalarRMW(op Op, ins Instr) (bool, bool, error) { + if !c.namedAtomicLock { + return false, false, nil + } + form, recognized, err := x86LockScalarRMWProperties(op, ins, c.goarch) + if !recognized { + return false, false, nil + } + if err != nil { + return true, false, err + } + if err := x86LockScalarNativeMemoryBound(form, ins, c.goarch); err != nil { + return true, false, err + } + if c.lockScalarNS == "" { + return true, false, fmt.Errorf("%w: scalar LOCK has no compiler-owned complete bridge", ErrProbeNeedsContext) + } + pointer, pointerType, err := c.compareExchangePointer(form.destination) + if err != nil { + return true, false, err + } + if pointerType != x86LockScalarPointerType(form) { + return true, false, fmt.Errorf("%w: scalar LOCK native address space changed", ErrProbeNeedsContext) + } + typ, word := amd64IntegerTypeForBits(form.spec.bits), I64 + if c.goarch == "386" { + word = I32 + } + source := "0" + if !form.spec.unary { + if form.source.Kind == OpImm { + source = fmt.Sprint(form.source.Imm) + } else if source, err = c.evalIntSized(form.source, typ); err != nil { + return true, false, err + } + } + carry := "0" + if form.spec.carryInput { + flag := c.loadFlag(c.flagsCFSlot) + name := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", name, flag, word) + carry = "%" + name + } + flags := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, %s %s)\n", flags, word, + x86LockScalarHelperName(c.lockScalarNS, form, c.goarch), pointerType, pointer, typ, source, word, carry) + for _, flag := range []struct { + mask uint8 + bit int + slot string + }{ + {x86ByteFlagCF, 0, c.flagsCFSlot}, + {x86ByteFlagPF, 2, c.flagsPFSlot}, + {x86ByteFlagAF, 4, c.flagsAFSlot}, + {x86ByteFlagZF, 6, c.flagsZSlot}, + {x86ByteFlagSF, 7, c.flagsSltSlot}, + {x86ByteFlagOF, 11, c.flagsOFSlot}, + } { + if form.spec.flagsDefined&flag.mask == 0 { + continue // Preserved/undefined bits are not forged into defined facts. + } + shift, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", shift, word, flags, flag.bit) + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", value, word, shift) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", value, flag.slot) + } + return true, false, nil +} diff --git a/amd64_lock_scalar_rmw_objects_test.go b/amd64_lock_scalar_rmw_objects_test.go new file mode 100644 index 000000000..8911748f2 --- /dev/null +++ b/amd64_lock_scalar_rmw_objects_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "encoding/hex" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86LockScalarRMWThreeAPIsFiveTargetsNativeObjects(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 compiler and disassembler are required") + } + for _, target := range x86OrderingHintTargets { + var source, declarations strings.Builder + declarations.WriteString("package ordering\n") + type expectedInstruction struct { + op string + bits int + segment x86asm.Reg + } + want := make(map[string]expectedInstruction) + for index, row := range x86LockScalarOracleForms(target.arch) { + for _, segment := range []string{"native", "FS", "GS"} { + memory := "1(SI)" + if segment != "native" { + memory = "1(" + segment + ")" + } + body := strings.Replace(row.text, "1(SI)", memory, 1) + name := fmt.Sprintf("rmw%d_%s", index, strings.ToLower(segment)) + original := strings.Replace(x86NamedLockOracleSource(target.arch, + "CMPL AX,AX\n\tLOCK\n\t"+body), "·locked", "·"+name, 1) + x86LockScalarOriginalInstruction(t, target.arch, original) + source.WriteString(original) + fmt.Fprintf(&declarations, "func %s(p *[16]byte)\n", name) + + bits := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[row.op[len(row.op)-1]] + stem := strings.ToLower(row.op[:len(row.op)-1]) + helper := "__plan9asm_lock_rmw_0_" + target.arch + "_" + + stem + fmt.Sprint(bits) + "_" + strings.ToLower(segment) + if strings.Contains(row.text, "$1") { + helper += "_imm_0000000000000001" + } + physicalSegment := x86asm.Reg(0) + if segment == "FS" { + physicalSegment = x86asm.FS + } else if segment == "GS" { + physicalSegment = x86asm.GS + } + want[helper] = expectedInstruction{strings.ToUpper(stem), bits, physicalSegment} + } + } + + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), + declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "atomicrmw ") || strings.Contains(ir, " = cmpxchg ") { + t.Fatal("source RMW acquired a guessed LLVM atomic alignment or software replacement") + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + dir := t.TempDir() + input, object := filepath.Join(dir, "lock.ll"), filepath.Join(dir, "lock.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, + "-mtriple="+target.triple, "-filetype=obj", input, "-o", object)) + listing := x86FloatCompareCaptureCommand(t, + exec.Command(disassembler, "--disassemble", object)) + sections := regexp.MustCompile(`(?m)^[0-9a-f]+ <([^>]+)>:\n`).FindAllSubmatchIndex(listing, -1) + seen := make(map[string]bool) + for index, section := range sections { + name := string(listing[section[2]:section[3]]) + if strings.HasPrefix(name, "___plan9asm_lock_rmw_") { + name = strings.TrimPrefix(name, "_") // Mach-O/COFF leading underscore. + } + expected, known := want[name] + if !known { + continue + } + if seen[name] { + t.Fatal("duplicate compiler helper", name) + } + end := len(listing) + if index+1 < len(sections) { + end = sections[index+1][0] + } + var code []byte + for _, line := range strings.Split(string(listing[section[1]:end]), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasSuffix(fields[0], ":") { + continue + } + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := hex.DecodeString(field) + if err != nil { + break + } + code = append(code, value[0]) + } + } + mode := 64 + if target.arch == "386" { + mode = 32 + } + count := 0 + for len(code) != 0 { + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len <= 0 || physical.Len > len(code) { + t.Fatalf("incomplete physical helper %s: %x %v", name, code, err) + } + machine := code[:physical.Len] + code = code[physical.Len:] + locked, segmentPrefixes := false, 0 + for _, prefix := range physical.Prefix { + if byte(prefix) == 0xf0 { + locked = true + } + if byte(prefix) == 0x64 || byte(prefix) == 0x65 { + segmentPrefixes++ + } + } + if !locked { + continue + } + count++ + var memory x86asm.Mem + foundMemory := false + for _, argument := range physical.Args { + if value, ok := argument.(x86asm.Mem); ok { + memory, foundMemory = value, true + } + } + wantSegments := 0 + if expected.segment != 0 { + wantSegments = 1 + } + if physical.Op.String() != expected.op || !foundMemory || + physical.MemBytes*8 != expected.bits || + memory.Segment != expected.segment || segmentPrefixes != wantSegments || + physical.AddrSize != mode { + t.Fatalf("native width/segment/address/prefix identity changed: %s bytes=%x decoded=%s memBytes=%d segmentPrefixes=%d expected=%+v", + name, machine, x86asm.GoSyntax(physical, 0, nil), physical.MemBytes, segmentPrefixes, expected) + } + } + if count != 1 { + t.Fatalf("%s helper has %d complete native RMW instructions", name, count) + } + seen[name] = true + } + if len(seen) != len(want) { + t.Fatalf("emitted %d/%d complete native helpers\n%s", len(seen), len(want), listing) + } + t.Logf("actual %s object independently decoded %d whole native RMW helpers", target.triple, len(seen)) + }) + } + }) + } + } +} diff --git a/amd64_lock_scalar_rmw_runtime_test.go b/amd64_lock_scalar_rmw_runtime_test.go new file mode 100644 index 000000000..2d1a9519e --- /dev/null +++ b/amd64_lock_scalar_rmw_runtime_test.go @@ -0,0 +1,459 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +type x86LockScalarRuntimeCase struct { + name, stem, body string + width int + immediate bool + literal uint64 +} + +func x86LockScalarRuntimeCases(arch string) []x86LockScalarRuntimeCase { + var rows []x86LockScalarRuntimeCase + for _, form := range x86LockScalarOracleForms(arch) { + width := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[form.op[len(form.op)-1]] + stem := form.op[:len(form.op)-1] + immediate := strings.Contains(form.text, "$1") + rows = append(rows, x86LockScalarRuntimeCase{ + name: fmt.Sprintf("sample%d", len(rows)), stem: stem, + body: strings.ReplaceAll(strings.Replace(form.text, "1(SI)", "(SI)", 1), "CL,", "CX,"), + width: width, immediate: immediate, literal: 1, + }) + if !immediate || stem == "BTC" || stem == "BTR" || stem == "BTS" { + continue + } + // Independent Yi32 boundaries and equal-bit-pattern spellings. Native Q + // sign-extends imm32; the smaller widths retain their low encoded bits. + for _, literal := range []uint64{0x7fffffff, 0x80000000, 0xffffffff} { + rows = append(rows, x86LockScalarRuntimeCase{ + name: fmt.Sprintf("sample%d", len(rows)), stem: stem, + body: strings.Replace(strings.Replace(form.text, "1(SI)", "(SI)", 1), + "$1,", fmt.Sprintf("$%d,", literal), 1), + width: width, immediate: true, literal: literal, + }) + } + } + return rows +} + +func x86LockScalarRuntimeFixture(arch string) (declarations, assembly, driver, reference string, count int) { + rows := x86LockScalarRuntimeCases(arch) + word, suffix, slot := "uint64", "Q", 8 + if arch == "386" { + word, suffix, slot = "uint32", "L", 4 + } + var decl, source, goRows, prototypes, cRows strings.Builder + decl.WriteString("package main\n") + for _, row := range rows { + fmt.Fprintf(&decl, "func %s(memory *[128]byte, output *[8]uint64, offset %s, value %s, seed %s)\n", + row.name, word, word, word) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n", row.name, slot*5) + fmt.Fprintf(&source, "MOV%s memory+0(FP),SI\nMOV%s output+%d(FP),DI\nMOV%s offset+%d(FP),AX\n", + suffix, suffix, slot, suffix, slot*2) + fmt.Fprintf(&source, "ADD%s AX,SI\nMOV%s value+%d(FP),CX\nMOV%s seed+%d(FP),DX\n", + suffix, suffix, slot*3, suffix, slot*4) + fmt.Fprintf(&source, "PUSH%s DX\nPOPF%s\nSTMXCSR 24(DI)\nLOCK\n%s\nPUSHF%s\nPOP%s AX\n", + suffix, suffix, row.body, suffix, suffix) + fmt.Fprintf(&source, "MOV%s AX,0(DI)\nMOV%s CX,8(DI)\nMOV%s DX,16(DI)\nSTMXCSR 32(DI)\nRET\n", + suffix, suffix, suffix) + fmt.Fprintf(&goRows, "{%s,%q,%d,%t,%d},\n", row.name, row.stem, row.width, row.immediate, row.literal) + cWord := "uint64_t" + if arch == "386" { + cWord = "uint32_t" + } + fmt.Fprintf(&prototypes, "extern void %s(uint8_t *, uint64_t *, %s, %s, %s);\n", row.name, cWord, cWord, cWord) + imm := 0 + if row.immediate { + imm = 1 + } + fmt.Fprintf(&cRows, "{%s,%q,%d,%d,UINT64_C(%d)},\n", row.name, row.stem, row.width, imm, row.literal) + } + driver = strings.NewReplacer("@WORD@", word, "@ROWS@", goRows.String()).Replace(x86LockScalarOriginalReference) + cWord := "uint64_t" + if arch == "386" { + cWord = "uint32_t" + } + reference = strings.NewReplacer("@WORD@", cWord, "@PROTOTYPES@", prototypes.String(), "@ROWS@", cRows.String()).Replace(x86LockScalarIndependentC) + return decl.String(), source.String(), driver, reference, len(rows) +} + +func TestCrossLinuxRuntimeMatrixX86LockScalarRMWOriginalAndLLVM(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly, driver, reference, count := x86LockScalarRuntimeFixture("amd64") + if count != 174 { + t.Fatal("independent complete runtime kernel denominator changed", count) + } + original := x86OrderingHintBuildOriginal(t, t.TempDir(), runtime.GOOS, "amd64", declarations, assembly, driver) + want := fmt.Sprintf("lock observations %d\n", count*292*2*8) + if output := execution.run(t, original); string(output) != want { + t.Fatalf("actual original Go initialized-memory/defined-flags/canary oracle: %q, want %q", output, want) + } + t.Logf("actual original Go: %d kernels / %d complete initialized-memory observations", count, count*292*2*8) + ir := x86NarrowOracleIR(t, assembly, declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + if output := execution.compileAndRunLLVM(t, "lock-rmw", ir, reference, level); string(output) != want { + t.Fatalf("independent integer arithmetic/bit-string/memory/flags/canary oracle: %q", output) + } + t.Logf("%d complete observations: all initialized memory bytes, defined/preserved status bits, source registers, MXCSR and canaries", count*292*2*8) + }) + } +} + +func TestX86LockScalarRMWOriginal386StaticELFCompileOnly(t *testing.T) { + declarations, assembly, driver, _, count := x86LockScalarRuntimeFixture("386") + if count != 129 { + t.Fatal("complete mode-32 runtime kernel denominator changed", count) + } + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go %d-kernel static IA32 ELF: %d bytes SHA256=%x; execution is a distinct required native axis", + count, sealed.size, sealed.hash) +} + +const x86LockScalarOriginalReference = `package main + +import ( + "fmt" + "math/bits" + "runtime" +) + +type word = @WORD@ +type kernel struct { + fn func(*[128]byte, *[8]uint64, word, word, word) + op string + width int + immediate bool + literal uint64 +} +var kernels = []kernel{ + @ROWS@ +} + +func load(p *[128]byte, offset, width int) uint64 { + var value uint64 + for i := 0; i < width/8; i++ { + value |= uint64(p[offset+i]) << (8*i) + } + return value +} + +func store(p *[128]byte, offset, width int, value uint64) { + for i := 0; i < width/8; i++ { + p[offset+i] = byte(value >> (8*i)) + } +} + +func reference(k kernel, memory *[128]byte, offset int, b, seed uint64) (uint64, uint64) { + mask := ^uint64(0) + if k.width < 64 { + mask = (uint64(1) << k.width) - 1 + } + if k.immediate { + b = uint64(int64(int32(k.literal))) + } + b &= mask + if k.op == "BTC" || k.op == "BTR" || k.op == "BTS" { + index := int64(b) + if k.width == 16 { + index = int64(int16(b)) + } else if k.width == 32 { + index = int64(int32(b)) + } + if k.immediate { + index = int64(k.literal % uint64(k.width)) + } + quotient, bit := index/int64(k.width), index%int64(k.width) + if bit < 0 { + quotient-- + bit += int64(k.width) + } + offset += int(quotient) * (k.width / 8) + a := load(memory, offset, k.width) + cf := (a >> uint(bit)) & 1 + switch k.op { + case "BTC": + a ^= uint64(1) << uint(bit) + case "BTR": + a &^= uint64(1) << uint(bit) + case "BTS": + a |= uint64(1) << uint(bit) + } + store(memory, offset, k.width, a) + return cf | seed&64, 65 // Other status bits are architecturally undefined. + } + a := load(memory, offset, k.width) + if k.op == "NOT" { + store(memory, offset, k.width, ^a) + return seed & 0x8d5, 0x8d5 + } + if k.op == "AND" || k.op == "OR" || k.op == "XOR" { + switch k.op { + case "AND": + a &= b + case "OR": + a |= b + case "XOR": + a ^= b + } + a &= mask + store(memory, offset, k.width, a) + flags := uint64(0) + if bits.OnesCount8(uint8(a))%2 == 0 { flags |= 4 } + if a == 0 { flags |= 64 } + if a>>(k.width-1) != 0 { flags |= 128 } + return flags, 0x8c5 // AF is undefined, not preserved or forced to zero. + } + carry := uint64(0) + if k.op == "ADC" || k.op == "SBB" { + carry = seed & 1 + } + add := k.op == "ADD" || k.op == "ADC" || k.op == "INC" + if k.op == "INC" || k.op == "DEC" { b = 1 } + if k.op == "NEG" { b, a = a, 0 } + result, cf := bits.Sub64(a, b, carry) + af := (a & 15) < (b & 15) + carry + if add { + result, cf = bits.Add64(a, b, carry) + if k.width < 64 && result > mask { cf = 1 } + af = (a & 15) + (b & 15) + carry > 15 + } + result &= mask + sign := uint64(1) << (k.width - 1) + overflow := (a^b)&(a^result)&sign != 0 + if add { overflow = ^(a^b)&(a^result)&sign != 0 } + if k.op == "INC" || k.op == "DEC" { cf = seed & 1 } + flags := cf + if bits.OnesCount8(uint8(result))%2 == 0 { flags |= 4 } + if af { flags |= 16 } + if result == 0 { flags |= 64 } + if result&sign != 0 { flags |= 128 } + if overflow { flags |= 2048 } + store(memory, offset, k.width, result) + return flags, 0x8d5 +} + +func main() { + runtime.LockOSThread() + edges := []uint64{0,1,15,16,17,127,128,255,256,32767,32768,65535, + 0x7fffffff,0x80000000,0xffffffff,0x7fffffffffffffff,0x8000000000000000,^uint64(0)} + const sentinel = uint64(0xa7a7a7a7a7a7a7a7) + observations := 0 + var memory, expected [128]byte + var out [8]uint64 + for _, k := range kernels { + for sample := 0; sample < 292; sample++ { + a, b := uint64(sample/16), uint64(sample%16) + if sample >= 256 { + a = edges[(sample-256)/2] + b = edges[((sample-256)/2 + 1 + 6*(sample%2))%len(edges)] + } + if (k.op == "BTC" || k.op == "BTR" || k.op == "BTS") && !k.immediate { + index := []int{-2*k.width-1,-k.width,-1,0,1,k.width-1,k.width,k.width+1,2*k.width} + b = uint64(int64(index[sample%len(index)])) + } + b = uint64(word(b)) + for _, seed := range []uint64{0x8c7, 0x12} { + for unaligned := 0; unaligned < 8; unaligned++ { + for i := range memory { memory[i] = byte(i*73+19) ^ byte(sample*29) } + offset := 64 + unaligned + store(&memory, offset, k.width, a) + expected = memory + for i := range out { out[i] = sentinel } + flags, valid := reference(k, &expected, offset, b, seed) + k.fn(&memory, &out, word(offset), word(b), word(seed)) + wordMask := uint64(^word(0)) + if memory != expected || out[0]&valid != flags || + out[1]&wordMask != b || out[2]&wordMask != seed || + uint32(out[3]) != uint32(out[4]) { + panic(fmt.Sprintf("%s/%d imm=%t literal=%x sample=%d unaligned=%d b=%x seed=%x flags=%x/%x mask=%x out=%x", + k.op,k.width,k.immediate,k.literal,sample,unaligned,b,seed,out[0],flags,valid,out)) + } + for i := 0; i < 3; i++ { + if out[i]&^wordMask != sentinel&^wordMask { panic("upper output canary") } + } + if out[3]>>32 != sentinel>>32 || out[4]>>32 != sentinel>>32 || + out[5] != sentinel || out[6] != sentinel || out[7] != sentinel { + panic("output extent canary") + } + observations++ + } + } + } + } + fmt.Printf("lock observations %d\n", observations) +} +` + +const x86LockScalarIndependentC = ` +#include +#include +#include + +#if defined(__i386__) +// The largest IA32 operation is 32 bits; 64-bit intermediates close its carry +// and signed-overflow bounds without relying on unsupported int128 on i386. +typedef uint64_t wide_unsigned; +typedef int64_t wide_signed; +#else +typedef __uint128_t wide_unsigned; +typedef __int128 wide_signed; +#endif + +@PROTOTYPES@ +typedef @WORD@ word; +typedef void (*function)(uint8_t *, uint64_t *, word, word, word); +struct kernel { + function fn; + const char *op; + unsigned width, immediate; + uint64_t literal; +}; +static const struct kernel kernels[] = { + @ROWS@ +}; + +static uint64_t load(const uint8_t *memory, int offset, unsigned width) { + uint64_t value = 0; + for (unsigned i = 0; i < width / 8; i++) { + value |= (uint64_t)memory[offset + i] << (8 * i); + } + return value; +} +static void store(uint8_t *memory, int offset, unsigned width, uint64_t value) { + for (unsigned i = 0; i < width / 8; i++) { + memory[offset + i] = value >> (8 * i); + } +} +static uint64_t reference(const struct kernel *k, uint8_t *memory, + int offset, uint64_t b, uint64_t seed, uint64_t *valid) { + uint64_t mask = k->width == 64 ? UINT64_MAX : (UINT64_C(1) << k->width) - 1; + if (k->immediate) { + b = (uint64_t)(int64_t)(int32_t)k->literal; + } + b &= mask; + if (!strcmp(k->op, "BTC") || !strcmp(k->op, "BTR") || !strcmp(k->op, "BTS")) { + int64_t index = (int64_t)b; + if (k->width == 16) { index = (int16_t)b; } + if (k->width == 32) { index = (int32_t)b; } + if (k->immediate) { index = k->literal % k->width; } + int64_t quotient = index / (int64_t)k->width; + int64_t bit = index % (int64_t)k->width; + if (bit < 0) { + quotient--; + bit += k->width; + } + offset += quotient * (k->width / 8); + uint64_t a = load(memory, offset, k->width), cf = (a >> bit) & 1; + uint64_t selected = UINT64_C(1) << bit; + if (!strcmp(k->op, "BTC")) { a ^= selected; } + if (!strcmp(k->op, "BTR")) { a &= ~selected; } + if (!strcmp(k->op, "BTS")) { a |= selected; } + store(memory, offset, k->width, a); + *valid = 65; + return cf | (seed & 64); + } + uint64_t a = load(memory, offset, k->width), result; + *valid = 0x8d5; + if (!strcmp(k->op, "NOT")) { + store(memory, offset, k->width, ~a); + return seed & *valid; + } + unsigned cf = 0, af = 0, of = 0; + if (!strcmp(k->op, "AND") || !strcmp(k->op, "OR") || !strcmp(k->op, "XOR")) { + result = !strcmp(k->op, "AND") ? a & b : !strcmp(k->op, "OR") ? a | b : a ^ b; + *valid = 0x8c5; + } else { + unsigned carry = !strcmp(k->op, "ADC") || !strcmp(k->op, "SBB") ? seed & 1 : 0; + unsigned add = !strcmp(k->op, "ADD") || !strcmp(k->op, "ADC") || !strcmp(k->op, "INC"); + if (!strcmp(k->op, "INC") || !strcmp(k->op, "DEC")) { b = 1; } + if (!strcmp(k->op, "NEG")) { b = a; a = 0; } + wide_unsigned wide = add ? (wide_unsigned)a + b + carry : (wide_unsigned)a - b - carry; + result = wide & mask; + cf = add ? wide > mask : (wide_unsigned)a < (wide_unsigned)b + carry; + af = add ? (a & 15) + (b & 15) + carry > 15 : (a & 15) < (b & 15) + carry; + wide_signed sign = (wide_signed)1 << (k->width - 1); + wide_signed sa = (a & (uint64_t)sign) ? (wide_signed)a - 2 * sign : a; + wide_signed sb = (b & (uint64_t)sign) ? (wide_signed)b - 2 * sign : b; + wide_signed signed_wide = add ? sa + sb + carry : sa - sb - carry; + of = signed_wide < -sign || signed_wide >= sign; + if (!strcmp(k->op, "INC") || !strcmp(k->op, "DEC")) { cf = seed & 1; } + } + result &= mask; + unsigned ones = 0; + for (unsigned bit = 0; bit < 8; bit++) { ones += (result >> bit) & 1; } + uint64_t flags = cf | ((ones % 2 == 0) << 2) | (af << 4) | + ((result == 0) << 6) | ((result >> (k->width - 1)) << 7) | (of << 11); + store(memory, offset, k->width, result); + return flags; +} + +int main(void) { + static const uint64_t edges[] = {0,1,15,16,17,127,128,255,256,32767,32768,65535, + 0x7fffffff,0x80000000,0xffffffff,0x7fffffffffffffff,0x8000000000000000,UINT64_MAX}; + const uint64_t sentinel = UINT64_C(0xa7a7a7a7a7a7a7a7); + unsigned observations = 0; + for (unsigned kernel = 0; kernel < sizeof(kernels) / sizeof(kernels[0]); kernel++) { + const struct kernel *k = &kernels[kernel]; + for (unsigned sample = 0; sample < 292; sample++) { + uint64_t a = sample / 16, b = sample % 16; + if (sample >= 256) { + a = edges[(sample - 256) / 2]; + b = edges[((sample - 256) / 2 + 1 + 6 * (sample % 2)) % 18]; + } + if ((!strcmp(k->op, "BTC") || !strcmp(k->op, "BTR") || !strcmp(k->op, "BTS")) && !k->immediate) { + const int index[] = {-2*(int)k->width-1,-(int)k->width,-1,0,1, + (int)k->width-1,(int)k->width,(int)k->width+1,2*(int)k->width}; + b = (uint64_t)(int64_t)index[sample % 9]; + } + b = (word)b; + for (unsigned s = 0; s < 2; s++) { + uint64_t seed = s ? 0x12 : 0x8c7; + for (unsigned unaligned = 0; unaligned < 8; unaligned++) { + uint8_t memory[128], expected[128]; + uint64_t out[8]; + for (unsigned i = 0; i < 128; i++) { memory[i] = (i*73+19) ^ (sample*29); } + int offset = 64 + unaligned; + store(memory, offset, k->width, a); + memcpy(expected, memory, 128); + for (unsigned i = 0; i < 8; i++) { out[i] = sentinel; } + uint64_t valid, flags = reference(k, expected, offset, b, seed, &valid); + unsigned before_csr, after_csr; + __asm__ volatile("stmxcsr %0" : "=m"(before_csr)); + k->fn(memory, out, offset, b, seed); + __asm__ volatile("stmxcsr %0" : "=m"(after_csr)); + uint64_t word_mask = (word)~(word)0; + if (memcmp(memory, expected, 128) || (out[0] & valid) != flags || + (out[1] & word_mask) != b || (out[2] & word_mask) != seed || + (uint32_t)out[3] != before_csr || (uint32_t)out[4] != before_csr || + after_csr != before_csr) { + fprintf(stderr, "%s/%u imm=%u literal=%llx sample=%u unaligned=%u b=%llx seed=%llx flags=%llx/%llx mask=%llx\n", + k->op,k->width,k->immediate,(unsigned long long)k->literal,sample,unaligned, + (unsigned long long)b,(unsigned long long)seed,(unsigned long long)out[0], + (unsigned long long)flags,(unsigned long long)valid); + return 1; + } + for (unsigned i = 0; i < 3; i++) { + if ((out[i] & ~word_mask) != (sentinel & ~word_mask)) { return 2; } + } + if (out[3] >> 32 != sentinel >> 32 || out[4] >> 32 != sentinel >> 32 || + out[5] != sentinel || out[6] != sentinel || out[7] != sentinel) { return 3; } + observations++; + } + } + } + } + printf("lock observations %u\n", observations); + return 0; +} +` diff --git a/amd64_lock_scalar_rmw_sdk_test.go b/amd64_lock_scalar_rmw_sdk_test.go new file mode 100644 index 000000000..bdbaa70f2 --- /dev/null +++ b/amd64_lock_scalar_rmw_sdk_test.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86LockScalarRMWActualSDKOr8(t *testing.T) { + var data []byte + var sourcePath string + for _, relative := range []string{ + "src/internal/runtime/atomic/atomic_amd64.s", + "src/runtime/internal/atomic/atomic_amd64.s", + } { + path := filepath.Join(runtime.GOROOT(), filepath.FromSlash(relative)) + actual, err := os.ReadFile(path) + if os.IsNotExist(err) { + continue // Actual SDK source layout, not an encoder-version guess. + } + if err != nil || sourcePath != "" { + t.Fatal("actual SDK atomic source is missing or ambiguous", err) + } + sourcePath, data = path, actual + } + start := strings.Index(string(data), "TEXT ·Or8(SB)") + if sourcePath == "" || start < 0 { + t.Fatal("actual SDK lacks the discovering Or8 source") + } + end := strings.Index(string(data[start:]), "\nTEXT ") + if end < 0 { + t.Fatal("SDK Or8 has no complete next-TEXT boundary") + } + body := string(data[start:start+end]) + "\n" + header, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "pkg/include/textflag.h")) + if err != nil { + t.Fatal(err) + } + // The original body is unchanged. The actual stock header, not a manual + // NOSPLIT substitution, supplies the same Go assembler definitions. + original := string(header) + "\n" + body + machine := x86LockScalarOriginalInstruction(t, "amd64", original) + t.Logf("actual %s Or8 source SHA256=%x header SHA256=%x whole native instruction=%x", + runtime.Version(), sha256.Sum256(data), sha256.Sum256(header), machine) + expanded, err := PreprocessAssemblySource(original, AssemblyPreprocessOptions{}) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required for the actual SDK sample") + } + for _, target := range x86OrderingHintTargets { + if target.arch != "amd64" { + continue // This original SDK file is explicitly AMD64. + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, expanded, + "package ordering\nfunc Or8(ptr *uint8, val uint8)\n", "amd64", target.triple) + if err != nil || !strings.Contains(ir, "lock; orb") { + t.Fatal("actual SDK Or8 retains its whole-source atomic effect", err) + } + compileLLVMToObject(t, llc, target.triple, "or8.ll", fmt.Sprintf("or8-%s.o", api), ir) + }) + } + } + if after, err := os.ReadFile(sourcePath); err != nil || sha256.Sum256(after) != sha256.Sum256(data) { + t.Fatal("actual SDK source changed during sample proof", err) + } +} diff --git a/amd64_lock_scalar_rmw_symbol_test.go b/amd64_lock_scalar_rmw_symbol_test.go new file mode 100644 index 000000000..e14123671 --- /dev/null +++ b/amd64_lock_scalar_rmw_symbol_test.go @@ -0,0 +1,46 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86LockScalarRMWAddressLiteralIsNotMemory(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LockScalarOracleForms(arch) { + body := strings.Replace(row.text, "1(SI)", "$·cell(SB)", 1) + t.Run(arch+"/"+body, func(t *testing.T) { + source := "GLOBL ·cell(SB),8,$16\n" + + x86NamedLockOracleSource(arch, "CLC\nLOCK\n"+body) + // An emitted address literal is Go TYPE_ADDR, not writable + // memory. This is actual assembler rejection, not execution. + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[len(file.Funcs[0].Instrs)-2] + form, recognized, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if !recognized || err == nil { + t.Error("typed memory grammar accepted an address literal", form, err) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, source, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if err == nil || ir != "" { + t.Error("address literal became a locked memory object", api, err) + } + } + ordinary := strings.Replace(source, "$·cell(SB)", "·cell(SB)", 1) + x86LockScalarOriginalInstruction(t, arch, ordinary) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86OrderingHintTranslateAPI(t, api, ordinary, + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if err != nil || !strings.Contains(ir, "lock; ") { + t.Error("ordinary bound SB memory lost its existing atomic effect", api, err) + } + } + }) + } + } +} diff --git a/amd64_lower_abs_sign.go b/amd64_lower_abs_sign.go new file mode 100644 index 000000000..9d884a735 --- /dev/null +++ b/amd64_lower_abs_sign.go @@ -0,0 +1,255 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedAbsSuffix struct { + broadcast bool + zeroing bool +} + +func parseAMD64PackedAbsSuffix(rawOp, baseOp string) (amd64PackedAbsSuffix, error) { + var result amd64PackedAbsSuffix + if rawOp == baseOp { + return result, nil + } + suffixes := strings.Split(strings.TrimPrefix(rawOp, baseOp+"."), ".") + for index, suffix := range suffixes { + switch suffix { + case "BCST": + if result.broadcast || result.zeroing || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .BCST suffix", baseOp) + } + result.broadcast = true + case "Z": + if result.zeroing || index != len(suffixes)-1 { + return result, fmt.Errorf("amd64 %s has invalid or misplaced .Z suffix", baseOp) + } + result.zeroing = true + default: + return result, fmt.Errorf("amd64 %s has unsupported suffix .%s", baseOp, suffix) + } + } + return result, nil +} + +// lowerPackedAbsSign implements Go 1.27's complete PABS*, VPABS*, PSIGN*, +// and VPSIGN* families. VPABS B/W/D share _yvmovddup, VPABSQ uses +// _yvexpandpd, and VPSIGN B/W/D are the two-width VEX-only _yvaddsubpd rows. +func (c *amd64Ctx) lowerPackedAbsSign(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if laneBits, matched := amd64PackedAbsLaneBits(baseOp); matched { + properties, err := parseAMD64PackedAbsSuffix(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + return c.lowerPackedAbs(baseOp, laneBits, ins, properties) + } + if laneBits, vector, matched := amd64PackedSignLaneBits(baseOp); matched { + if rawOp != baseOp { + return true, false, fmt.Errorf("amd64 %s does not accept suffixes: %q", baseOp, ins.Raw) + } + return c.lowerPackedSign(baseOp, laneBits, vector, ins) + } + return false, false, nil +} + +func amd64PackedAbsLaneBits(op string) (int, bool) { + switch op { + case "PABSB", "VPABSB": + return 8, true + case "PABSW", "VPABSW": + return 16, true + case "PABSD", "VPABSD": + return 32, true + case "VPABSQ": + return 64, true + default: + return 0, false + } +} + +func amd64PackedSignLaneBits(op string) (laneBits int, vector bool, matched bool) { + switch op { + case "PSIGNB", "VPSIGNB": + laneBits = 8 + case "PSIGNW", "VPSIGNW": + laneBits = 16 + case "PSIGND", "VPSIGND": + laneBits = 32 + default: + return 0, false, false + } + return laneBits, strings.HasPrefix(op, "V"), true +} + +func (c *amd64Ctx) lowerPackedAbs(baseOp string, laneBits int, ins Instr, properties amd64PackedAbsSuffix) (bool, bool, error) { + vector := strings.HasPrefix(baseOp, "V") + if !vector { + if properties != (amd64PackedAbsSuffix{}) || len(ins.Args) != 2 { + return true, false, fmt.Errorf("amd64 %s expects X/m128 source and X destination: %q", baseOp, ins.Raw) + } + if !c.isGoLegacyPackedAbsSignRegister(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's X register class: %q", c.goarch, baseOp, ins.Raw) + } + if (ins.Args[0].Kind == OpReg && !c.isGoLegacyPackedAbsSignRegister(ins.Args[0])) || + (ins.Args[0].Kind != OpReg && !isAMD64MemoryOperand(ins.Args[0])) { + return true, false, fmt.Errorf("%s %s source must be an in-range X register or memory: %q", c.goarch, baseOp, ins.Raw) + } + values, err := c.loadPackedCompareLanes(ins.Args[0], 16, laneBits, false) + if err != nil { + return true, false, err + } + result := c.emitPackedAbs(values, 16, laneBits) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <16 x i8>\n", out, 128/laneBits, laneBits, result) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) + } + + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("amd64 %s expects source, [K mask,] destination: %q", baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s .Z requires K1-K7: %q", baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + if properties.broadcast && laneBits != 32 && laneBits != 64 { + return true, false, fmt.Errorf("amd64 %s has no Go 1.27 broadcast form: %q", baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s destination must be X, Y, or Z: %q", baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 %s destination is outside Go 1.27's vector class: %q", baseOp, ins.Raw) + } + source := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory source: %q", baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("amd64 %s source width does not match its destination: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("amd64 %s source must be a matching vector register or memory: %q", baseOp, ins.Raw) + } + values, err := c.loadPackedCompareLanes(source, byteWidth, laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + result := c.emitPackedAbs(values, byteWidth, laneBits) + lanes := byteWidth * 8 / laneBits + if masked { + mask, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedAbs(values string, byteWidth, laneBits int) string { + lanes := byteWidth * 8 / laneBits + negative := c.newTmp() + negated := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i%d> %s, zeroinitializer\n", negative, lanes, laneBits, values) + fmt.Fprintf(c.b, " %%%s = sub <%d x i%d> zeroinitializer, %s\n", negated, lanes, laneBits, values) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %%%s, <%d x i%d> %s\n", result, lanes, negative, lanes, laneBits, negated, lanes, laneBits, values) + return "%" + result +} + +func (c *amd64Ctx) lowerPackedSign(baseOp string, laneBits int, vector bool, ins Instr) (bool, bool, error) { + wantArgs := 2 + if vector { + wantArgs = 3 + } + if len(ins.Args) != wantArgs { + return true, false, fmt.Errorf("amd64 %s expects %d operands: %q", baseOp, wantArgs, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s destination must be X or Y: %q", baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if vector { + if byteWidth != 16 && byteWidth != 32 || !c.isGoVEXPackedAbsSignRegister(destination, byteWidth) || !c.isGoVEXPackedAbsSignRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s requires matching in-range X or Y data registers: %q", c.goarch, baseOp, ins.Raw) + } + } else { + byteWidth = 16 + if !c.isGoLegacyPackedAbsSignRegister(destination) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's X register class: %q", c.goarch, baseOp, ins.Raw) + } + } + control := ins.Args[0] + if control.Kind == OpReg { + valid := c.isGoVEXPackedAbsSignRegister(control, byteWidth) + if !vector { + valid = c.isGoLegacyPackedAbsSignRegister(control) + } + if !valid { + return true, false, fmt.Errorf("%s %s control source is outside Go 1.27's register class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(control) { + return true, false, fmt.Errorf("amd64 %s control source must be a matching vector or memory: %q", baseOp, ins.Raw) + } + controlValues, err := c.loadPackedCompareLanes(control, byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + dataOperand := destination + if vector { + dataOperand = ins.Args[1] + } + dataValues, err := c.loadPackedCompareLanes(dataOperand, byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / laneBits + positive := c.newTmp() + negative := c.newTmp() + negated := c.newTmp() + nonpositive := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt <%d x i%d> %s, zeroinitializer\n", positive, lanes, laneBits, controlValues) + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i%d> %s, zeroinitializer\n", negative, lanes, laneBits, controlValues) + fmt.Fprintf(c.b, " %%%s = sub <%d x i%d> zeroinitializer, %s\n", negated, lanes, laneBits, dataValues) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %%%s, <%d x i%d> zeroinitializer\n", nonpositive, lanes, negative, lanes, laneBits, negated, lanes, laneBits) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %s, <%d x i%d> %%%s\n", result, lanes, positive, lanes, laneBits, dataValues, lanes, laneBits, nonpositive) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) isGoLegacyPackedAbsSignRegister(operand Operand) bool { + if !amd64VEXVectorRegister(operand, 16) { + return false + } + index, _ := amd64ParseXReg(operand.Reg) + return c.goarch != "386" || index < 8 +} + +func (c *amd64Ctx) isGoVEXPackedAbsSignRegister(operand Operand, byteWidth int) bool { + return amd64VEXVectorRegister(operand, byteWidth) +} diff --git a/amd64_lower_abs_sign_test.go b/amd64_lower_abs_sign_test.go new file mode 100644 index 000000000..c19a1c6c8 --- /dev/null +++ b/amd64_lower_abs_sign_test.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedAbsSignCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses yxm_q4 for the legacy PABS*/PSIGN* rows, + // _yvmovddup for VPABSB/W/D, _yvexpandpd for VPABSQ, and + // _yvaddsubpd for the VEX-only VPSIGN* rows. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedabssignforms(SB),NOSPLIT,$0-0\n") + for index, suffix := range []string{"B", "W", "D"} { + fmt.Fprintf(&source, "\tPABS%s X0, X1\n", suffix) + fmt.Fprintf(&source, "\tPABS%s %d(AX), X2\n", suffix, 16*index) + fmt.Fprintf(&source, "\tPSIGN%s X3, X4\n", suffix) + fmt.Fprintf(&source, "\tPSIGN%s %d(AX), X5\n", suffix, 48+16*index) + fmt.Fprintf(&source, "\tVPABS%s X6, X7\n", suffix) + fmt.Fprintf(&source, "\tVPABS%s %d(AX), Y1\n", suffix, 96+32*index) + fmt.Fprintf(&source, "\tVPABS%s X20, X21\n", suffix) + fmt.Fprintf(&source, "\tVPABS%s X8, K1, X9\n", suffix) + fmt.Fprintf(&source, "\tVPABS%s.Z %d(AX), K2, Y3\n", suffix, 192+32*index) + fmt.Fprintf(&source, "\tVPABS%s Z4, K3, Z5\n", suffix) + fmt.Fprintf(&source, "\tVPSIGN%s X8, X9, X10\n", suffix) + fmt.Fprintf(&source, "\tVPSIGN%s %d(AX), Y11, Y12\n", suffix, 288+32*index) + } + source.WriteString(` VPABSD.BCST 384(AX), Y11 + VPABSD.BCST.Z 388(AX), K4, Z6 + VPABSQ X0, X1 + VPABSQ 400(AX), Y2 + VPABSQ Z3, K5, Z4 + VPABSQ.Z 464(AX), K6, Z5 + VPABSQ.BCST 472(AX), X6 + VPABSQ.BCST.Z 480(AX), K7, Y7 + RET +`) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedabssignforms": {Name: "packedabssignforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"icmp slt <", "icmp sgt <", "sub <", "select <"} { + if !strings.Contains(ll, want) { + t.Fatalf("packed abs/sign IR is missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-abs-sign-"+target.name+".ll", "packed-abs-sign-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedAbsSignRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PABSB Y0, X1", + "PABSQ X0, X1", + "PABSD.Z X0, X1", + "VPABSB X0, Y1", + "VPABSW Y0, X1", + "VPABSD Z0, Y1", + "VPABSQ X0, Y1", + "VPABSQ X0, K0, X1", + "VPABSB.Z X0, X1", + "VPABSW.BCST (AX), X1", + "VPABSQ.BCST X0, X1", + "PSIGNB Y0, X1", + "PSIGNQ X0, X1", + "VPSIGNB X0, X1", + "VPSIGNW Z0, Z1, Z2", + "VPSIGND X0, K1, X1, X2", + "VPSIGNB.Z X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedAbsSignRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PABSB X8, X0", + "PABSW X0, X8", + "PSIGND X8, X0", + } { + assertX86PackedAbsSignRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedAbsSignRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed abs/sign tables", instruction) + } +} + +func TestAMD64PackedAbsSignRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedabssign(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ control+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), SI + KMOVQ SI, K1 + + MOVOU (BX), X0 + PABSB X0, X0 + MOVOU X0, 0(AX) + + MOVOU (BX), X0 + PSIGNW (CX), X0 + MOVOU X0, 16(AX) + + VMOVDQU (DX), Y1 + VPABSD.BCST (BX), K1, Y1 + VMOVDQU Y1, 32(AX) + + VMOVDQU (BX), Y0 + VPSIGND (CX), Y0, Y2 + VMOVDQU Y2, 64(AX) + + VMOVDQU64 (BX), Z0 + VPABSQ.Z Z0, K1, Z1 + VMOVDQU64 Z1, 96(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedabssign": { + Name: "packedabssign", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +extern void packedabssign(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint8_t abs8(uint8_t v) { return (v & 0x80) ? (uint8_t)(0-v) : v; } +static int16_t sign16(int16_t value, int16_t control) { + return control > 0 ? value : control < 0 ? (int16_t)(0-(uint16_t)value) : 0; +} +static int32_t sign32(int32_t value, int32_t control) { + return control > 0 ? value : control < 0 ? (int32_t)(0-(uint32_t)value) : 0; +} +static int32_t abs32(int32_t value) { return value < 0 ? (int32_t)(0-(uint32_t)value) : value; } +static int64_t abs64(int64_t value) { return value < 0 ? (int64_t)(0-(uint64_t)value) : value; } +int main(void) { + union { uint8_t b[64]; int16_t w[32]; int32_t d[16]; int64_t q[8]; } source, control, old; + uint8_t out[160]; + for (int i = 0; i < 64; i++) { + source.b[i] = (uint8_t)(i * 29 + 0x80); + control.b[i] = (uint8_t)(i * 17 - 71); + old.b[i] = (uint8_t)(0xf0 - i); + } + control.w[0] = 1; control.w[1] = 0; control.w[2] = -1; control.w[3] = INT16_MIN; + control.d[0] = 1; control.d[1] = 0; control.d[2] = -1; control.d[3] = INT32_MIN; + memset(out, 0xcc, sizeof(out)); + const uint64_t mask = 0xa5; + packedabssign(out, source.b, control.b, old.b, mask); + for (int i = 0; i < 16; i++) if (out[i] != abs8(source.b[i])) return 10+i; + const int16_t *got16 = (const int16_t *)(out+16); + for (int i = 0; i < 8; i++) if (got16[i] != sign16(source.w[i], control.w[i])) return 40+i; + const int32_t *gotAbs32 = (const int32_t *)(out+32); + int32_t broadcast = abs32(source.d[0]); + for (int i = 0; i < 8; i++) { + int32_t want = (mask >> i) & 1 ? broadcast : old.d[i]; + if (gotAbs32[i] != want) return 60+i; + } + const int32_t *gotSign32 = (const int32_t *)(out+64); + for (int i = 0; i < 8; i++) if (gotSign32[i] != sign32(source.d[i], control.d[i])) return 80+i; + const int64_t *gotAbs64 = (const int64_t *)(out+96); + for (int i = 0; i < 8; i++) { + int64_t want = (mask >> i) & 1 ? abs64(source.q[i]) : 0; + if (gotAbs64[i] != want) return 100+i; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_abs_sign", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_accumulator_sign_extend.go b/amd64_lower_accumulator_sign_extend.go new file mode 100644 index 000000000..19f96868e --- /dev/null +++ b/amd64_lower_accumulator_sign_extend.go @@ -0,0 +1,50 @@ +package plan9asm + +import "fmt" + +// lowerAccumulatorSignExtend implements the complete Go 1.27 ynone family +// sharing opcodes 98/99: CBW/CWDE/CDQE extend into AX/EAX/RAX, while +// CWD/CDQ/CQO produce the sign half in DX/EDX/RDX. None of them modify flags. +func (c *amd64Ctx) lowerAccumulatorSignExtend(op Op, ins Instr) (ok bool, terminated bool, err error) { + sourceType := LLVMType("") + targetType := LLVMType("") + destination := AX + extendValue := true + switch op { + case "CBW": + sourceType, targetType = I8, I16 + case "CWDE": + sourceType, targetType = I16, I32 + case "CDQE": + if c.goarch == "386" { + return true, false, fmt.Errorf("386 CDQE is illegal in 32-bit mode: %q", ins.Raw) + } + sourceType, targetType = I32, I64 + case "CWD": + sourceType, targetType, destination, extendValue = I16, I16, DX, false + case "CDQ": + sourceType, targetType, destination, extendValue = I32, I32, DX, false + case "CQO": + if c.goarch == "386" { + return true, false, fmt.Errorf("386 CQO is illegal in 32-bit mode: %q", ins.Raw) + } + sourceType, targetType, destination, extendValue = I64, I64, DX, false + default: + return false, false, nil + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands in Go 1.27's ynone table: %q", c.goarch, op, ins.Raw) + } + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, sourceType) + if err != nil { + return true, false, err + } + result := c.newTmp() + if extendValue { + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", result, sourceType, value, targetType) + } else { + bits, _ := amd64IntegerTypeBits(sourceType) + fmt.Fprintf(c.b, " %%%s = ashr %s %s, %d\n", result, sourceType, value, bits-1) + } + return true, false, c.storeRegSized(destination, targetType, "%"+result) +} diff --git a/amd64_lower_accumulator_sign_extend_test.go b/amd64_lower_accumulator_sign_extend_test.go new file mode 100644 index 000000000..1deb194f1 --- /dev/null +++ b/amd64_lower_accumulator_sign_extend_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func x86AccumulatorSignExtendCompleteFormsSource(goarch string) string { + ops := []string{"CBW", "CWDE", "CWD", "CDQ"} + if goarch == "amd64" { + ops = append(ops, "CDQE", "CQO") + } + var source strings.Builder + source.WriteString("TEXT signextendforms(SB),$0-0\n") + for _, op := range ops { + source.WriteString("\t" + op + "\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86AccumulatorSignExtendCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86AccumulatorSignExtendCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"signextendforms": {Name: "signextendforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"sext i8", "sext i16", "ashr i16", "ashr i32"} { + if !strings.Contains(ir, want) { + t.Fatalf("accumulator sign-extension lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" { + for _, want := range []string{"sext i32", "ashr i64"} { + if !strings.Contains(ir, want) { + t.Fatalf("amd64 accumulator sign-extension lowering omitted %q:\n%s", want, ir) + } + } + } + compileLLVMToObject(t, llc, target.triple, "accumulator-sign-extend-"+target.name+".ll", "accumulator-sign-extend-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86AccumulatorSignExtendRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CBW AX"}, + {goarch: "amd64", instruction: "CWDE $1"}, + {goarch: "amd64", instruction: "CDQ AX, DX"}, + {goarch: "amd64", instruction: "CWD.Z"}, + {goarch: "386", instruction: "CDQE"}, + {goarch: "386", instruction: "CQO"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ynone tables", test.instruction) + } + }) + } +} + +func TestAMD64AccumulatorSignExtendRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT signextendsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + + MOVQ $0x1122334455660080, AX + CBW + MOVQ AX, 0(DI) + + MOVQ $0x1122334455667788, DX + MOVW $0x8001, AX + CWD + MOVQ DX, 8(DI) + + MOVQ $0x1122334455668001, AX + CWDE + MOVQ AX, 16(DI) + + MOVL $0x80000001, AX + MOVQ $0x1122334455667788, DX + CDQ + MOVQ DX, 24(DI) + + MOVL $0x80000001, AX + CDQE + MOVQ AX, 32(DI) + + MOVQ $0x8000000000000001, AX + CQO + MOVQ DX, 40(DI) + + SETCS 48(DI) + SETOS 49(DI) + SETEQ 50(DI) + SETMI 51(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "signextendsemantics": { + Name: "signextendsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void signextendsemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[56] = {0}; + signextendsemantics(out); + if (load64(out+0) != UINT64_C(0x112233445566ff80)) return 10; + if (load64(out+8) != UINT64_C(0x112233445566ffff)) return 11; + if (load64(out+16) != UINT64_C(0x00000000ffff8001)) return 12; + if (load64(out+24) != UINT64_C(0x00000000ffffffff)) return 13; + if (load64(out+32) != UINT64_C(0xffffffff80000001)) return 14; + if (load64(out+40) != UINT64_C(0xffffffffffffffff)) return 15; + if (out[48] != 1 || out[49] != 1 || out[50] != 0 || out[51] != 1) return 16; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "accumulator_sign_extend_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_adc_sbb.go b/amd64_lower_adc_sbb.go new file mode 100644 index 000000000..8ddee0cc9 --- /dev/null +++ b/amd64_lower_adc_sbb.go @@ -0,0 +1,152 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarADCSBB implements every operand row in Go 1.27's yxorb (B) +// and yaddl (W/L/Q) tables for add-with-carry and subtract-with-borrow. +func (c *amd64Ctx) lowerScalarADCSBB(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + add, bits, recognized := amd64ScalarADCSBBProperties(baseOp) + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects source and destination: %q", c.goarch, baseOp, ins.Raw) + } + if err := validateAMD64ScalarAddSubOperands(c.goarch, baseOp, bits, ins.Args[0], ins.Args[1], ins); err != nil { + return true, false, err + } + + source, destination := ins.Args[0], ins.Args[1] + typ := amd64IntegerTypeForBits(bits) + var sourceValue string + if source.Kind == OpImm { + sourceValue = amd64ScalarAddSubImmediate(source.Imm, bits) + } else { + sourceValue, destination, err = c.evalScalarArithmeticSource(ins, typ) + if err != nil { + return true, false, err + } + } + destinationValue, store, err := c.loadIntDestination(destination, typ) + if err != nil { + return true, false, err + } + + wideType := LLVMType(fmt.Sprintf("i%d", bits*2)) + widen := func(value string) string { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", wide, typ, value, wideType) + return "%" + wide + } + wideDestination := widen(destinationValue) + wideSource := widen(sourceValue) + carry := c.loadFlag(c.flagsCFSlot) + wideCarryName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", wideCarryName, carry, wideType) + wideCarry := "%" + wideCarryName + + partial := c.newTmp() + wideResult := c.newTmp() + var carryOut string + if add { + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", partial, wideType, wideDestination, wideSource) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %s\n", wideResult, wideType, partial, wideCarry) + carryOut = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", carryOut, wideType, wideResult, amd64UnsignedMaximum(bits)) + } else { + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", partial, wideType, wideSource, wideCarry) + fmt.Fprintf(c.b, " %%%s = sub %s %s, %%%s\n", wideResult, wideType, wideDestination, partial) + carryOut = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %%%s\n", carryOut, wideType, wideDestination, partial) + } + + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", result, wideType, wideResult, typ) + resultValue := "%" + result + if err := store(resultValue); err != nil { + return true, false, err + } + c.setScalarADCSBBFlags(typ, add, destinationValue, sourceValue, resultValue, "%"+carryOut) + return true, false, nil +} + +func amd64ScalarADCSBBProperties(op string) (add bool, bits int, ok bool) { + switch op { + case "ADCB": + return true, 8, true + case "ADCW": + return true, 16, true + case "ADCL": + return true, 32, true + case "ADCQ": + return true, 64, true + case "SBBB": + return false, 8, true + case "SBBW": + return false, 16, true + case "SBBL": + return false, 32, true + case "SBBQ": + return false, 64, true + default: + return false, 0, false + } +} + +func amd64UnsignedMaximum(bits int) string { + switch bits { + case 8: + return "255" + case 16: + return "65535" + case 32: + return "4294967295" + case 64: + return "18446744073709551615" + default: + panic(fmt.Sprintf("unsupported x86 integer width %d", bits)) + } +} + +func (c *amd64Ctx) setScalarADCSBBFlags(typ LLVMType, add bool, destination, source, result, carry string) { + c.setAuxiliaryCarryFlag(typ, destination, source, result) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", carry, c.flagsCFSlot) + + xorOperands := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorOperands, typ, destination, source) + overflowInput := "%" + xorOperands + if add { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", inverted, typ, overflowInput) + overflowInput = "%" + inverted + } + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorResult, typ, destination, result) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", overflowBits, typ, overflowInput, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", overflow, typ, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + + zero := c.newTmp() + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + c.setParityFlagSized(typ, result) +} diff --git a/amd64_lower_adc_sbb_test.go b/amd64_lower_adc_sbb_test.go new file mode 100644 index 000000000..be2e687cc --- /dev/null +++ b/amd64_lower_adc_sbb_test.go @@ -0,0 +1,171 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86ADCSBBCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT adcsbbforms(SB),NOSPLIT,$32-0\n") + for _, stem := range []string{"ADC", "SBB"} { + for _, width := range []string{"B", "W", "L", "Q"} { + if target.goarch == "386" && width == "Q" { + continue + } + op := stem + width + srcReg, dstReg := "BX", "AX" + if width == "B" { + srcReg, dstReg = "BL", "AL" + } + fmt.Fprintf(&source, "\t%s $1, %s\n", op, dstReg) + fmt.Fprintf(&source, "\t%s %s, %s\n", op, srcReg, dstReg) + fmt.Fprintf(&source, "\t%s 8(BX), %s\n", op, dstReg) + fmt.Fprintf(&source, "\t%s $1, 16(BX)\n", op) + fmt.Fprintf(&source, "\t%s %s, 24(BX)\n", op, srcReg) + if target.goarch == "386" && width != "B" { + fmt.Fprintf(&source, "\t%s $0, SP\n", op) + } + if target.goarch == "amd64" && width == "Q" { + fmt.Fprintf(&source, "\t%s $4294967295, AX\n", op) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"adcsbbforms": {Name: "adcsbbforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "adc-sbb-"+target.name+".ll", "adc-sbb-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ADCSBBRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCB SP, AL"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SBBB AL, SP"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCQ 0(BX), 8(CX)"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "SBBL X0, AX"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCW.Z AX, BX"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ADCQ $4294967296, AX"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "ADCQ AX, BX"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "SBBQ $1, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "ADCL R8, AX"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+test.instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ADC/SBB tables for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64ADCSBBMemoryRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT adcsbbmemory(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0xffffffffffffffff, 0(DI) + CLC + ADCQ $1, 0(DI) + SETCS 32(DI) + SBBQ $0, 0(DI) + SETCS 33(DI) + MOVL $0x7fffffff, 8(DI) + STC + ADCL $0, 8(DI) + SETOS 34(DI) + MOVW $0, 16(DI) + STC + SBBW $0, 16(DI) + SETCS 35(DI) + MOVB $0xff, 24(DI) + CLC + ADCB $1, 24(DI) + SETCS 36(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "adcsbbmemory": {Name: "adcsbbmemory", Args: []LLVMType{Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void adcsbbmemory(uint8_t *); +int main(void) { + uint8_t out[40] = {0}; + adcsbbmemory(out); + if (*(uint64_t *)(out+0) != UINT64_MAX) return 10; + if (*(uint32_t *)(out+8) != 0x80000000U) return 11; + if (*(uint16_t *)(out+16) != 0xffffU) return 12; + if (out[24] != 0) return 13; + for (int i = 32; i <= 36; i++) if (out[i] != 1) return 20+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "adc_sbb_memory", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_addsub.go b/amd64_lower_addsub.go new file mode 100644 index 000000000..b4e890a71 --- /dev/null +++ b/amd64_lower_addsub.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarAddSub implements all operand rows in Go 1.27's yxorb (B) and +// yaddl (W/L/Q) tables for ADD and SUB. +func (c *amd64Ctx) lowerScalarAddSub(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + add, bits, recognized := amd64ScalarAddSubProperties(baseOp) + if !recognized { + return false, false, nil + } + if err := validateAMD64ScalarAddSubInstruction(c.goarch, ins); err != nil { + return true, false, err + } + source, destination := ins.Args[0], ins.Args[1] + typ := amd64IntegerTypeForBits(bits) + var sourceValue string + if source.Kind == OpImm { + sourceValue = amd64ScalarAddSubImmediate(source.Imm, bits) + } else { + sourceValue, destination, err = c.evalScalarArithmeticSource(ins, typ) + if err != nil { + return true, false, err + } + } + destinationValue, store, err := c.loadIntDestination(destination, typ) + if err != nil { + return true, false, err + } + result := c.newTmp() + llvmOp := "sub" + if add { + llvmOp = "add" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", result, llvmOp, typ, destinationValue, sourceValue) + resultValue := "%" + result + if err := store(resultValue); err != nil { + return true, false, err + } + c.setScalarAddSubFlags(typ, add, destinationValue, sourceValue, resultValue) + return true, false, nil +} + +// validateAMD64ScalarAddSubInstruction is deliberately independent of a +// lowering context so the same Go-assembler table is enforced before either +// the direct/linear path or the CFG path is selected. +func validateAMD64ScalarAddSubInstruction(goarch string, ins Instr) error { + rawOp := strings.ToUpper(string(ins.Op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + _, bits, recognized := amd64ScalarAddSubProperties(baseOp) + if !recognized { + return nil + } + if rawOp != baseOp { + return fmt.Errorf("%s %s does not accept instruction suffixes: %q", goarch, baseOp, ins.Raw) + } + if goarch == "386" && bits == 64 { + return fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 2 { + return fmt.Errorf("%s %s expects source and destination: %q", goarch, baseOp, ins.Raw) + } + return validateAMD64ScalarAddSubOperands(goarch, baseOp, bits, ins.Args[0], ins.Args[1], ins) +} + +func validateAMD64ScalarAddSubFunction(goarch string, fn Func) error { + for _, ins := range fn.Instrs { + if err := validateAMD64ScalarAddSubInstruction(goarch, ins); err != nil { + return err + } + } + return nil +} + +func amd64ScalarAddSubProperties(op string) (add bool, bits int, ok bool) { + switch op { + case "ADDB": + return true, 8, true + case "ADDW": + return true, 16, true + case "ADDL": + return true, 32, true + case "ADDQ": + return true, 64, true + case "SUBB": + return false, 8, true + case "SUBW": + return false, 16, true + case "SUBL": + return false, 32, true + case "SUBQ": + return false, 64, true + default: + return false, 0, false + } +} + +func validateAMD64ScalarAddSubOperands(goarch, baseOp string, bits int, source, destination Operand, ins Instr) error { + registerAllowed := func(reg Reg) bool { + if bits == 8 { + // Although SP is in the wider general-register class, Go's + // ADD/SUB byte tables do not accept it as a direct byte operand. + return reg != SP && isGoYmbRegisterForArch(reg, goarch) + } + return isX86YrlRegisterForArch(reg, goarch) + } + destinationRegister := destination.Kind == OpReg && registerAllowed(destination.Reg) + destinationMemory := isAMD64MemoryOperand(destination) + if !destinationRegister && !destinationMemory { + return fmt.Errorf("%s %s destination is outside Go 1.27's Ymb/Yml classes: %q", goarch, baseOp, ins.Raw) + } + switch source.Kind { + case OpImm: + if goarch != "386" && (source.Imm < -2147483648 || source.Imm > 4294967295) { + return fmt.Errorf("amd64 %s immediate is outside Go 1.27's Yi32 class: %q", baseOp, ins.Raw) + } + case OpReg: + if !registerAllowed(source.Reg) { + return fmt.Errorf("%s %s source is outside Go 1.27's Yrb/Yrl classes: %q", goarch, baseOp, ins.Raw) + } + case OpSym: + if x86AbsoluteRelocationSource(source, goarch, bits) { + return nil // Yi32 address, not a TYPE_MEM read; retain the source operand. + } + fallthrough + default: + if !isAMD64MemoryOperand(source) || !destinationRegister { + return fmt.Errorf("%s %s operands do not match Go 1.27's register/memory rows: %q", goarch, baseOp, ins.Raw) + } + } + return nil +} + +// The assembler emits only the low operand-width bits. Q-width immediates are +// encoded as imm32 and sign-extended by the CPU. +func amd64ScalarAddSubImmediate(immediate int64, bits int) string { + return fmt.Sprintf("%d", amd64ScalarAddSubImmediateInt64(immediate, bits)) +} + +func amd64ScalarAddSubImmediateInt64(immediate int64, bits int) int64 { + var normalized int64 + switch bits { + case 8: + normalized = int64(int8(uint8(immediate))) + case 16: + normalized = int64(int16(uint16(immediate))) + case 32, 64: + normalized = int64(int32(uint32(immediate))) + } + return normalized +} + +func (c *amd64Ctx) setScalarAddSubFlags(typ LLVMType, add bool, destination, source, result string) { + c.setAuxiliaryCarryFlag(typ, destination, source, result) + carry := c.newTmp() + if add { + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", carry, typ, result, destination) + } else { + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", carry, typ, destination, source) + } + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + + xorOperands := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorOperands, typ, destination, source) + overflowInput := "%" + xorOperands + if add { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", inverted, typ, overflowInput) + overflowInput = "%" + inverted + } + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorResult, typ, destination, result) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", overflowBits, typ, overflowInput, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", overflow, typ, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + + zero := c.newTmp() + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + c.setParityFlagSized(typ, result) +} diff --git a/amd64_lower_addsub_float.go b/amd64_lower_addsub_float.go new file mode 100644 index 000000000..84919b773 --- /dev/null +++ b/amd64_lower_addsub_float.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerAlternatingFloatingAddSubtract implements the complete Go 1.27 yxm +// and _yvaddsubpd families. Even lanes subtract the Plan 9 first source from +// the destination/base source; odd lanes add it. +func (c *amd64Ctx) lowerAlternatingFloatingAddSubtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + laneBits := 0 + vector := false + switch baseOp { + case "ADDSUBPS": + laneBits = 32 + case "ADDSUBPD": + laneBits = 64 + case "VADDSUBPS": + laneBits, vector = 32, true + case "VADDSUBPD": + laneBits, vector = 64, true + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed forms in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if vector { + return c.lowerVectorAlternatingFloatingAddSubtract(baseOp, laneBits, ins) + } + return c.lowerLegacyAlternatingFloatingAddSubtract(baseOp, laneBits, ins) +} + +func (c *amd64Ctx) lowerLegacyAlternatingFloatingAddSubtract(baseOp string, laneBits int, ins Instr) (bool, bool, error) { + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg || !c.isGoLegacyXReg(ins.Args[1].Reg) { + return true, false, fmt.Errorf("%s %s expects X/m128, X using Go 1.27's yxm table: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if !c.isGoLegacyXReg(ins.Args[0].Reg) { + return true, false, fmt.Errorf("%s %s source is outside the legacy X class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s source must be X or memory: %q", c.goarch, baseOp, ins.Raw) + } + first, err := c.loadHorizontalFloatingOperand(ins.Args[0], 16, laneBits) + if err != nil { + return true, false, err + } + second, err := c.loadHorizontalFloatingOperand(ins.Args[1], 16, laneBits) + if err != nil { + return true, false, err + } + result := c.emitAlternatingFloatingAddSubtract(16, laneBits, first, second) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <16 x i8>\n", out, amd64FMA3LLVMType(128/laneBits, laneBits), result) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) +} + +func (c *amd64Ctx) lowerVectorAlternatingFloatingAddSubtract(baseOp string, laneBits int, ins Instr) (bool, bool, error) { + if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects src1, src2, destination: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(ins.Args[2].Reg) + if byteWidth != 16 && byteWidth != 32 || + !amd64VEXVectorRegister(ins.Args[1], byteWidth) || + !amd64VEXVectorRegister(ins.Args[2], byteWidth) { + return true, false, fmt.Errorf("%s %s second source and destination must be same-width VEX X/Y registers: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if !amd64VEXVectorRegister(ins.Args[0], byteWidth) { + return true, false, fmt.Errorf("%s %s first source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s first source must be a vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + first, err := c.loadHorizontalFloatingOperand(ins.Args[0], byteWidth, laneBits) + if err != nil { + return true, false, err + } + second, err := c.loadHorizontalFloatingOperand(ins.Args[1], byteWidth, laneBits) + if err != nil { + return true, false, err + } + result := c.emitAlternatingFloatingAddSubtract(byteWidth, laneBits, first, second) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, amd64FMA3LLVMType(byteWidth*8/laneBits, laneBits), result, byteWidth) + return true, false, c.storeVectorBytes(ins.Args[2].Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitAlternatingFloatingAddSubtract(byteWidth, laneBits int, first, second string) string { + lanes := byteWidth * 8 / laneBits + typeName := amd64FMA3LLVMType(lanes, laneBits) + elementType := amd64FMA3LLVMType(1, laneBits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + left := c.newTmp() + right := c.newTmp() + value := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", left, typeName, second, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", right, typeName, first, lane) + operation := "fsub" + if lane%2 == 1 { + operation = "fadd" + } + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %%%s\n", value, operation, elementType, left, right) + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, typeName, result, elementType, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_addsub_float_test.go b/amd64_lower_addsub_float_test.go new file mode 100644 index 000000000..dc6c30ad9 --- /dev/null +++ b/amd64_lower_addsub_float_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86AlternatingFloatingAddSubtractCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT alternatingfloataddsubforms(SB),NOSPLIT,$0-0\n") + for _, element := range []string{"PS", "PD"} { + op := "ADDSUB" + element + fmt.Fprintf(&src, "\t%s X0, X1\n", op) + fmt.Fprintf(&src, "\t%s (AX), X7\n", op) + vop := "V" + op + fmt.Fprintf(&src, "\t%s X0, X1, X2\n", vop) + fmt.Fprintf(&src, "\t%s (AX), X14, X15\n", vop) + fmt.Fprintf(&src, "\t%s Y0, Y1, Y2\n", vop) + fmt.Fprintf(&src, "\t%s (AX), Y14, Y15\n", vop) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "alternatingfloataddsubforms": {Name: "alternatingfloataddsubforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "alternating-float-addsub-"+target.name+".ll", "alternating-float-addsub-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86AlternatingFloatingAddSubtractRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, op := range []string{"ADDSUBPS", "ADDSUBPD", "VADDSUBPS", "VADDSUBPD"} { + forms := []string{"X0", "Y0, Y1", "X0, X1, X2"} + valid := "X0, X1" + if strings.HasPrefix(op, "V") { + forms = []string{"X0", "X0, X1", "X0, Y1, Y2", "Z0, Z1, Z2", "X0, X1, K1, X2"} + valid = "X0, X1, X2" + } + for _, operands := range forms { + instruction := op + " " + operands + t.Run(strings.NewReplacer(" ", "_", ",", "_").Replace(instruction), func(t *testing.T) { + assertX86AlternatingAddSubRejected(t, instruction) + }) + } + for _, suffix := range []string{".Z", ".BCST", ".SAE", ".RN_SAE"} { + instruction := op + suffix + " " + valid + t.Run(strings.NewReplacer(" ", "_", ".", "_").Replace(instruction), func(t *testing.T) { + assertX86AlternatingAddSubRejected(t, instruction) + }) + } + } +} + +func assertX86AlternatingAddSubRejected(t *testing.T, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yxm/_yvaddsubpd tables", instruction) + } +} diff --git a/amd64_lower_addvec.go b/amd64_lower_addvec.go new file mode 100644 index 000000000..53a472dbb --- /dev/null +++ b/amd64_lower_addvec.go @@ -0,0 +1,141 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedAddMode uint8 + +const ( + amd64PackedAddWrap amd64PackedAddMode = iota + amd64PackedAddSignedSaturating + amd64PackedAddUnsignedSaturating +) + +type amd64PackedAddForm uint8 + +const ( + // amd64PackedAddLegacyYMM is Go's ymm table: MMX/m64 and XMM/m128. + // Go only exposes the MMX rows in amd64 mode. + amd64PackedAddLegacyYMM amd64PackedAddForm = iota + // amd64PackedAddLegacyYXM is Go's yxm table: XMM/m128 only. + amd64PackedAddLegacyYXM + // amd64PackedAddVEXEVEX is Go's _yvandnpd table: X/Y/Z sources, + // optional EVEX masking, and opt-in D/Q memory broadcast. + amd64PackedAddVEXEVEX +) + +type amd64PackedAddSpec struct { + canonical Op + form amd64PackedAddForm + laneBits int + mode amd64PackedAddMode + allowBroadcast bool +} + +// amd64PackedAddSpecs is the complete typed grammar for Go 1.27's packed-add +// instruction family. Keep aliases here rather than duplicating lowering: +// cmd/asm maps PADDD to APADDL, while its VEX spelling remains VPADDD. +var amd64PackedAddSpecs = map[Op]amd64PackedAddSpec{ + "PADDB": {canonical: "PADDB", form: amd64PackedAddLegacyYMM, laneBits: 8, mode: amd64PackedAddWrap}, + "PADDW": {canonical: "PADDW", form: amd64PackedAddLegacyYMM, laneBits: 16, mode: amd64PackedAddWrap}, + "PADDL": {canonical: "PADDL", form: amd64PackedAddLegacyYMM, laneBits: 32, mode: amd64PackedAddWrap}, + "PADDD": {canonical: "PADDL", form: amd64PackedAddLegacyYMM, laneBits: 32, mode: amd64PackedAddWrap}, + "PADDQ": {canonical: "PADDQ", form: amd64PackedAddLegacyYXM, laneBits: 64, mode: amd64PackedAddWrap}, + "PADDSB": {canonical: "PADDSB", form: amd64PackedAddLegacyYMM, laneBits: 8, mode: amd64PackedAddSignedSaturating}, + "PADDSW": {canonical: "PADDSW", form: amd64PackedAddLegacyYMM, laneBits: 16, mode: amd64PackedAddSignedSaturating}, + "PADDUSB": {canonical: "PADDUSB", form: amd64PackedAddLegacyYMM, laneBits: 8, mode: amd64PackedAddUnsignedSaturating}, + "PADDUSW": {canonical: "PADDUSW", form: amd64PackedAddLegacyYMM, laneBits: 16, mode: amd64PackedAddUnsignedSaturating}, + "VPADDB": {canonical: "VPADDB", form: amd64PackedAddVEXEVEX, laneBits: 8, mode: amd64PackedAddWrap}, + "VPADDW": {canonical: "VPADDW", form: amd64PackedAddVEXEVEX, laneBits: 16, mode: amd64PackedAddWrap}, + "VPADDD": {canonical: "VPADDD", form: amd64PackedAddVEXEVEX, laneBits: 32, mode: amd64PackedAddWrap, allowBroadcast: true}, + "VPADDQ": {canonical: "VPADDQ", form: amd64PackedAddVEXEVEX, laneBits: 64, mode: amd64PackedAddWrap, allowBroadcast: true}, + "VPADDSB": {canonical: "VPADDSB", form: amd64PackedAddVEXEVEX, laneBits: 8, mode: amd64PackedAddSignedSaturating}, + "VPADDSW": {canonical: "VPADDSW", form: amd64PackedAddVEXEVEX, laneBits: 16, mode: amd64PackedAddSignedSaturating}, + "VPADDUSB": {canonical: "VPADDUSB", form: amd64PackedAddVEXEVEX, laneBits: 8, mode: amd64PackedAddUnsignedSaturating}, + "VPADDUSW": {canonical: "VPADDUSW", form: amd64PackedAddVEXEVEX, laneBits: 16, mode: amd64PackedAddUnsignedSaturating}, +} + +// lowerPackedIntegerAdd implements the complete Go 1.27 packed-integer add +// family: the legacy ymm/yxm tables and the _yvandnpd VEX/EVEX table. +func (c *amd64Ctx) lowerPackedIntegerAdd(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerPackedIntegerAddForm(op, ins) +} + +func (c *amd64Ctx) loadLegacyMMXPackedSource(src Operand) (string, error) { + if src.Kind == OpReg { + if _, ok := amd64ParseMReg(src.Reg); !ok { + return "", fmt.Errorf("expected MMX register or memory, got %s", src.String()) + } + return c.loadReg(src.Reg) + } + if !isAMD64MemoryOperand(src) { + return "", fmt.Errorf("expected MMX register or memory, got %s", src.String()) + } + return c.evalIntSized(src, I64) +} + +func (c *amd64Ctx) bitcastI64ToIntegerLanes(lanes, laneBits int, value string) string { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <%d x i%d>\n", cast, value, lanes, laneBits) + return "%" + cast +} + +func (c *amd64Ctx) emitPackedIntegerAdd(lanes, laneBits int, first, second string, mode amd64PackedAddMode) string { + vecType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + if mode == amd64PackedAddWrap { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", result, vecType, second, first) + return "%" + result + } + + wideBits := laneBits * 2 + wideType := fmt.Sprintf("<%d x i%d>", lanes, wideBits) + firstWide := c.newTmp() + secondWide := c.newTmp() + extend := "sext" + if mode == amd64PackedAddUnsignedSaturating { + extend = "zext" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", firstWide, extend, vecType, first, wideType) + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", secondWide, extend, vecType, second, wideType) + sum := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", sum, wideType, secondWide, firstWide) + + max := int64((uint64(1) << laneBits) - 1) + predicate := "ugt" + if mode == amd64PackedAddSignedSaturating { + max = int64((uint64(1) << (laneBits - 1)) - 1) + predicate = "sgt" + } + above := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s %s %%%s, %s\n", above, predicate, wideType, sum, llvmSplatSignedInteger(lanes, wideBits, max)) + capped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %%%s\n", capped, lanes, above, wideType, llvmSplatSignedInteger(lanes, wideBits, max), wideType, sum) + clamped := "%" + capped + if mode == amd64PackedAddSignedSaturating { + min := -int64(uint64(1) << (laneBits - 1)) + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, %s\n", below, wideType, clamped, llvmSplatSignedInteger(lanes, wideBits, min)) + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", bounded, lanes, below, wideType, llvmSplatSignedInteger(lanes, wideBits, min), wideType, clamped) + clamped = "%" + bounded + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, wideType, clamped, vecType) + return "%" + result +} + +func llvmSplatSignedInteger(lanes, bits int, value int64) string { + var b strings.Builder + b.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "i%d %d", bits, value) + } + b.WriteByte('>') + return b.String() +} diff --git a/amd64_lower_adx.go b/amd64_lower_adx.go new file mode 100644 index 000000000..6654b8994 --- /dev/null +++ b/amd64_lower_adx.go @@ -0,0 +1,107 @@ +package plan9asm + +import "fmt" + +type amd64ADXCarry uint8 + +const ( + amd64ADXCarryCF amd64ADXCarry = iota + amd64ADXCarryOF +) + +type amd64ADXSpec struct { + bits int + prefix byte + carry amd64ADXCarry +} + +// Go 1.27 asm6.go uses yml_rl for every ADX form. One spec binds the +// register/memory grammar, encoded mandatory prefix, width and carry chain. +var amd64ADXSpecs = map[Op]amd64ADXSpec{ + "ADCXL": {bits: 32, prefix: 0x66, carry: amd64ADXCarryCF}, + "ADCXQ": {bits: 64, prefix: 0x66, carry: amd64ADXCarryCF}, + "ADOXL": {bits: 32, prefix: 0xf3, carry: amd64ADXCarryOF}, + "ADOXQ": {bits: 64, prefix: 0xf3, carry: amd64ADXCarryOF}, +} + +// lowerADX implements ADCXL/Q and ADOXL/Q. All four opcodes use Go 1.27's +// yml_rl row: register/memory source and register destination. ADCX consumes +// and updates CF; ADOX independently consumes and updates OF. +func (c *amd64Ctx) lowerADX(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, supported := amd64ADXSpecs[op] + if !supported { + return false, false, nil + } + bits, flagSlot := spec.bits, c.flagsCFSlot + if spec.carry == amd64ADXCarryOF { + flagSlot = c.flagsOFSlot + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", op, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects register/memory source and register destination: %q", c.goarch, op, ins.Raw) + } + source, destination := ins.Args[0], ins.Args[1] + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + sourceValue, err := c.evalADXSource(source, typ, ins.x86AddressBits) + if err != nil { + return true, false, err + } + destinationValue, err := c.evalIntSized(destination, typ) + if err != nil { + return true, false, err + } + carryIn := c.loadFlag(flagSlot) + carryValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", carryValue, carryIn, typ) + sum := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", sum, typ, destinationValue, sourceValue) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", result, typ, sum, carryValue) + carryFromOperands := c.newTmp() + carryFromInput := c.newTmp() + carryOut := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %s\n", carryFromOperands, typ, sum, destinationValue) + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %%%s\n", carryFromInput, typ, result, sum) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", carryOut, carryFromOperands, carryFromInput) + if err := c.storeRegSized(destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carryOut, flagSlot) + return true, false, nil +} + +func (c *amd64Ctx) evalADXSource(source Operand, typ LLVMType, addressBits int) (string, error) { + wordBits := 64 + if c.goarch == "386" { + wordBits = 32 + } + if addressBits == 0 || addressBits == wordBits || source.Kind != OpMem { + return c.evalIntSized(source, typ) + } + if addressBits != wordBits/2 { + return "", fmt.Errorf("invalid ADX address size %d for %s", addressBits, c.goarch) + } + ptr, ptrType, err := c.ptrFromMem(source.Mem) + if err != nil { + return "", err + } + // Wrap the effective offset before applying an FS/GS address space. + address, wrapped, value := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint %s %s to i%d\n", address, ptrType, ptr, addressBits) + fmt.Fprintf(c.b, " %%%s = inttoptr i%d %%%s to %s\n", wrapped, addressBits, address, ptrType) + fmt.Fprintf(c.b, " %%%s = load %s, %s %%%s, align 1\n", value, typ, ptrType, wrapped) + return "%" + value, nil +} diff --git a/amd64_lower_adx_test.go b/amd64_lower_adx_test.go new file mode 100644 index 000000000..cd42d6437 --- /dev/null +++ b/amd64_lower_adx_test.go @@ -0,0 +1,245 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86ADXCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "adxdata" + size := 8 + if goarch == "386" { + symbol = "adxdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT adxforms(SB),$0-0\n") + widths := []string{"L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, stem := range []string{"ADCX", "ADOX"} { + for _, width := range widths { + op := stem + width + fmt.Fprintf(&source, "\t%s AX, BX\n", op) + fmt.Fprintf(&source, "\t%s 8(BX), CX\n", op) + fmt.Fprintf(&source, "\t%s %s(SB), DX\n", op, symbol) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12\n", op) + } else { + fmt.Fprintf(&source, "\t%s AX, SP\n", op) + } + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86ADXCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86ADXCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"adxforms": {Name: "adxforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"add i32", "icmp ult i32", `"target-features"="+adx"`} { + if !strings.Contains(ll, want) { + t.Fatalf("ADX lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "add i64") { + t.Fatalf("ADX Q-width lowering omitted add i64:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "adx-"+target.name+".ll", "adx-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ADXRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "ADCXQ $1, AX"}, + {goarch: "amd64", instruction: "ADOXL AX, 0(BX)"}, + {goarch: "amd64", instruction: "ADCXL X0, AX"}, + {goarch: "amd64", instruction: "ADOXQ AX, X0"}, + {goarch: "amd64", instruction: "ADCXQ AX"}, + {goarch: "amd64", instruction: "ADOXL.Z AX, BX"}, + {goarch: "386", instruction: "ADCXL R8, AX"}, + {goarch: "386", instruction: "ADOXQ AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ADX yml_rl table", test.instruction) + } + }) + } +} + +func TestAMD64ADXRuntimeSemanticsAndIndependentFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT adxsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + + // ADCXL consumes CF, updates only CF, and zero-extends its destination. + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVL $-1, AX + MOVL $0, BX + ADCXL AX, BX + MOVQ BX, 0(DI) + SETCS 8(DI) + SETOS 9(DI) + SETEQ 10(DI) + SETMI 11(DI) + + // ADOXL consumes OF, updates only OF, and leaves CF untouched. + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVL $-1, AX + MOVL $0, BX + ADOXL AX, BX + MOVQ BX, 16(DI) + SETOS 24(DI) + SETCS 25(DI) + SETEQ 26(DI) + SETMI 27(DI) + + // The Q-width memory form uses CF while preserving OF and ZF. + XORQ R8, R8 + STC + MOVQ $-1, 32(DI) + MOVQ $0, BX + ADCXQ 32(DI), BX + MOVQ BX, 40(DI) + SETCS 48(DI) + SETOS 49(DI) + SETEQ 50(DI) + + // ADOXQ memory form consumes OF without changing CF, ZF or SF. + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVQ $0, BX + ADOXQ 32(DI), BX + MOVQ BX, 56(DI) + SETOS 64(DI) + SETCS 65(DI) + SETEQ 66(DI) + SETMI 67(DI) + RET +` + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + const mainC = ` +#include +#include +extern void adxsemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[72] = {0}; + adxsemantics(out); + if (load64(out + 0) != 0) return 10; + const uint8_t adcx_flags[4] = {1, 1, 0, 1}; + for (int i = 0; i < 4; i++) if (out[8+i] != adcx_flags[i]) return 20+i; + if (load64(out + 16) != 0) return 30; + const uint8_t adox_flags[4] = {1, 1, 0, 1}; + for (int i = 0; i < 4; i++) if (out[24+i] != adox_flags[i]) return 40+i; + if (load64(out + 40) != 0) return 50; + const uint8_t adcxq_flags[3] = {1, 0, 1}; + for (int i = 0; i < 3; i++) if (out[48+i] != adcxq_flags[i]) return 60+i; + if (load64(out + 56) != 0) return 70; + const uint8_t adoxq_flags[4] = {1, 1, 0, 1}; + for (int i = 0; i < 4; i++) if (out[64+i] != adoxq_flags[i]) return 80+i; + return 0; +} +` + rawSource := strings.NewReplacer( + "ADCXL AX, BX", "BYTE $0x66; BYTE $0x0f; BYTE $0x38; BYTE $0xf6; BYTE $0xd8", + "ADOXL AX, BX", "BYTE $0xf3; BYTE $0x0f; BYTE $0x38; BYTE $0xf6; BYTE $0xd8", + "ADCXQ 32(DI), BX", "BYTE $0x66; BYTE $0x48; BYTE $0x0f; BYTE $0x38; BYTE $0xf6; BYTE $0x5f; BYTE $0x20", + "ADOXQ 32(DI), BX", "BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0x38; BYTE $0xf6; BYTE $0x5f; BYTE $0x20", + ).Replace(source) + for _, form := range []struct{ name, source string }{{"text", source}, {"raw", rawSource}} { + t.Run(form.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, form.source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "adxsemantics": { + Name: "adxsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "adx_semantics_"+form.name, triple, ir, mainC, runPrefix) + }) + } +} diff --git a/amd64_lower_aes.go b/amd64_lower_aes.go new file mode 100644 index 000000000..8cd670d89 --- /dev/null +++ b/amd64_lower_aes.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64AESRoundIntrinsics = map[Op]string{ + "AESENC": "@llvm.x86.aesni.aesenc", + "AESENCLAST": "@llvm.x86.aesni.aesenclast", + "AESDEC": "@llvm.x86.aesni.aesdec", + "AESDECLAST": "@llvm.x86.aesni.aesdeclast", +} + +// lowerAES implements the complete Go 1.27 AES/VAES family. Wide VAES round +// instructions apply the corresponding 128-bit AES operation independently +// to every lane. +func (c *amd64Ctx) lowerAES(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86AESProperties(rawOp, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + source, destination := f.sourceOperand, ins.Args[f.destination] + for _, slot := range c.fpResults { + if c.fixedLocal != nil { + break + } // its independent whole-source proof owns native local reads/results + if slot.Type == I8 || slot.Type == I16 || slot.Type == I32 || slot.Type == I64 || slot.Type == LLVMType("float") || slot.Type == LLVMType("double") { + // An intervening X→GP move, unrelated FP store or RET fallback + // cannot bypass the whole original-source must-initialized proof. + if err := c.scalarFPResultInitialized(); err != nil { + return true, false, err + } + break + } + } + if f.memory >= 0 { + // Native stack bytes need their independent exact backing certificate. + // Type 4 retains a 16-byte alignment exception for legacy SSE. A + // logical FP value is not the original physical address and cannot + // preserve that exception. VAES has no corresponding alignment fault. + if !f.spec.vector && source.Kind == OpFP { + return true, false, fmt.Errorf("%w: legacy AES FP source needs its original native storage and alignment", ErrProbeNeedsContext) + } + if source.Kind == OpMem && + !x86OrdinaryNativeMemoryOperand(source, c.goarch) && + !x86CallerMemoryCertified(ins, f.memory) && !c.fixedLocalMemoryCertified(ins, f.memory) { + return true, false, fmt.Errorf("%w: AES native stack source has no original storage certificate", ErrProbeNeedsContext) + } + if source.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, f.byteWidth); err != nil { + return true, false, err + } + readResult := false + for offset := source.FPOffset; offset < source.FPOffset+int64(f.byteWidth); offset++ { + slot, result, _ := c.fpVectorSlot(offset) + if slot.Type != I8 && slot.Type != I16 && slot.Type != I32 && slot.Type != I64 && slot.Type != LLVMType("float") && slot.Type != LLVMType("double") { + return true, false, fmt.Errorf("%w: AES FP byte is not declared numerical storage", ErrProbeNeedsContext) + } + readResult = readResult || result + } + if readResult { + if err := c.scalarFPResultInitialized(); err != nil { + return true, false, err + } + } + } + } + var value string + if f.memory >= 0 && !f.spec.vector && source.Kind != OpFP { + value, err = c.emitAESNativeMemory(ins, f) + } else { + // VAES has no fault-suppression rows. Even a dead key-assist result + // reads all m128 bytes, not only its numerical dwords 1 and 3. + var key string + key, err = c.loadX86PackedShiftBytes(source, f.byteWidth, true) + if err == nil && f.spec.operation < x86AESInverseMix { + var state string + state, err = c.loadPackedCompareBytes(ins.Args[f.state], f.byteWidth) + if err == nil { + legacyOp := Op(strings.TrimPrefix(strings.ToUpper(string(rawOp)), "V")) + value = c.emitAESRound(f.byteWidth, state, key, amd64AESRoundIntrinsics[legacyOp]) + } + } else if err == nil { + carrier, call, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", carrier, key) + if f.spec.operation == x86AESInverseMix { + fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %%%s)\n", call, carrier) + } else { + fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %%%s, i8 %d)\n", call, carrier, uint8(ins.Args[0].Imm)) + } + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, call) + value = "%" + bytes + } + } + if err != nil { + return true, false, err + } + if !f.spec.vector { + return true, false, c.storeLegacyPackedLogical(destination, value) + } + return true, false, c.storePackedMoveOperand(destination, f.byteWidth, value) +} + +// Retain the original legacy opcode and its complete memory envelope even +// when the result is dead, including legacy SSE's Type 4 alignment exception. +func (c *amd64Ctx) emitAESNativeMemory(ins Instr, f x86AESForm) (string, error) { + source := f.sourceOperand + if f.spec.operation < x86AESInverseMix { + data, err := c.loadPackedCompareLanes(ins.Args[f.state], 16, 64, false) + if err != nil { + return "", err + } + result, err := c.emitPackedBinaryNativeMemory(source, 64, data, strings.ToLower(string(ins.Op))) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %s to <16 x i8>\n", value, result) + return "%" + value, nil + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + assembly := strings.ToLower(string(ins.Op)) + " " + segment + "($1), $0" + if f.spec.operation == x86AESKeyAssist { + assembly = fmt.Sprintf("aeskeygenassist $$%d, %s($1), $0", uint8(ins.Args[0].Imm), segment) + } + call, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, \"=x,r,~{memory}\"(%s %s)\n", call, assembly, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", value, call) + return "%" + value, nil +} + +func (c *amd64Ctx) emitAESRound(byteWidth int, stateBytes, keyBytes, intrinsic string) string { + qwords := byteWidth / 8 + state := c.newTmp() + key := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", state, byteWidth, stateBytes, qwords) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", key, byteWidth, keyBytes, qwords) + result := "zeroinitializer" + for lane := 0; lane < byteWidth/16; lane++ { + laneState := c.newTmp() + laneKey := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", laneState, qwords, state, qwords, lane*2, lane*2+1) + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", laneKey, qwords, key, qwords, lane*2, lane*2+1) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <2 x i64> %s(<2 x i64> %%%s, <2 x i64> %%%s)\n", call, intrinsic, laneState, laneKey) + for word := 0; word < 2; word++ { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 %d\n", extracted, call, word) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, qwords, result, extracted, lane*2+word) + result = "%" + inserted + } + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytes, qwords, result, byteWidth) + return "%" + bytes +} diff --git a/amd64_lower_aes_test.go b/amd64_lower_aes_test.go new file mode 100644 index 000000000..a7787018f --- /dev/null +++ b/amd64_lower_aes_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86AESCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zLast := 31 + if target.goarch == "386" { + legacyLast = 7 + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT aesforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"AESENC", "AESENCLAST", "AESDEC", "AESDECLAST"} { + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + vop := "V" + op + source.WriteString("\t" + vop + " X0, X1, X2\n") + source.WriteString("\t" + vop + " 16(AX), X1, X2\n") + source.WriteString("\t" + vop + " Y0, Y1, Y2\n") + source.WriteString("\t" + vop + " 32(AX), Y1, Y2\n") + if target.goarch == "amd64" { + source.WriteString("\t" + vop + " X16, X30, X31\n") + } + fmt.Fprintf(&source, "\t%s Z0, Z1, Z%d\n", vop, zLast) + } + fmt.Fprintf(&source, "\tAESIMC X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tAESIMC 8(AX), X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESIMC X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESIMC 8(AX), X%d\n", legacyLast) + fmt.Fprintf(&source, "\tAESKEYGENASSIST $0, X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tAESKEYGENASSIST $255, 8(AX), X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESKEYGENASSIST $-128, X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tVAESKEYGENASSIST $255, 8(AX), X%d\n", legacyLast) + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"aesforms": {Name: "aesforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "aes-"+target.name+".ll", "aes-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86AESRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "AESENC Y0, X1", + "AESENC X0, X16", + "AESENC.Z X0, X1", + "AESIMC X0, (AX)", + "AESKEYGENASSIST $-1, X0, X1", + "AESKEYGENASSIST $256, X0, X1", + "VAESENC X0, Y1, Y2", + "VAESENC X0, X1", + "VAESENC X0, X1, K1, X2", + "VAESENC.Z X0, X1, X2", + "VAESENC X0, X1, (AX)", + "VAESIMC Y0, Y1", + "VAESIMC X16, X1", + "VAESKEYGENASSIST $-129, X0, X1", + "VAESKEYGENASSIST $256, X0, X1", + "VAESKEYGENASSIST $1, Y0, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86AESRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "AESENC X8, X0", + "AESIMC X0, X8", + "AESKEYGENASSIST $1, X8, X0", + "VAESIMC X8, X0", + "VAESKEYGENASSIST $1, X0, X8", + "VAESENC Z8, Z0, Z1", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86AESRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86AESRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's AES/VAES tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_align_elements.go b/amd64_lower_align_elements.go new file mode 100644 index 000000000..c59a6caf8 --- /dev/null +++ b/amd64_lower_align_elements.go @@ -0,0 +1,133 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// amd64VectorAlignSpecs is the complete Go 1.27 VALIGN family. Both opcodes +// use _yvalignd; they differ only in element and broadcast width. +var amd64VectorAlignSpecs = map[Op]int{ + "VALIGND": 32, + "VALIGNQ": 64, +} + +func (c *amd64Ctx) lowerVectorAlign(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + laneBits, recognized := amd64VectorAlignSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + // The 386 assembler frontend rejects the required four- and five-operand + // spellings before reaching the shared _yvalignd table. + if c.goarch != "amd64" { + return true, false, fmt.Errorf("%s %s is rejected by Go 1.27's x86 assembler operand limit: %q", c.goarch, baseOp, ins.Raw) + } + + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("amd64 %s has a suffix absent from Go 1.27's EVEX encoding: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 4 && len(ins.Args) != 5 { + return true, false, fmt.Errorf("amd64 %s expects unsigned-imm8, src1, src2, [K mask,] destination: %q", baseOp, ins.Raw) + } + masked := len(ins.Args) == 5 + if zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s zeroing requires a K1-K7 mask: %q", baseOp, ins.Raw) + } + if !amd64UnsignedImmediate(ins.Args[0], 8) { + return true, false, fmt.Errorf("amd64 %s first operand must be Go 1.27's unsigned-imm8 class: %q", baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s destination must be an X, Y, or Z register: %q", baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !amd64EVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 %s destination is outside Go 1.27's EVEX vector class: %q", baseOp, ins.Raw) + } + if !amd64EVEXVectorRegister(ins.Args[2], byteWidth) { + return true, false, fmt.Errorf("amd64 %s second source must be a same-width EVEX vector register: %q", baseOp, ins.Raw) + } + if broadcast { + if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory first source: %q", baseOp, ins.Raw) + } + } else if ins.Args[1].Kind == OpReg { + if !amd64EVEXVectorRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("amd64 %s first source must match the destination width: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s first source must be a vector register or memory: %q", baseOp, ins.Raw) + } + + mask := "" + if masked { + if !amd64NonzeroKOperand(ins.Args[3]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err = c.loadK(ins.Args[3].Reg) + if err != nil { + return true, false, err + } + } + + lanes := byteWidth * 8 / laneBits + var first string + if broadcast { + scalar, loadErr := c.evalIntSized(ins.Args[1], amd64IntegerTypeForBits(laneBits)) + if loadErr != nil { + return true, false, loadErr + } + first = amd64SplatInteger(c, lanes, laneBits, scalar) + } else { + firstBytes, loadErr := c.loadPackedCompareBytes(ins.Args[1], byteWidth) + if loadErr != nil { + return true, false, loadErr + } + first = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, firstBytes) + } + secondBytes, err := c.loadPackedCompareBytes(ins.Args[2], byteWidth) + if err != nil { + return true, false, err + } + second := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, secondBytes) + result := c.emitVectorAlign(lanes, laneBits, first, second, uint8(ins.Args[0].Imm)) + if masked { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitVectorAlign(lanes, laneBits int, first, second string, immediate uint8) string { + count := int(immediate) & (lanes - 1) + indices := make([]string, lanes) + for lane := range indices { + indices[lane] = fmt.Sprintf("i32 %d", lane+count) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> %s, <%d x i32> <%s>\n", + result, lanes, laneBits, first, lanes, laneBits, second, lanes, strings.Join(indices, ", ")) + return "%" + result +} diff --git a/amd64_lower_align_elements_test.go b/amd64_lower_align_elements_test.go new file mode 100644 index 000000000..4712da34c --- /dev/null +++ b/amd64_lower_align_elements_test.go @@ -0,0 +1,216 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateAMD64VectorAlignCompleteGo127Forms(t *testing.T) { + // VALIGND/Q share Go 1.27's six-row _yvalignd table: X/Y/Z, + // each with an unmasked and K1-K7 masked row. Their EVEX attributes add + // D/Q scalar broadcast and zeroing. + var source strings.Builder + source.WriteString("TEXT vector_align_forms(SB),$0-0\n") + for _, op := range []string{"VALIGND", "VALIGNQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s $0, %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s $255, 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&source, "\t%s $1, %s5, %s6, K1, %s7\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z $2, 16(BX), %s8, K2, %s9\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST $3, 24(BX), %s10, %s11\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(BX), %s12, K3, %s13\n", op, width, width) + } + fmt.Fprintf(&source, "\t%s $5, X29, X30, X31\n", op) + fmt.Fprintf(&source, "\t%s $6, Y29, Y30, Y31\n", op) + fmt.Fprintf(&source, "\t%s $7, Z29, Z30, Z31\n", op) + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "vector_align_forms": {Name: "vector_align_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "vector-align-"+target.name+".ll", "vector-align-"+target.name+".o", ir) + }) + } +} + +func TestTranslateAMD64VectorAlignRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VALIGND X0, X1, X2", + "VALIGND $0, X0, X1", + "VALIGNQ $0, X0, X1, X2, X3, X4", + "VALIGND $-1, X0, X1, X2", + "VALIGNQ $256, X0, X1, X2", + "VALIGND AX, X0, X1, X2", + "VALIGND $0, Y0, X1, X2", + "VALIGNQ $0, X0, Y1, X2", + "VALIGND $0, X0, X1, Y2", + "VALIGNQ $0, X0, 8(BX), X2", + "VALIGND $0, X0, X1, 8(BX)", + "VALIGND $0, X0, X1, K0, X2", + "VALIGNQ $0, X0, X1, K8, X2", + "VALIGND.Z $0, X0, X1, X2", + "VALIGNQ.BCST $0, X0, X1, X2", + "VALIGND.BCST.Z $0, 8(BX), X1, X2", + "VALIGNQ.Z.BCST $0, 8(BX), X1, K1, X2", + "VALIGND.SAE $0, X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VALIGND/Q table", instruction) + } + }) + } +} + +func TestTranslate386VectorAlignMatchesGoAssemblerOperandLimit(t *testing.T) { + for _, instruction := range []string{ + "VALIGND $0, X0, X1, X2", + "VALIGNQ $255, 8(BX), Y1, Y2", + "VALIGND.Z $1, Z0, Z1, K1, Z2", + "VALIGNQ.BCST.Z $2, 16(BX), X1, K7, X2", + } { + t.Run(strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted 386 %q although Go 1.27 rejects its operand count", instruction) + } + } + }) + } +} + +func TestAMD64VectorAlignRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vectorAlignSemantics(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ scalar+24(FP), DX + VMOVDQU32 (BX), X0 + VMOVDQU32 (CX), X1 + VALIGND $1, X0, X1, X2 + VMOVDQU32 X2, 0(AX) + VALIGND $255, X0, X1, X3 + VMOVDQU32 X3, 16(AX) + VALIGNQ $1, X0, X1, X4 + VMOVDQU32 X4, 32(AX) + KMOVQ mask+32(FP), K1 + VMOVDQU32 X0, X5 + VALIGND $1, X0, X1, K1, X5 + VMOVDQU32 X5, 48(AX) + VALIGND.Z $1, X0, X1, K1, X6 + VMOVDQU32 X6, 64(AX) + VALIGND.BCST $1, (DX), X1, X7 + VMOVDQU32 X7, 80(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "vectorAlignSemantics": { + Name: "vectorAlignSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void vectorAlignSemantics(uint32_t *, const uint32_t *, const uint32_t *, const uint32_t *, uint64_t); +int main(void) { + uint32_t first[4] = {10, 11, 12, 13}; + uint32_t second[4] = {20, 21, 22, 23}; + uint32_t scalar = 99; + uint32_t out[24] = {0}; + const uint32_t want[24] = { + 11, 12, 13, 20, + 13, 20, 21, 22, + 12, 13, 20, 21, + 11, 11, 13, 13, + 11, 0, 13, 0, + 99, 99, 99, 20, + }; + vectorAlignSemantics(out, first, second, &scalar, 5); + for (int i = 0; i < 24; i++) if (out[i] != want[i]) return i + 1; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vector_align", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_align_right.go b/amd64_lower_align_right.go new file mode 100644 index 000000000..dba8bfd78 --- /dev/null +++ b/amd64_lower_align_right.go @@ -0,0 +1,141 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedAlignRight implements the complete Go 1.27 PALIGNR and +// VPALIGNR tables. Each 128-bit lane is formed from the first Plan 9 source +// as its low half and the second source (or the legacy destination) as its +// high half, then shifted right by the immediate byte count. +func (c *amd64Ctx) lowerPackedAlignRight(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + raw := strings.ToUpper(string(rawOp)) + parts := strings.Split(raw, ".") + baseOp := Op(parts[0]) + if baseOp != "PALIGNR" && baseOp != "VPALIGNR" { + return false, false, nil + } + if len(parts) > 2 || len(parts) == 2 && parts[1] != "Z" { + return true, false, fmt.Errorf("amd64 %s has a suffix outside Go 1.27's align-right tables: %q", baseOp, ins.Raw) + } + zeroing := len(parts) == 2 + + if baseOp == "PALIGNR" { + if zeroing { + return true, false, fmt.Errorf("amd64 PALIGNR does not accept instruction suffixes: %q", ins.Raw) + } + if len(ins.Args) != 3 || !amd64UnsignedImmediate(ins.Args[0], 8) { + return true, false, fmt.Errorf("amd64 PALIGNR expects $imm8, X/m128, X: %q", ins.Raw) + } + source, destination := ins.Args[1], ins.Args[2] + if !c.isGoVEXVectorRegister(destination, 16, false) { + return true, false, fmt.Errorf("amd64 PALIGNR destination is outside Go 1.27's X register class: %q", ins.Raw) + } + if !c.isGoVEXVectorRegister(source, 16, true) { + return true, false, fmt.Errorf("amd64 PALIGNR source is outside Go 1.27's X/m128 class: %q", ins.Raw) + } + low, err := c.loadPackedCompareBytes(source, 16) + if err != nil { + return true, false, err + } + high, err := c.loadX(destination.Reg) + if err != nil { + return true, false, err + } + result := c.emitPackedAlignRightBytes(16, low, high, ins.Args[0].Imm) + return true, false, c.storeX(destination.Reg, result) + } + + // cmd/asm's 386 parser cannot represent VPALIGNR's four- and five- + // operand spellings, even though the shared encoder table is present. + if c.goarch == "386" { + return true, false, fmt.Errorf("386 VPALIGNR is outside Go 1.27's accepted instruction forms: %q", ins.Raw) + } + masked := len(ins.Args) == 5 + if len(ins.Args) != 4 && !masked || !amd64UnsignedImmediate(ins.Args[0], 8) { + return true, false, fmt.Errorf("amd64 VPALIGNR expects $imm8, X|Y|Z/mem, X|Y|Z, [K,] X|Y|Z: %q", ins.Raw) + } + if zeroing && !masked { + return true, false, fmt.Errorf("amd64 VPALIGNR.Z requires a writemask: %q", ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + byteWidth := amd64VectorByteWidth(destination.Reg) + if destination.Kind != OpReg || byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 VPALIGNR destination is outside Go 1.27's X/Y/Z register classes: %q", ins.Raw) + } + first, second := ins.Args[1], ins.Args[2] + if first.Kind == OpReg { + if !c.isGoEVEXVectorRegister(first, byteWidth) { + return true, false, fmt.Errorf("amd64 VPALIGNR first source must match its destination width: %q", ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 VPALIGNR first source must be a vector register or memory: %q", ins.Raw) + } + if !c.isGoEVEXVectorRegister(second, byteWidth) { + return true, false, fmt.Errorf("amd64 VPALIGNR second source must match its destination width: %q", ins.Raw) + } + + maskValue := "" + if masked { + mask := ins.Args[3] + maskIndex, validMask := amd64ParseKReg(mask.Reg) + if mask.Kind != OpReg || !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 VPALIGNR masked form requires K1..K7: %q", ins.Raw) + } + maskValue, err = c.loadK(mask.Reg) + if err != nil { + return true, false, err + } + } + low, err := c.loadPackedCompareBytes(first, byteWidth) + if err != nil { + return true, false, err + } + high, err := c.loadPackedCompareBytes(second, byteWidth) + if err != nil { + return true, false, err + } + result := c.emitPackedAlignRightBytes(byteWidth, low, high, ins.Args[0].Imm) + if masked { + old, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, byteWidth, 8, result, old, maskValue, zeroing) + } + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, result) +} + +func amd64UnsignedImmediate(operand Operand, bits int) bool { + if operand.Kind != OpImm || bits <= 0 || bits >= 63 { + return false + } + return operand.Imm >= 0 && operand.Imm < int64(uint64(1)<= 32 { + return "zeroinitializer" + } + result := "zeroinitializer" + for laneBase := 0; laneBase < byteWidth; laneBase += 16 { + for destinationByte := 0; destinationByte < 16; destinationByte++ { + sourceByte := int64(destinationByte) + count + if sourceByte >= 32 { + continue + } + source := low + if sourceByte >= 16 { + source = high + sourceByte -= 16 + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", extracted, byteWidth, source, laneBase+int(sourceByte)) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, result, extracted, laneBase+destinationByte) + result = "%" + inserted + } + } + return result +} diff --git a/amd64_lower_align_right_test.go b/amd64_lower_align_right_test.go new file mode 100644 index 000000000..128c3be9e --- /dev/null +++ b/amd64_lower_align_right_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedAlignRightCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + fmt.Fprintf(&source, "TEXT packedalignrightforms(SB),$0-0\n\tPALIGNR $0, X0, X%d\n\tPALIGNR $255, 8(AX), X%d\n", last, last) + if target.goarch == "amd64" { + source.WriteString("\tVPALIGNR $0, X0, X1, X2\n") + source.WriteString("\tVPALIGNR $1, 8(AX), X1, X2\n") + source.WriteString("\tVPALIGNR $2, Y0, Y1, Y2\n") + source.WriteString("\tVPALIGNR $3, 8(AX), Y1, Y2\n") + source.WriteString("\tVPALIGNR $4, Z0, Z1, Z2\n") + source.WriteString("\tVPALIGNR $5, 8(AX), Z1, Z2\n") + source.WriteString("\tVPALIGNR $6, X16, X31, X0\n") + source.WriteString("\tVPALIGNR $7, X16, X31, K1, X0\n") + source.WriteString("\tVPALIGNR.Z $8, 16(AX), Y31, K2, Y0\n") + source.WriteString("\tVPALIGNR $9, Z16, Z31, K3, Z0\n") + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedalignrightforms": {Name: "packedalignrightforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-align-right-"+target.name+".ll", "packed-align-right-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedAlignRightRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PALIGNR $-1, X0, X1", + "PALIGNR $256, X0, X1", + "PALIGNR $1, M0, M1", + "PALIGNR $1, X16, X1", + "PALIGNR $1, Y0, Y1", + "PALIGNR.Z $1, X0, X1", + "PALIGNR $1, X0, (AX)", + "VPALIGNR $-1, X0, X1, X2", + "VPALIGNR $256, X0, X1, X2", + "VPALIGNR $1, X0, Y1, Y2", + "VPALIGNR $1, X0, X1, K0, X2", + "VPALIGNR.Z $1, X0, X1, X2", + "VPALIGNR.BCST $1, X0, X1, X2", + "VPALIGNR $1, X0, (AX), X2", + "VPALIGNR $1, X0, X1, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedAlignRightRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PALIGNR $1, X8, X0", + "PALIGNR $1, X0, X8", + "VPALIGNR $1, X0, X1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedAlignRightRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedAlignRightRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PALIGNR/VPALIGNR tables for %s", instruction, goarch) + } +} + +func TestAMD64PackedAlignRightRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT alignrightlegacy(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ low+8(FP), BX + MOVQ high+16(FP), CX + MOVOU (CX), X0 + PALIGNR $8, (BX), X0 + MOVOU X0, (AX) + RET +TEXT alignrightvector(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ low+8(FP), BX + MOVQ high+16(FP), CX + VMOVDQU (BX), Y0 + VMOVDQU (CX), Y1 + VPALIGNR $8, Y0, Y1, Y2 + VMOVDQU Y2, (AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "alignrightlegacy": {Name: "alignrightlegacy", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + "alignrightvector": {Name: "alignrightvector", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void alignrightlegacy(uint8_t *, const uint8_t *, const uint8_t *); +extern void alignrightvector(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t low[32], high[32], out[32]; + for (int i = 0; i < 32; i++) { low[i] = (uint8_t)i; high[i] = (uint8_t)(100+i); out[i] = 0; } + alignrightlegacy(out, low, high); + for (int i = 0; i < 8; i++) if (out[i] != low[i+8]) return 10+i; + for (int i = 8; i < 16; i++) if (out[i] != high[i-8]) return 30+i; + for (int i = 0; i < 32; i++) out[i] = 0; + alignrightvector(out, low, high); + for (int lane = 0; lane < 2; lane++) { + int base = lane*16; + for (int i = 0; i < 8; i++) if (out[base+i] != low[base+i+8]) return 60+base+i; + for (int i = 8; i < 16; i++) if (out[base+i] != high[base+i-8]) return 100+base+i; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_align_right", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_andn_test.go b/amd64_lower_andn_test.go new file mode 100644 index 000000000..c7f7c1a89 --- /dev/null +++ b/amd64_lower_andn_test.go @@ -0,0 +1,265 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86RawANDNReportedEncoding(t *testing.T) { + const source = ` +TEXT rawANDN(SB),$0-0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0x38 + BYTE $0xf2 + BYTE $0xe2 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawANDN": {Name: "rawANDN", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("raw ANDN did not infer BMI1 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-andn.ll", "raw-andn.o", ir) + }) + } +} + +func TestX86ANDNCompleteGo127FormsAcrossTargets(t *testing.T) { + registers386 := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + registersAMD64 := append(append([]string{}, registers386...), "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + targets := []struct { + name string + goarch string + triple string + registers []string + }{ + {name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", registers: registers386}, + {name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", registers: registers386}, + {name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", registers: registersAMD64}, + {name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registers: registersAMD64}, + {name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registers: registersAMD64}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT andnall(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + for index, first := range target.registers { + second := target.registers[(index+3)%len(target.registers)] + destination := target.registers[(index+5)%len(target.registers)] + fmt.Fprintf(&source, "\tANDN%s %s, %s, %s\n", width, first, second, destination) + } + fmt.Fprintf(&source, "\tANDN%s 8(AX), CX, DX\n", width) + fmt.Fprintf(&source, "\tANDN%s 16(AX)(CX*4), DX, BX\n", width) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"andnall": {Name: "andnall", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("ANDN family did not infer BMI1 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "andn-family.ll", "andn-family.o", ir) + }) + } +} + +func TestX86RawANDNCompleteRegisterFields(t *testing.T) { + for _, mode := range []int{32, 64} { + registerCount := 8 + if mode == 64 { + registerCount = 16 + } + for _, width64 := range []bool{false, true} { + for first := 0; first < registerCount; first++ { + for second := 0; second < registerCount; second++ { + for destination := 0; destination < registerCount; destination++ { + p0 := byte((^destination>>3)&1)<<7 | 1<<6 | byte((^first>>3)&1)<<5 | 2 + p1 := byte((^second)&15) << 3 + if width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0xc4, p0, p1, 0xf2, modRM} + instruction, length, ok, err := decodedX86ANDNInstruction(encoding, mode) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("mode=%d width64=%v first=%d second=%d destination=%d: ok=%v length=%d err=%v", mode, width64, first, second, destination, ok, length, err) + } + wantOp := Op("ANDNL") + if width64 && mode == 64 { + wantOp = "ANDNQ" + } + wantFirst, _ := decodedX86GeneralRegister(first) + wantSecond, _ := decodedX86GeneralRegister(second) + wantDestination, _ := decodedX86GeneralRegister(destination) + if instruction.Op != wantOp || len(instruction.Args) != 3 || instruction.Args[0].Reg != wantFirst || instruction.Args[1].Reg != wantSecond || instruction.Args[2].Reg != wantDestination { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + } + } + } + } + } +} + +func TestX86RawANDNMemoryForms(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + }{ + {name: "direct", encoding: []byte{0xc4, 0xe2, 0x30, 0xf2, 0x13}}, + {name: "extended base", encoding: []byte{0xc4, 0xc2, 0x30, 0xf2, 0x13}}, + {name: "sib disp8", encoding: []byte{0xc4, 0x82, 0x88, 0xf2, 0x54, 0x88, 0x10}}, + {name: "base less", encoding: []byte{0xc4, 0xe2, 0x30, 0xf2, 0x14, 0x25, 0x78, 0x56, 0x34, 0x12}}, + {name: "segment", encoding: []byte{0x64, 0xc4, 0xe2, 0x30, 0xf2, 0x54, 0x48, 0x08}}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86ANDNInstruction(test.encoding, 64) + if !ok || err != nil || length != len(test.encoding) || len(instruction.Args) != 3 || instruction.Args[0].Kind != OpMem { + t.Fatalf("encoding %x: instruction=%#v ok=%v length=%d err=%v", test.encoding, instruction, ok, length, err) + } + }) + } +} + +func TestX86RawANDNRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex l", mode: 64, encoding: []byte{0xc4, 0xe2, 0x34, 0xf2, 0xd2}}, + {name: "vex pp", mode: 64, encoding: []byte{0xc4, 0xe2, 0x31, 0xf2, 0xd2}}, + {name: "missing modrm", mode: 64, encoding: []byte{0xc4, 0xe2, 0x30, 0xf2}}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe2, 0x30, 0xf2, 0xd2}}, + {name: "rip relative", mode: 64, encoding: []byte{0xc4, 0xe2, 0x30, 0xf2, 0x15, 0, 0, 0, 0}}, + {name: "extended first 386", mode: 32, encoding: []byte{0xc4, 0xc2, 0x70, 0xf2, 0xc0}}, + {name: "extended second 386", mode: 32, encoding: []byte{0xc4, 0xe2, 0x38, 0xf2, 0xc0}}, + {name: "extended destination 386", mode: 32, encoding: []byte{0xc4, 0x62, 0x70, 0xf2, 0xc0}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86ANDNInstruction(test.encoding, test.mode) + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} + +func TestX86ANDNRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT andnsemantics(SB),$0-8 + MOVQ out+0(FP), DI + MOVQ $0xf0, AX + MOVQ $0xcc, BX + ANDNQ AX, BX, CX + MOVQ CX, 0(DI) + SETCS 24(DI) + SETEQ 25(DI) + SETMI 26(DI) + SETOS 27(DI) + MOVQ $0, AX + MOVQ $0, BX + ANDNQ AX, BX, CX + MOVQ CX, 8(DI) + SETCS 28(DI) + SETEQ 29(DI) + SETMI 30(DI) + SETOS 31(DI) + MOVL $0x80000000, AX + MOVL $0, BX + ANDNL AX, BX, CX + MOVQ CX, 16(DI) + SETCS 32(DI) + SETEQ 33(DI) + SETMI 34(DI) + SETOS 35(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "andnsemantics": {Name: "andnsemantics", Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void andnsemantics(uint8_t *); +int main(void) { + uint8_t out[40] = {0}; + andnsemantics(out); + const uint64_t want[3] = {UINT64_C(0x30), 0, UINT64_C(0x80000000)}; + for (int i = 0; i < 3; i++) { + uint64_t got; + memcpy(&got, out + i * 8, sizeof(got)); + if (got != want[i]) return 10 + i; + } + const uint8_t flags[12] = {0,0,0,0, 0,1,0,0, 0,0,1,0}; + for (int i = 0; i < 12; i++) if (out[24+i] != flags[i]) return 30+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "andn_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_approximate_math.go b/amd64_lower_approximate_math.go new file mode 100644 index 000000000..c43391e29 --- /dev/null +++ b/amd64_lower_approximate_math.go @@ -0,0 +1,285 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ApproximateMathMode uint8 + +const ( + amd64ApproximateExp2 amd64ApproximateMathMode = iota + amd64ApproximateReciprocal28 + amd64ApproximateReciprocalSqrt28 +) + +type amd64ApproximateMathSpec struct { + laneBits int + scalar bool + mode amd64ApproximateMathMode +} + +// amd64ApproximateMathSpecs models the complete AVX512ER family exposed by +// Go 1.27's _yvexp2pd and _yvgetexpsd tables. The type axes describe the +// mathematical operation, lane type, and packed/scalar operand shape; the +// parser below derives every legal mask, zero, broadcast, and SAE form. +var amd64ApproximateMathSpecs = map[Op]amd64ApproximateMathSpec{ + "VEXP2PS": {laneBits: 32, mode: amd64ApproximateExp2}, + "VEXP2PD": {laneBits: 64, mode: amd64ApproximateExp2}, + "VRCP28PS": {laneBits: 32, mode: amd64ApproximateReciprocal28}, + "VRCP28PD": {laneBits: 64, mode: amd64ApproximateReciprocal28}, + "VRCP28SS": {laneBits: 32, scalar: true, mode: amd64ApproximateReciprocal28}, + "VRCP28SD": {laneBits: 64, scalar: true, mode: amd64ApproximateReciprocal28}, + "VRSQRT28PS": {laneBits: 32, mode: amd64ApproximateReciprocalSqrt28}, + "VRSQRT28PD": {laneBits: 64, mode: amd64ApproximateReciprocalSqrt28}, + "VRSQRT28SS": {laneBits: 32, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, + "VRSQRT28SD": {laneBits: 64, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, +} + +type amd64ApproximateMathForm struct { + properties amd64BinaryFloatingSuffix + source Operand + passthrough Operand + destination Operand + mask string +} + +func (c *amd64Ctx) lowerApproximateMath(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64ApproximateMathSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + form, err := c.parseApproximateMathForm(baseOp, suffix, spec, ins) + if err != nil { + return true, false, err + } + if spec.scalar { + return c.lowerScalarApproximateMath(spec, form) + } + return c.lowerPackedApproximateMath(spec, form) +} + +func (c *amd64Ctx) parseApproximateMathForm(baseOp, suffix string, spec amd64ApproximateMathSpec, ins Instr) (amd64ApproximateMathForm, error) { + var form amd64ApproximateMathForm + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.rounding != "" || spec.scalar && properties.broadcast { + return form, fmt.Errorf("%s %s has a suffix absent from Go 1.27's AVX512ER tables: %q", c.goarch, baseOp, ins.Raw) + } + wantArgs := 2 + if spec.scalar { + wantArgs = 3 + } + if len(ins.Args) != wantArgs && len(ins.Args) != wantArgs+1 { + return form, fmt.Errorf("%s %s has the wrong operand count for its packed/scalar grammar: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == wantArgs+1 + if c.goarch == "386" && spec.scalar && masked { + return form, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return form, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + maskIndex := len(ins.Args) - 2 + if masked && !amd64NonzeroKOperand(ins.Args[maskIndex]) { + return form, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return form, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + wantWidth := 64 + if spec.scalar { + wantWidth = 16 + } + if amd64VectorByteWidth(destination.Reg) != wantWidth || !c.isGoEVEXVectorRegister(destination, wantWidth) { + return form, fmt.Errorf("%s %s destination must be an in-range %s register: %q", c.goarch, baseOp, map[bool]string{true: "X", false: "Z"}[spec.scalar], ins.Raw) + } + + source := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return form, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, wantWidth) { + return form, fmt.Errorf("%s %s source register must match its destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return form, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae && source.Kind != OpReg { + return form, fmt.Errorf("%s %s.SAE requires a register source: %q", c.goarch, baseOp, ins.Raw) + } + + passthrough := Operand{} + if spec.scalar { + passthrough = ins.Args[1] + if !c.isGoEVEXVectorRegister(passthrough, 16) { + return form, fmt.Errorf("%s %s scalar passthrough must be an in-range X register: %q", c.goarch, baseOp, ins.Raw) + } + } + mask := "" + if masked { + var err error + mask, err = c.loadK(ins.Args[maskIndex].Reg) + if err != nil { + return form, err + } + } + return amd64ApproximateMathForm{ + properties: properties, source: source, passthrough: passthrough, + destination: destination, mask: mask, + }, nil +} + +func (c *amd64Ctx) lowerPackedApproximateMath(spec amd64ApproximateMathSpec, form amd64ApproximateMathForm) (bool, bool, error) { + lanes := 512 / spec.laneBits + typ := amd64FloatingVectorType(lanes, spec.laneBits) + var source string + if form.properties.broadcast { + scalar, err := c.loadFloatingScalarOperand(form.source, spec.laneBits) + if err != nil { + return true, false, err + } + source = c.splatFloatingScalar(scalar, lanes, spec.laneBits) + } else { + var err error + source, err = c.loadFloatingVectorOperand(form.source, 64, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + } + + result := c.newTmp() + assembly := approximateMathAssembly(spec, form.properties.sae, form.mask != "", form.properties.zeroing) + clobbers := "~{dirflag},~{fpsr},~{flags}" + if form.mask == "" { + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s)\n", result, typ, assembly, "=v,v,"+clobbers, typ, source) + } else { + merge := "zeroinitializer" + if !form.properties.zeroing { + var err error + merge, err = c.loadFloatingVectorOperand(form.destination, 64, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + } + mask := c.approximateMathMask32(form.mask) + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s, %s %s, i32 %s)\n", + result, typ, assembly, "=v,0,v,r,~{k1},"+clobbers, typ, merge, typ, source, mask) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <64 x i8>\n", bytes, typ, result) + return true, false, c.storeVectorBytes(form.destination.Reg, 64, "%"+bytes) +} + +func (c *amd64Ctx) lowerScalarApproximateMath(spec amd64ApproximateMathSpec, form amd64ApproximateMathForm) (bool, bool, error) { + lanes := 128 / spec.laneBits + typ := amd64FloatingVectorType(lanes, spec.laneBits) + var source string + if form.source.Kind == OpReg { + var err error + source, err = c.loadFloatingVectorOperand(form.source, 16, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + } else { + scalar, err := c.loadFloatingScalarOperand(form.source, spec.laneBits) + if err != nil { + return true, false, err + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s zeroinitializer, %s %s, i32 0\n", inserted, typ, amd64FloatingScalarType(spec.laneBits), scalar) + source = "%" + inserted + } + passthrough, err := c.loadFloatingVectorOperand(form.passthrough, 16, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + + result := c.newTmp() + assembly := approximateMathAssembly(spec, form.properties.sae, form.mask != "", form.properties.zeroing) + clobbers := "~{dirflag},~{fpsr},~{flags}" + if form.mask == "" { + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s, %s %s)\n", + result, typ, assembly, "=v,v,v,"+clobbers, typ, source, typ, passthrough) + } else { + merge := "zeroinitializer" + if !form.properties.zeroing { + merge, err = c.loadFloatingVectorOperand(form.destination, 16, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + } + mask := c.approximateMathMask32(form.mask) + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s, %s %s, %s %s, i32 %s)\n", + result, typ, assembly, "=v,0,v,v,r,~{k1},"+clobbers, + typ, merge, typ, source, typ, passthrough, mask) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytes, typ, result) + return true, false, c.storeVectorBytes(form.destination.Reg, 16, "%"+bytes) +} + +func approximateMathAssembly(spec amd64ApproximateMathSpec, sae, masked, zeroing bool) string { + assembly := "" + if masked { + maskOperand := 3 + if spec.scalar { + maskOperand = 4 + } + assembly = fmt.Sprintf("kmovw ${%d:k}, %%k1; ", maskOperand) + } + assembly += approximateMathMnemonic(spec) + if sae { + assembly += " {sae}," + } + if spec.scalar { + if masked { + assembly += " $2, $3, $0" + } else { + assembly += " $1, $2, $0" + } + } else if masked { + assembly += " $2, $0" + } else { + assembly += " $1, $0" + } + if masked { + assembly += " {%k1}" + if zeroing { + assembly += " {z}" + } + } + return assembly +} + +func approximateMathMnemonic(spec amd64ApproximateMathSpec) string { + stem := map[amd64ApproximateMathMode]string{ + amd64ApproximateExp2: "vexp2", + amd64ApproximateReciprocal28: "vrcp28", + amd64ApproximateReciprocalSqrt28: "vrsqrt28", + }[spec.mode] + if spec.scalar { + if spec.laneBits == 32 { + return stem + "ss" + } + return stem + "sd" + } + if spec.laneBits == 32 { + return stem + "ps" + } + return stem + "pd" +} + +func (c *amd64Ctx) approximateMathMask32(mask string) string { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", truncated, mask) + return "%" + truncated +} diff --git a/amd64_lower_approximate_math_test.go b/amd64_lower_approximate_math_test.go new file mode 100644 index 000000000..472dff957 --- /dev/null +++ b/amd64_lower_approximate_math_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type approximateMathTestSpec struct { + op string + laneBits int + scalar bool + mode amd64ApproximateMathMode +} + +func go127ApproximateMathSpecs() []approximateMathTestSpec { + return []approximateMathTestSpec{ + {op: "VEXP2PS", laneBits: 32, mode: amd64ApproximateExp2}, + {op: "VEXP2PD", laneBits: 64, mode: amd64ApproximateExp2}, + {op: "VRCP28PS", laneBits: 32, mode: amd64ApproximateReciprocal28}, + {op: "VRCP28PD", laneBits: 64, mode: amd64ApproximateReciprocal28}, + {op: "VRCP28SS", laneBits: 32, scalar: true, mode: amd64ApproximateReciprocal28}, + {op: "VRCP28SD", laneBits: 64, scalar: true, mode: amd64ApproximateReciprocal28}, + {op: "VRSQRT28PS", laneBits: 32, mode: amd64ApproximateReciprocalSqrt28}, + {op: "VRSQRT28PD", laneBits: 64, mode: amd64ApproximateReciprocalSqrt28}, + {op: "VRSQRT28SS", laneBits: 32, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, + {op: "VRSQRT28SD", laneBits: 64, scalar: true, mode: amd64ApproximateReciprocalSqrt28}, + } +} + +func TestAMD64ApproximateMathGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127ApproximateMathSpecs() + if len(amd64ApproximateMathSpecs) != len(expected) { + t.Fatalf("approximate-math grammar has %d entries, want %d", len(amd64ApproximateMathSpecs), len(expected)) + } + for _, want := range expected { + got, ok := amd64ApproximateMathSpecs[Op(want.op)] + if !ok { + t.Errorf("approximate-math grammar omitted %s", want.op) + continue + } + if got.laneBits != want.laneBits || got.scalar != want.scalar || got.mode != want.mode { + t.Errorf("approximate-math grammar %s = %+v, want laneBits=%d scalar=%v mode=%d", want.op, got, want.laneBits, want.scalar, want.mode) + } + } +} + +func TestTranslateX86ApproximateMathCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + vectorDestination := 23 + if target.goarch == "386" { + vectorDestination = 7 + } + var source strings.Builder + source.WriteString("TEXT approximatemathforms(SB),$0-0\n") + for index, spec := range go127ApproximateMathSpecs() { + if spec.scalar { + fmt.Fprintf(&source, "\t%s X1, X2, X23\n", spec.op) + fmt.Fprintf(&source, "\t%s %d(AX), X2, X23\n", spec.op, index*8) + fmt.Fprintf(&source, "\t%s.SAE X1, X2, X23\n", spec.op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X2, K1, X23\n", spec.op) + fmt.Fprintf(&source, "\t%s.Z %d(AX), X2, K2, X23\n", spec.op, index*8) + fmt.Fprintf(&source, "\t%s.SAE.Z X1, X2, K3, X23\n", spec.op) + } + continue + } + fmt.Fprintf(&source, "\t%s Z1, Z%d\n", spec.op, vectorDestination) + fmt.Fprintf(&source, "\t%s %d(AX), Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s Z1, K1, Z%d\n", spec.op, vectorDestination) + fmt.Fprintf(&source, "\t%s.Z %d(AX), K2, Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s.BCST %d(AX), Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s.BCST.Z %d(AX), K3, Z%d\n", spec.op, index*8, vectorDestination) + fmt.Fprintf(&source, "\t%s.SAE Z1, Z%d\n", spec.op, vectorDestination) + fmt.Fprintf(&source, "\t%s.SAE.Z Z1, K4, Z%d\n", spec.op, vectorDestination) + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"approximatemathforms": {Name: "approximatemathforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, mnemonic := range []string{"vexp2ps", "vexp2pd", "vrcp28ps", "vrcp28pd", "vrcp28ss", "vrcp28sd", "vrsqrt28ps", "vrsqrt28pd", "vrsqrt28ss", "vrsqrt28sd"} { + if !strings.Contains(ir, mnemonic) { + t.Fatalf("translation omitted inline %s:\n%s", mnemonic, ir) + } + } + for _, fragment := range []string{"kmovw", "{%k1}", "{sae}", `"target-features"="+avx512f"`} { + if !strings.Contains(ir, fragment) { + t.Fatalf("translation omitted %q:\n%s", fragment, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "approximate-math-"+target.name+".ll", "approximate-math-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ApproximateMathRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VEXP2PS X0, X1"}, + {goarch: "amd64", instruction: "VEXP2PD Y0, Y1"}, + {goarch: "amd64", instruction: "VRCP28PS Z0, K0, Z1"}, + {goarch: "amd64", instruction: "VRSQRT28PD.Z Z0, Z1"}, + {goarch: "amd64", instruction: "VEXP2PS.BCST Z0, Z1"}, + {goarch: "amd64", instruction: "VRCP28PD.SAE 0(AX), Z1"}, + {goarch: "amd64", instruction: "VRCP28SS X0, X1"}, + {goarch: "amd64", instruction: "VRSQRT28SD Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRCP28SS.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRSQRT28SD.SAE 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VEXP2PS.RN_SAE Z0, Z1"}, + {goarch: "386", instruction: "VEXP2PS Z0, Z8"}, + {goarch: "386", instruction: "VRCP28SS X0, X1, K1, X2"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's approximate-math tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_arith.go b/amd64_lower_arith.go index 2a25025ce..687c4cee3 100644 --- a/amd64_lower_arith.go +++ b/amd64_lower_arith.go @@ -24,6 +24,17 @@ func (c *amd64Ctx) loadIntDestination(dst Operand, ty LLVMType) (string, func(st fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", ty, out, ptrType, p) return nil }, nil + case OpFP: + value, err := c.evalFPToI64(dst.FPOffset) + if err != nil { + return "", nil, err + } + if ty != I64 { + value = c.truncI64(value, ty) + } + return value, func(out string) error { + return c.storeFPResult(dst.FPOffset, ty, out) + }, nil case OpSym: p, err := c.ptrFromSB(dst.Sym) if err != nil { @@ -45,22 +56,65 @@ type x86FlagSlot struct { bit uint } -func (c *amd64Ctx) x86FlagSlots() []x86FlagSlot { - flags := []x86FlagSlot{ +type x86StatusFlagMask uint16 + +const ( + x86StatusCF x86StatusFlagMask = 1 << 0 + x86StatusPF x86StatusFlagMask = 1 << 2 + x86StatusAF x86StatusFlagMask = 1 << 4 + x86StatusZF x86StatusFlagMask = 1 << 6 + x86StatusSF x86StatusFlagMask = 1 << 7 + x86StatusOF x86StatusFlagMask = 1 << 11 + x86StatusAll = x86StatusCF | x86StatusPF | x86StatusAF | x86StatusZF | x86StatusSF | x86StatusOF +) + +func (c *amd64Ctx) x86StatusFlagSlots() []x86FlagSlot { + return []x86FlagSlot{ {c.flagsCFSlot, 0}, + {c.flagsPFSlot, 2}, + {c.flagsAFSlot, 4}, {c.flagsZSlot, 6}, {c.flagsSltSlot, 7}, + {c.flagsOFSlot, 11}, } +} + +func (c *amd64Ctx) x86FlagSlots() []x86FlagSlot { + flags := c.x86StatusFlagSlots() if c.directionSlot != "" { flags = append(flags, x86FlagSlot{c.directionSlot, 10}) } - flags = append(flags, x86FlagSlot{c.flagsOFSlot, 11}) if c.flagsIDSlot != "" { flags = append(flags, x86FlagSlot{c.flagsIDSlot, 21}) } return flags } +// Clear exactly the ISA-defined status bits. Reusing the complete slot model +// prevents a consumer's hand-written subset from silently omitting AF. +func (c *amd64Ctx) clearX86StatusFlags(mask x86StatusFlagMask) { + for _, flag := range c.x86StatusFlagSlots() { + if mask&(1<= uint(operandBits) { + continue + } shifted := packed if flag.bit != 0 { shift := c.newTmp() @@ -100,68 +159,106 @@ func (c *amd64Ctx) truncI64(v string, ty LLVMType) string { return "%" + t } -func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err error) { - switch op { - case "PUSHL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 PUSHL requires GOARCH=386") - } - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 PUSHL expects src: %q", ins.Raw) - } - v32, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - return true, false, c.pushI32(v32) - case "POPL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 POPL requires GOARCH=386") - } - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 POPL expects dst: %q", ins.Raw) - } - v32, err := c.popI32() - if err != nil { - return true, false, err - } - switch ins.Args[0].Kind { - case OpReg: - return true, false, c.storeRegSized(ins.Args[0].Reg, I32, v32) - case OpMem: - p, ptrType, err := c.ptrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i32 %s, %s %s, align 1\n", v32, ptrType, p) - return true, false, nil +func isGoYmbRegisterForArch(r Reg, goarch string) bool { + if goarch == "386" { + // Go classifies SP/BP/SI/DI as Yrl32 in 386 mode, so they do not + // directly cover Ymb. Its assembler nevertheless accepts BP/SI/DI + // and synthesizes an equivalent sequence through BX; only a direct + // SP spelling remains invalid. + switch r { + case AX, BX, CX, DX, BP, SI, DI, AL, AH, BL, BH, CL, CH, DL, DH: + return true default: - return true, false, fmt.Errorf("amd64 POPL expects reg/mem dst: %q", ins.Raw) - } - case "PUSHFL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 PUSHFL requires GOARCH=386") - } - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("amd64 PUSHFL takes no operands: %q", ins.Raw) - } - flags := c.truncI64(c.packX86Flags(), I32) - return true, false, c.pushI32(flags) - case "POPFL": - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64 POPFL requires GOARCH=386") - } - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("amd64 POPFL takes no operands: %q", ins.Raw) + return false } - flags32, err := c.popI32() - if err != nil { - return true, false, err + } + if isX86YrlRegisterForArch(r, goarch) { + return true + } + switch r { + case AL, AH, BL, BH, CL, CH, DL, DH, + BPB, SIB, DIB, R8B, R9B, R10B, R11B, R12B, R13B, R14B, R15B: + return true + default: + return false + } +} + +// amd64YmbEffectiveRegister mirrors the raw ModRM numbering used when Go's +// broad Ymb table accepts a byte-register spelling for a wider NEG/NOT opcode. +func amd64YmbEffectiveRegister(r Reg, bits int) Reg { + if bits == 8 { + return r + } + switch r { + case AL: + return AX + case CL: + return CX + case DL: + return DX + case BL: + return BX + case AH: + return SP + case CH: + return BP + case DH: + return SI + case BH: + return DI + default: + if base, _, ok := amd64ByteAlias(r); ok { + return base } - flags64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", flags64, flags32) - c.unpackX86Flags("%" + flags64) - return true, false, nil + return r + } +} + +func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ok, terminated, err := c.lowerX86CPUQuery(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerX86OrderingHint(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerStackWidth(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerAccumulatorSignExtend(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerBZHI(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerBEXTR(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerADX(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarBitCount(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerBMI2VariableShift(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarUnaryYmb(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerLowestSetBitManipulation(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerHardwareRandom(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarADCSBB(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerScalarMultiplyDivide(op, ins); ok { + return ok, terminated, err + } + switch op { case "PUSHAL": if c.goarch != "386" { return true, false, fmt.Errorf("amd64 PUSHAL requires GOARCH=386") @@ -223,140 +320,34 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e } } return true, false, nil - case "PUSHQ": - // Stack-manipulation appears in syscall asm stubs (e.g. preserve return - // address register around SYSCALL). Lower to the local virtual stack. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 PUSHQ expects src: %q", ins.Raw) - } - v, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - c.pushI64(v) - return true, false, nil - case "POPQ": - if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 POPQ expects dstReg: %q", ins.Raw) - } - v := c.popI64() - if err := c.storeReg(ins.Args[0].Reg, v); err != nil { - return true, false, err - } - return true, false, nil - case "PUSHFQ": - // Keep the historical amd64 stack-shape model. The 32-bit PUSHFL path - // above carries flags because 386 runtime asm consumes them. - c.pushI64("0") - return true, false, nil - case "POPFQ": - _ = c.popI64() - return true, false, nil - case "LFENCE", "MFENCE", "SFENCE", "PAUSE": - if c.goarch == "386" { - fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", strings.ToLower(string(op)), "~{memory}") + case "PREFETCHNTA", "PREFETCHT0", "PREFETCHT1", "PREFETCHT2", + "PREFETCHRST2", "PREFETCHIT0", "PREFETCHIT1": + // Go 1.27's shared yprefetch table has exactly one memory operand for + // each named locality hint. The raw-only 0F 18 /4, /6 and /7 forms + // share that one-memory-operand grammar. Cache hints have no + // SSA-visible result. + if len(ins.Args) != 1 || !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s expects one memory operand: %q", c.goarch, op, ins.Raw) } return true, false, nil - case "PREFETCHNTA", "EMMS": - // Prefetch does not change SSA-visible values. MMX registers are modeled - // as ordinary integers, so there is no hardware x87 tag state to clear. + case "EMMS": + // MMX registers are modeled as ordinary integers, so there is no + // hardware x87 tag state to clear. return true, false, nil case "UNDEF": - if c.goarch == "386" { - c.emitX86Trap() - return true, true, nil - } - // Preserve the historical amd64 approximation. - return true, false, nil - case "RDTSC": - if c.goarch == "386" { - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 RDTSC takes no operands: %q", ins.Raw) - } - return true, false, c.lower386Timestamp(false) - } - if err := c.storeReg(AX, "0"); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "0"); err != nil { - return true, false, err - } - return true, false, nil - case OpCPUID: - eax64, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - ecx64, err := c.loadReg(CX) - if err != nil { - return true, false, err - } - eax32 := c.newTmp() - ecx32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", eax32, eax64) - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ecx32, ecx64) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call { i32, i32, i32, i32 } asm sideeffect \"cpuid\", \"={ax},={bx},={cx},={dx},{ax},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %%%s, i32 %%%s)\n", call, eax32, ecx32) - storeOut := func(idx int, reg Reg) error { - part := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i32, i32, i32, i32 } %%%s, %d\n", part, call, idx) - wide := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, part) - return c.storeReg(reg, "%"+wide) - } - if err := storeOut(0, AX); err != nil { - return true, false, err - } - if err := storeOut(1, BX); err != nil { - return true, false, err - } - if err := storeOut(2, CX); err != nil { - return true, false, err - } - if err := storeOut(3, DX); err != nil { - return true, false, err - } - return true, false, nil - case OpXGETBV: - ecx64, err := c.loadReg(CX) - if err != nil { - return true, false, err - } - ecx32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ecx32, ecx64) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call { i32, i32 } asm sideeffect \"xgetbv\", \"={ax},={dx},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %%%s)\n", call, ecx32) - storeOut := func(idx int, reg Reg) error { - part := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i32, i32 } %%%s, %d\n", part, call, idx) - wide := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, part) - return c.storeReg(reg, "%"+wide) - } - if err := storeOut(0, AX); err != nil { - return true, false, err - } - if err := storeOut(1, DX); err != nil { - return true, false, err - } - return true, false, nil - case "RDTSCP": - if c.goarch == "386" { - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 RDTSCP takes no operands: %q", ins.Raw) - } - return true, false, c.lower386Timestamp(true) - } - if err := c.storeReg(AX, "0"); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "0"); err != nil { - return true, false, err + return c.lowerX86UndefinedTrap(op, ins) + case "RDTSC", "RDTSCP": + return c.lowerX86Timestamp(op, ins) + case "RDPID": + if strings.ToUpper(string(ins.Op)) != string(op) || len(ins.Args) != 1 || ins.Args[0].Kind != OpReg || !isAMD64YrlRegister(ins.Args[0].Reg) { + return true, false, fmt.Errorf("%s RDPID expects one general register with no suffix: %q", c.goarch, ins.Raw) } - if err := c.storeReg(CX, "0"); err != nil { - return true, false, err + if c.goarch == "386" && amd64IsExtendedGPRegister(ins.Args[0].Reg) { + return true, false, fmt.Errorf("386 RDPID destination is outside the 386 register file: %q", ins.Raw) } - return true, false, nil + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.rdpid()\n", value) + return true, false, c.storeRegSized(ins.Args[0].Reg, I32, "%"+value) case "MOVSB": si, err := c.loadReg(SI) if err != nil { @@ -424,82 +415,6 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e return true, false, err } return true, false, nil - case "NEGL": - // 32-bit negate with x86 semantics: write back low 32 bits and zero-extend. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 NEGL expects dst: %q", ins.Raw) - } - switch ins.Args[0].Kind { - case OpReg: - dv, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - t32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", t32, dv) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 0, %%%s\n", neg, t32) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", cf, t32) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, neg) - if err := c.storeReg(ins.Args[0].Reg, "%"+z); err != nil { - return true, false, err - } - c.setZSFlagsFromI32("%" + neg) - return true, false, nil - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", ld, p) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 0, %%%s\n", neg, ld) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", cf, ld) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - fmt.Fprintf(c.b, " store i32 %%%s, ptr %s, align 1\n", neg, p) - c.setZSFlagsFromI32("%" + neg) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 NEGL expects reg/mem dst: %q", ins.Raw) - } - case "RCRQ": - // Rotate through carry right (count=1) used by runtime time division path. - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm { - return true, false, fmt.Errorf("amd64 RCRQ expects $count, dstReg: %q", ins.Raw) - } - if ins.Args[0].Imm != 1 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 RCRQ currently supports $1, reg: %q", ins.Raw) - } - dv, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - oldCF := c.loadFlag(c.flagsCFSlot) - lsb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 1\n", lsb, dv) - newCF := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", newCF, lsb) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", newCF, c.flagsCFSlot) - shr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, 1\n", shr, dv) - cf64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", cf64, oldCF) - cfhi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 63\n", cfhi, cf64) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", out, shr, cfhi) - if err := c.storeReg(ins.Args[1].Reg, "%"+out); err != nil { - return true, false, err - } - c.setZSFlagsFromI64("%" + out) - return true, false, nil - case "ADDQ", "SUBQ", "XORQ", "ANDQ", "ORQ": if len(ins.Args) != 2 { return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) @@ -566,7 +481,24 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e default: fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) } - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + if op == "ADDQ" || op == "SUBQ" { + xorOperands := c.newTmp() + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", xorOperands, dv, src) + if op == "ADDQ" { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %%%s, -1\n", inverted, xorOperands) + xorOperands = inverted + } + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", xorResult, dv, r) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %%%s\n", overflowBits, xorOperands, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt i64 %%%s, 0\n", overflow, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + } else { + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + } c.setZSFlagsFromI64(r) return true, false, nil @@ -759,53 +691,6 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sf, c.flagsSltSlot) return true, false, nil - case "ADCXQ", "ADOXQ": - // BMI2 add-with-carry chains. - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - carryIn := c.loadFlag(c.flagsCFSlot) - flagOut := c.flagsCFSlot - if op == "ADOXQ" { - carryIn = c.loadFlag(c.flagsOFSlot) - flagOut = c.flagsOFSlot - } - cf64t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", cf64t, carryIn) - cf64 := "%" + cf64t - sum := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", sum, dv, src) - res := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %s\n", res, sum, cf64) - out := "%" + res - if err := c.storeReg(dst, out); err != nil { - return true, false, err - } - dv128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", dv128, dv) - src128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", src128, src) - cf128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", cf128, cf64) - total1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i128 %%%s, %%%s\n", total1, dv128, src128) - total2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i128 %%%s, %%%s\n", total2, total1, cf128) - carry := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ugt i128 %%%s, 18446744073709551615\n", carry, total2) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, flagOut) - // ADCX/ADOX do not define ZF/SF in the same way as ADD; keep current bits. - return true, false, nil - case "ADDL", "SUBL", "XORL", "ANDL", "ORL": // 32-bit arithmetic/logical ops: src, dst. if len(ins.Args) != 2 { @@ -1007,6 +892,12 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e fmt.Fprintf(c.b, " store i64 %s, ptr %s, align 1\n", out, p) return nil } + case OpFP: + var err error + v, storeDst, err = c.loadIntDestination(ins.Args[0], I64) + if err != nil { + return true, false, err + } default: return true, false, fmt.Errorf("amd64 %s expects reg/mem dst: %q", op, ins.Raw) } @@ -1055,11 +946,17 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e fmt.Fprintf(c.b, " store i32 %s, ptr %s, align 1\n", out32, p) return nil } + case OpFP: + var err error + v64, storeDst, err = c.loadIntDestination(ins.Args[0], I32) + if err != nil { + return true, false, err + } default: return true, false, fmt.Errorf("amd64 %s expects reg/mem dst: %q", op, ins.Raw) } tr := c.newTmp() - if ins.Args[0].Kind == OpMem { + if ins.Args[0].Kind == OpMem || ins.Args[0].Kind == OpFP { fmt.Fprintf(c.b, " %%%s = add i32 0, %s\n", tr, v64) } else { fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) @@ -1076,21 +973,33 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e c.setZSFlagsFromI32("%" + x) return true, false, nil - case "LEAQ", "LEAL": - // LEA{Q,L} srcAddr, dstReg - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects srcAddr, dstReg: %q", op, ins.Raw) + case "LEAW", "LEAL", "LEAQ": + // All three widths use Go 1.27's sole ym_rl row: an addressable + // memory expression and an in-range general-register destination. + if ins.Op != "" && Op(strings.ToUpper(string(ins.Op))) != op { + return true, false, fmt.Errorf("%s LEA dispatch %q disagrees with instruction %q", c.goarch, op, ins.Op) + } + // Internal helpers may omit Instr.Op. Bind only this local copy so + // grammar validation and destination emission use one identity; the + // original source/caller instruction remains independently required. + ins.Op = op + if _, err := x86LEAOperandWidth(ins, c.goarch); err != nil { + return true, false, err } dst := ins.Args[1].Reg storeLEA := func(addr string) error { - if op == "LEAL" { + switch op { + case "LEAW": + t := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i16\n", t, addr) + return c.storeRegSized(dst, I16, "%"+t) + case "LEAL": t := c.newTmp() fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", t, addr) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, t) - return c.storeReg(dst, "%"+z) + return c.storeRegSized(dst, I32, "%"+t) + default: + return c.storeRegSized(dst, I64, addr) } - return c.storeReg(dst, addr) } switch ins.Args[0].Kind { case OpMem: @@ -1098,46 +1007,24 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e if err != nil { return true, false, err } + if bits := ins.x86AddressBits; bits != 0 { + // Address size is independent of the result register width. + // Narrow the complete sum first, then zero-extend that effective + // offset before W/L/Q destination preservation/extension. + typ := I32 + if bits == 16 { + typ = I16 + } + low := c.truncI64(addr, typ) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, typ, low) + addr = "%" + wide + } return true, false, storeLEA(addr) case OpFP: - if c.classicFrame != "" { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - addr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", addr, c.classicFramePtr(ins.Args[0].FPOffset)) - return true, false, storeLEA("%" + addr) - } - // LEA of a return slot, e.g. "LEAQ ret+32(FP), R8". - alloca, _, ok := c.fpResultAlloca(ins.Args[0].FPOffset) - if ok { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", t, alloca) - return true, false, storeLEA("%" + t) - } - // Fallback: treat FP slot value as pointer-like integer address. - v, err := c.evalFPToI64(ins.Args[0].FPOffset) - if err != nil { - v = "0" - } - return true, false, storeLEA(v) - case OpFPAddr: - if c.classicFrame != "" { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - addr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", addr, c.classicFramePtr(ins.Args[0].FPOffset)) - return true, false, storeLEA("%" + addr) - } - // Address of a return slot alloca. - alloca, _, ok := c.fpResultAlloca(ins.Args[0].FPOffset) - if ok { - c.markFPResultAddrTaken(ins.Args[0].FPOffset) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to i64\n", t, alloca) - return true, false, storeLEA("%" + t) - } - v, err := c.evalFPToI64(ins.Args[0].FPOffset) + v, err := c.boundFPAddress(ins.Args[0].FPOffset) if err != nil { - v = "0" + return true, false, err } return true, false, storeLEA(v) case OpSym: @@ -1152,80 +1039,11 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e return true, false, fmt.Errorf("amd64 %s unsupported src: %q", op, ins.Raw) } - case "POPCNTL", "POPCNTQ": - // POPCNT{L,Q} src, dstReg accepts the same register-or-memory source - // family as the Go x86 assembler. + case "BSFW", "BSRW": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) } - validFullReg := func(r Reg) bool { - if _, _, ok := amd64ByteAlias(r); ok || r == FS || r == GS { - return false - } - _, ok := amd64FullRegBase(r) - return ok - } - if !validFullReg(ins.Args[1].Reg) { - return false, false, nil - } - src := ins.Args[0] - switch src.Kind { - case OpReg: - if !validFullReg(src.Reg) { - return false, false, nil - } - case OpMem, OpFP: - case OpSym: - if strings.HasPrefix(strings.TrimSpace(src.Sym), "$") { - return false, false, nil - } - default: - return false, false, nil - } - if op == "POPCNTL" { - srcv, err := c.evalIntSized(src, I32) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.ctpop.i32(i32 %s)\n", call, srcv) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - } - srcv, err := c.evalIntSized(src, I64) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.ctpop.i64(i64 %s)\n", call, srcv) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+call) - - case "TZCNTQ": - // TZCNTQ srcReg, dstReg. - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 TZCNTQ expects srcReg, dstReg: %q", ins.Raw) - } - srcv, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.cttz.i64(i64 %s, i1 false)\n", call, srcv) - if err := c.storeReg(ins.Args[1].Reg, "%"+call); err != nil { - return true, false, err - } - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 0\n", cf, srcv) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - c.setZSFlagsFromI64("%" + call) - return true, false, nil - - case "BSFW", "BSRW": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) - } - src, err := c.evalIntSized(ins.Args[0], I16) + src, err := c.evalIntSized(ins.Args[0], I16) if err != nil { return true, false, err } @@ -1344,34 +1162,20 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e cond := "" switch op { case "SETEQ": - cond = c.loadFlag(c.flagsZSlot) + cond, err = c.x86Condition("EQ") case "SETLT": - cond = c.loadFlag(c.flagsSltSlot) + cond, err = c.x86Condition("LT") case "SETGT": - // signed > - slt := c.loadFlag(c.flagsSltSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, slt, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 + cond, err = c.x86Condition("GT") case "SETGE": - slt := c.loadFlag(c.flagsSltSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, slt) - cond = "%" + t + cond, err = c.x86Condition("GE") case "SETHI": - // unsigned > - cf := c.loadFlag(c.flagsCFSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, cf, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 + cond, err = c.x86Condition("HI") case "SETCS": - cond = c.loadFlag(c.flagsCFSlot) + cond, err = c.x86Condition("CS") + } + if err != nil { + return true, false, err } switch ins.Args[0].Kind { case OpReg: @@ -1393,17 +1197,23 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e return true, false, fmt.Errorf("amd64 %s expects reg, mem, or fp destination: %q", op, ins.Raw) } - case "CMOVQEQ", "CMOVQNE", "CMOVQCS", "CMOVQCC", "CMOVQGT": + case "CMOVQEQ", "CMOVQNE", "CMOVQCS", "CMOVQCC", "CMOVQGT", "CMOVQLE", "CMOVLGT", "CMOVWCC": // Conditional move: src, dstReg if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { return true, false, fmt.Errorf("amd64 %s expects src, dstReg: %q", op, ins.Raw) } - src, err := c.evalI64(ins.Args[0]) + ty := I64 + if op == "CMOVLGT" { + ty = I32 + } else if op == "CMOVWCC" { + ty = I16 + } + src, err := c.evalIntSized(ins.Args[0], ty) if err != nil { return true, false, err } dst := ins.Args[1].Reg - cur, err := c.loadReg(dst) + cur, err := c.evalIntSized(ins.Args[1], ty) if err != nil { return true, false, err } @@ -1418,12 +1228,12 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e cond = "%" + nz case "CMOVQCS": cond = c.loadFlag(c.flagsCFSlot) - case "CMOVQCC": + case "CMOVQCC", "CMOVWCC": cf := c.loadFlag(c.flagsCFSlot) nc := c.newTmp() fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", nc, cf) cond = "%" + nc - case "CMOVQGT": + case "CMOVQGT", "CMOVLGT": slt := c.loadFlag(c.flagsSltSlot) z := c.loadFlag(c.flagsZSlot) t1 := c.newTmp() @@ -1431,121 +1241,54 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e t2 := c.newTmp() fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) cond = "%" + t2 + case "CMOVQLE": + slt := c.loadFlag(c.flagsSltSlot) + z := c.loadFlag(c.flagsZSlot) + t := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t, slt, z) + cond = "%" + t } sel := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %s, i64 %s\n", sel, cond, src, cur) - return true, false, c.storeReg(dst, "%"+sel) + fmt.Fprintf(c.b, " %%%s = select i1 %s, %s %s, %s %s\n", sel, cond, ty, src, ty, cur) + return true, false, c.storeRegSized(dst, ty, "%"+sel) case "ANDNL", "ANDNQ": - // BMI1 ANDN: dst = ~src2 & src1 - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src1, src2, dstReg: %q", op, ins.Raw) - } - src1, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err + // BMI1 ANDN: destination = ~second & first. Go's _yandnl table + // accepts Yml,Yrl,Yrl for both spellings. VEX.W is ignored in 386. + if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[1].Reg, c.goarch) || ins.Args[2].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[2].Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s expects GP/memory source, GP source, GP destination: %q", c.goarch, op, ins.Raw) } - src2, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err + if ins.Args[0].Kind == OpReg { + if !isX86YrlRegisterForArch(ins.Args[0].Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s first source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s first source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) } - dst := ins.Args[2].Reg - if op == "ANDNQ" { - n := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %s, -1\n", n, src2) - a := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %s\n", a, n, src1) - return true, false, c.storeReg(dst, "%"+a) - } - s1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", s1, src1) - s2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", s2, src2) - n := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i32 %%%s, -1\n", n, s2) - a := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, %%%s\n", a, n, s1) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, a) - return true, false, c.storeReg(dst, "%"+z) - - case "BEXTRQ": - // BMI1 bit field extract: control, src, dst. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BEXTRQ expects control, src, dstReg: %q", ins.Raw) + typ := I32 + if op == "ANDNQ" && c.goarch == "amd64" { + typ = I64 } - ctrl, err := c.evalI64(ins.Args[0]) + first, err := c.evalIntSized(ins.Args[0], typ) if err != nil { return true, false, err } - src, err := c.evalI64(ins.Args[1]) + second, err := c.evalIntSized(ins.Args[1], typ) if err != nil { return true, false, err } - start := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 255\n", start, ctrl) - lenShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, 8\n", lenShift, ctrl) - length := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 255\n", length, lenShift) - startOK := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %%%s, 64\n", startOK, start) - safeStart := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 63\n", safeStart, startOK, start) - shifted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %%%s\n", shifted, src, safeStart) - rawLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 0\n", rawLen, startOK, length) - remain := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 64, %%%s\n", remain, safeStart) - useRawLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %%%s, %%%s\n", useRawLen, rawLen, remain) - effLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %%%s\n", effLen, useRawLen, rawLen, remain) - isFull := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 64\n", isFull, effLen) - safeLen := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", safeLen, effLen) - one := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 1, %%%s\n", one, safeLen) - maskTmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, -1\n", maskTmp, one) - mask := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 -1, i64 %%%s\n", mask, isFull, maskTmp) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %%%s\n", out, shifted, mask) - return true, false, c.storeReg(ins.Args[2].Reg, "%"+out) - - case "BZHIQ": - // BMI2 zero high bits: index, src, dst. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BZHIQ expects index, src, dstReg: %q", ins.Raw) - } - idx, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", inverted, typ, second) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", result, typ, inverted, first) + if typ == I64 { + c.setZSFlagsFromI64("%" + result) + } else { + c.setZSFlagsFromI32("%" + result) } - valid := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, 64\n", valid, idx) - isZero := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 0\n", isZero, idx) - safeIdx := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", safeIdx, idx) - one := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 1, %%%s\n", one, safeIdx) - maskTmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %%%s, -1\n", maskTmp, one) - maskOrZero := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 0, i64 %%%s\n", maskOrZero, isZero, maskTmp) - mask := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 -1\n", mask, valid, maskOrZero) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, %%%s\n", out, src, mask) - return true, false, c.storeReg(ins.Args[2].Reg, "%"+out) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + return true, false, c.storeRegSized(ins.Args[2].Reg, typ, "%"+result) case "SHRQ", "SHLQ", "SARQ", "SHLL", "SHRL", "SARL", "SALQ", "SALL": // Shift ops: @@ -1708,38 +1451,6 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i8 %%%s, i8 0\n", out, inRange, sh) return true, false, c.storeRegSized(dst, I8, "%"+out) - case "SHLXQ", "SHRXQ": - // BMI2 variable shifts: amt, src, dst. - if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects amt, srcReg, dstReg: %q", op, ins.Raw) - } - src, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - var amt string - switch ins.Args[0].Kind { - case OpImm: - amt = fmt.Sprintf("%d", ins.Args[0].Imm&63) - case OpReg: - av, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", m, av) - amt = "%" + m - default: - return true, false, fmt.Errorf("amd64 %s unsupported shift amt: %q", op, ins.Raw) - } - t := c.newTmp() - if op == "SHLXQ" { - fmt.Fprintf(c.b, " %%%s = shl i64 %s, %s\n", t, src, amt) - } else { - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %s\n", t, src, amt) - } - return true, false, c.storeReg(ins.Args[2].Reg, "%"+t) - case "ROLL": // 32-bit rotate-left: count, dstReg. if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { @@ -1903,18 +1614,29 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e return true, false, c.storeReg(dst, "%"+z) case "RORXL", "RORXQ": - // BMI2 rotate-right without flags: - // - RORXL $imm, srcReg, dstReg (32-bit) - // - RORXQ $imm, srcReg, dstReg (64-bit) - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, srcReg, dstReg: %q", op, ins.Raw) - } - src, err := c.loadReg(ins.Args[1].Reg) - if err != nil { + if _, _, err := x86ScalarShiftProperties(op, ins, c.goarch); err != nil { return true, false, err } + // BMI2 rotate-right without flags. Go's Yml source class accepts a + // general register or every ModRM/SIB memory form; Yrl is the complete + // destination class. In 386 mode Go accepts the Q spelling too, but + // VEX.W is ignored and the architectural operation remains 32-bit. + if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[2].Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s expects $imm, GP/memory source, in-range GP destination: %q", c.goarch, op, ins.Raw) + } + if ins.Args[1].Kind == OpReg { + if !isX86YrlRegisterForArch(ins.Args[1].Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } dst := ins.Args[2].Reg - if op == "RORXQ" { + if op == "RORXQ" && c.goarch == "amd64" { + src, err := c.evalIntSized(ins.Args[1], I64) + if err != nil { + return true, false, err + } n := uint64(ins.Args[0].Imm) & 63 neg := c.newTmp() fmt.Fprintf(c.b, " %%%s = sub i64 64, %d\n", neg, n) @@ -1928,61 +1650,29 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", rot, lhs, rhs) return true, false, c.storeReg(dst, "%"+rot) } + src, err := c.evalIntSized(ins.Args[1], I32) + if err != nil { + return true, false, err + } n := uint32(ins.Args[0].Imm) & 31 - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, src) neg := c.newTmp() fmt.Fprintf(c.b, " %%%s = sub i32 32, %d\n", neg, n) nm := c.newTmp() fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 31\n", nm, neg) lhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i32 %%%s, %d\n", lhs, tr, n) + fmt.Fprintf(c.b, " %%%s = lshr i32 %s, %d\n", lhs, src, n) rhs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, %%%s\n", rhs, tr, nm) + fmt.Fprintf(c.b, " %%%s = shl i32 %s, %%%s\n", rhs, src, nm) rot := c.newTmp() fmt.Fprintf(c.b, " %%%s = or i32 %%%s, %%%s\n", rot, lhs, rhs) z := c.newTmp() fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, rot) return true, false, c.storeReg(dst, "%"+z) - case "NOTB": - // NOT does not modify flags. For a register operand, only the selected - // low byte is changed and the remaining register bits are preserved. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 NOTB expects one operand: %q", ins.Raw) - } - switch ins.Args[0].Kind { - case OpReg: - r := ins.Args[0].Reg - v64, err := c.loadReg(r) - if err != nil { - return true, false, err - } - v8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", v8, v64) - not := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, -1\n", not, v8) - return true, false, c.storeRegSized(r, I8, "%"+not) - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - load := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i8, ptr %s, align 1\n", load, p) - not := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i8 %%%s, -1\n", not, load) - fmt.Fprintf(c.b, " store i8 %%%s, ptr %s, align 1\n", not, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 NOTB expects reg or mem: %q", ins.Raw) - } - - case "NOTL": - // 32-bit bitwise NOT, result zero-extended to 64-bit. + case "BSWAPL": + // BSWAPL reg: byte swap low 32 bits and zero-extend result to i64. if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 NOTL expects reg: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 BSWAPL expects reg: %q", ins.Raw) } r := ins.Args[0].Reg v64, err := c.loadReg(r) @@ -1991,94 +1681,67 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e } tr := c.newTmp() fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - x := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i32 %%%s, -1\n", x, tr) + bswap := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.bswap.i32(i32 %%%s)\n", bswap, tr) z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, x) + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, bswap) return true, false, c.storeReg(r, "%"+z) - case "NOTQ": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 NOTQ expects one operand: %q", ins.Raw) + case "MULXL": + // BMI2 MULXL src, loDst, hiDst multiplies the low 32 bits of DX by + // a 32-bit register or memory source and zero-extends both results. + if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 MULXL expects src, loDst, hiDst: %q", ins.Raw) } + var src32 string switch ins.Args[0].Kind { case OpReg: - r := ins.Args[0].Reg - v, err := c.loadReg(r) + src64, err := c.loadReg(ins.Args[0].Reg) if err != nil { return true, false, err } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %s, -1\n", t, v) - return true, false, c.storeReg(r, "%"+t) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", value, src64) + src32 = "%" + value case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) + pointer, pointerType, err := c.ptrFromMem(ins.Args[0].Mem) if err != nil { return true, false, err } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %%%s, -1\n", t, ld) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s, align 1\n", t, p) - return true, false, nil + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i32, %s %s, align 1\n", value, pointerType, pointer) + src32 = "%" + value default: - return true, false, fmt.Errorf("amd64 NOTQ expects reg or mem: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 MULXL source must be register or memory: %q", ins.Raw) } - - case "BSWAPL": - // BSWAPL reg: byte swap low 32 bits and zero-extend result to i64. - if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BSWAPL expects reg: %q", ins.Raw) - } - r := ins.Args[0].Reg - v64, err := c.loadReg(r) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - bswap := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.bswap.i32(i32 %%%s)\n", bswap, tr) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, bswap) - return true, false, c.storeReg(r, "%"+z) - - case "MULQ": - // MULQ src: RDX:RAX = RAX * src (unsigned). - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 MULQ expects src: %q", ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - ax, err := c.loadReg(AX) + dx64, err := c.loadReg(DX) if err != nil { return true, false, err } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", a128, ax) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i64 %s to i128\n", b128, src) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) + dx32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", dx32, dx64) + a64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", a64, dx32) + b64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", b64, src32) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %%%s\n", product, a64, b64) + lo32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", lo32, product) hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i128 %%%s, 64\n", hiShift, p) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", hi, hiShift) - if err := c.storeReg(AX, "%"+lo); err != nil { + fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, 32\n", hiShift, product) + hi32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", hi32, hiShift) + lo64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", lo64, lo32) + hi64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", hi64, hi32) + if err := c.storeReg(ins.Args[1].Reg, "%"+lo64); err != nil { return true, false, err } - if err := c.storeReg(DX, "%"+hi); err != nil { + if err := c.storeReg(ins.Args[2].Reg, "%"+hi64); err != nil { return true, false, err } - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", cf, hi) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) return true, false, nil case "MULXQ": @@ -2114,260 +1777,6 @@ func (c *amd64Ctx) lowerArith(op Op, ins Instr) (ok bool, terminated bool, err e } return true, false, nil - case "MULL": - // MULL src: EDX:EAX = EAX * src (unsigned 32-bit) - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 MULL expects src: %q", ins.Raw) - } - src32, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - ax64, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - ax32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ax32, ax64) - az := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", az, ax32) - bz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", bz, src32) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %%%s\n", p, az, bz) - lo32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", lo32, p) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, 32\n", hiShift, p) - hi32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", hi32, hiShift) - lo64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", lo64, lo32) - hi64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", hi64, hi32) - if err := c.storeReg(AX, "%"+lo64); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "%"+hi64); err != nil { - return true, false, err - } - return true, false, nil - - case "DIVL": - // DIVL src: unsigned divide EDX:EAX by src; quotient->EAX remainder->EDX. - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("amd64 DIVL expects src: %q", ins.Raw) - } - src32, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - ax64, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - dx64, err := c.loadReg(DX) - if err != nil { - return true, false, err - } - ax32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", ax32, ax64) - dx32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", dx32, dx64) - az := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", az, ax32) - dz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", dz, dx32) - divisor := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", divisor, src32) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 32\n", hi, dz) - dividend := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", dividend, hi, az) - q := c.newTmp() - fmt.Fprintf(c.b, " %%%s = udiv i64 %%%s, %%%s\n", q, dividend, divisor) - r := c.newTmp() - fmt.Fprintf(c.b, " %%%s = urem i64 %%%s, %%%s\n", r, dividend, divisor) - q32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", q32, q) - r32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", r32, r) - q64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", q64, q32) - r64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", r64, r32) - if err := c.storeReg(AX, "%"+q64); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "%"+r64); err != nil { - return true, false, err - } - return true, false, nil - - case "IMULL", "IMUL3L": - // Signed 32-bit multiply. The one-operand form writes DX:AX; the two- - // and three-operand forms write the low word to their explicit dst. - switch len(ins.Args) { - case 1: - src, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) - if err != nil { - return true, false, err - } - a64 := c.newTmp() - b64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i32 %s to i64\n", a64, ax) - fmt.Fprintf(c.b, " %%%s = sext i32 %s to i64\n", b64, src) - product := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %%%s\n", product, a64, b64) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", lo, product) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ashr i64 %%%s, 32\n", hiShift, product) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", hi, hiShift) - if err := c.storeRegSized(AX, I32, "%"+lo); err != nil { - return true, false, err - } - return true, false, c.storeRegSized(DX, I32, "%"+hi) - case 2: - if ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 IMULL expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - dst, err := c.evalIntSized(ins.Args[1], I32) - if err != nil { - return true, false, err - } - product := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i32 %s, %s\n", product, dst, src) - return true, false, c.storeRegSized(ins.Args[1].Reg, I32, "%"+product) - case 3: - if op != "IMUL3L" || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects imm, src, dstReg: %q", op, ins.Raw) - } - a, err := c.evalIntSized(ins.Args[0], I32) - if err != nil { - return true, false, err - } - bv, err := c.evalIntSized(ins.Args[1], I32) - if err != nil { - return true, false, err - } - product := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i32 %s, %s\n", product, bv, a) - return true, false, c.storeRegSized(ins.Args[2].Reg, I32, "%"+product) - default: - return true, false, fmt.Errorf("amd64 %s unsupported operand count: %q", op, ins.Raw) - } - - case "IMULQ", "IMUL3Q": - // IMULQ src -> RDX:RAX = signed RAX*src - // IMULQ src, dst -> dst = signed(dst*src) - // IMUL3Q imm,src,dst -> dst = signed(src*imm) - switch len(ins.Args) { - case 1: - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - ax, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", a128, ax) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", b128, src) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - hiShift := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ashr i128 %%%s, 64\n", hiShift, p) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", hi, hiShift) - if err := c.storeReg(AX, "%"+lo); err != nil { - return true, false, err - } - if err := c.storeReg(DX, "%"+hi); err != nil { - return true, false, err - } - return true, false, nil - case 2: - if ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 IMULQ expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst := ins.Args[1].Reg - dv, err := c.loadReg(dst) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", a128, dv) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", b128, src) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - return true, false, c.storeReg(dst, "%"+lo) - case 3: - if op != "IMUL3Q" || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects imm, src, dstReg: %q", op, ins.Raw) - } - imm, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - src, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - a128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", a128, src) - b128 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext i64 %s to i128\n", b128, imm) - p := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", p, a128, b128) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i128 %%%s to i64\n", lo, p) - return true, false, c.storeReg(ins.Args[2].Reg, "%"+lo) - default: - return true, false, fmt.Errorf("amd64 %s expects 1/2/3 operands: %q", op, ins.Raw) - } - - case "NEGQ": - // NEGQ reg - if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 NEGQ expects reg: %q", ins.Raw) - } - r := ins.Args[0].Reg - v, err := c.loadReg(r) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i64 0, %s\n", t, v) - out := "%" + t - if err := c.storeReg(r, out); err != nil { - return true, false, err - } - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %s, 0\n", cf, v) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - c.setZSFlagsFromI64(out) - return true, false, nil } return false, false, nil } @@ -2438,26 +1847,3 @@ func (c *amd64Ctx) lowerDoubleShift(op Op, ins Instr) error { fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %s, %s %%%s\n", out, zero, ty, dst, ty, merged) return storeDst("%" + out) } - -func (c *amd64Ctx) lower386Timestamp(serialized bool) error { - asm := "rdtsc" - resultType := "{ i32, i32 }" - constraints := "={ax},={dx},~{dirflag},~{fpsr},~{flags},~{memory}" - regs := []Reg{AX, DX} - if serialized { - asm = "rdtscp" - resultType = "{ i32, i32, i32 }" - constraints = "={ax},={dx},={cx},~{dirflag},~{fpsr},~{flags},~{memory}" - regs = append(regs, CX) - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", call, resultType, asm, constraints) - for i, reg := range regs { - value := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, %d\n", value, resultType, call, i) - if err := c.storeRegSized(reg, I32, "%"+value); err != nil { - return err - } - } - return nil -} diff --git a/amd64_lower_arithmetic_shift.go b/amd64_lower_arithmetic_shift.go new file mode 100644 index 000000000..75e19c782 --- /dev/null +++ b/amd64_lower_arithmetic_shift.go @@ -0,0 +1,241 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedArithmeticRightShift implements the two Go 1.27 packed uniform +// arithmetic-right-shift tables: legacy PSRAW/PSRAL use yps, VPSRAW/VPSRAD use +// _yvpslld, and EVEX-only VPSRAQ uses _yvpsraq. +func (c *amd64Ctx) lowerPackedArithmeticRightShift(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + switch baseOp { + case "PSRAW": + return c.lowerLegacyPackedArithmeticRightShift(baseOp, suffix, 16, ins) + case "PSRAL": + return c.lowerLegacyPackedArithmeticRightShift(baseOp, suffix, 32, ins) + case "VPSRAW": + return c.lowerVectorPackedArithmeticRightShift(baseOp, suffix, 16, false, ins) + case "VPSRAD": + return c.lowerVectorPackedArithmeticRightShift(baseOp, suffix, 32, true, ins) + case "VPSRAQ": + return c.lowerVectorPackedArithmeticRightShift(baseOp, suffix, 64, true, ins) + default: + return false, false, nil + } +} + +func (c *amd64Ctx) lowerLegacyPackedArithmeticRightShift(baseOp, suffix string, laneBits int, ins Instr) (bool, bool, error) { + if suffix != "" { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects count, packed destination: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[1].Reg + if _, mmx := amd64ParseMReg(destination); mmx { + if ins.Args[0].Kind != OpImm && ins.Args[0].Kind == OpReg { + if _, ok := amd64ParseMReg(ins.Args[0].Reg); !ok { + return true, false, fmt.Errorf("%s %s MMX form requires an immediate, MMX, or memory count: %q", c.goarch, baseOp, ins.Raw) + } + } else if ins.Args[0].Kind != OpImm && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s MMX form requires an immediate, MMX, or memory count: %q", c.goarch, baseOp, ins.Raw) + } + dataBits, err := c.loadReg(destination) + if err != nil { + return true, false, err + } + lanes := 64 / laneBits + data := c.bitcastI64ToIntegerLanes(lanes, laneBits, dataBits) + count, err := c.loadPackedUniformShiftCount(ins.Args[0], true, laneBits) + if err != nil { + return true, false, err + } + result := c.emitPackedArithmeticRightShift(lanes, laneBits, data, count) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", bits, lanes, laneBits, result) + return true, false, c.storeReg(destination, "%"+bits) + } + + if !c.isGoLegacyXReg(destination) { + return true, false, fmt.Errorf("%s %s destination must be an in-range MMX or X register: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg && !c.isGoLegacyXReg(ins.Args[0].Reg) { + return true, false, fmt.Errorf("%s %s XMM count must be an in-range X register: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm && ins.Args[0].Kind != OpReg && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s XMM form requires an immediate, X, or memory count: %q", c.goarch, baseOp, ins.Raw) + } + dataBytes, err := c.loadX(destination) + if err != nil { + return true, false, err + } + lanes := 128 / laneBits + data := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, dataBytes) + count, err := c.loadPackedUniformShiftCount(ins.Args[0], false, laneBits) + if err != nil { + return true, false, err + } + result := c.emitPackedArithmeticRightShift(lanes, laneBits, data, count) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <16 x i8>\n", out, lanes, laneBits, result) + return true, false, c.storeX(destination, "%"+out) +} + +func (c *amd64Ctx) lowerVectorPackedArithmeticRightShift(baseOp, suffix string, laneBits int, broadcastEnabled bool, ins Instr) (bool, bool, error) { + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s suffix is absent from its Go 1.27 table: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast && !broadcastEnabled { + return true, false, fmt.Errorf("%s %s does not enable EVEX broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects count, source, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7 as its third operand: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + + countArg, dataArg := ins.Args[0], ins.Args[1] + immediate := countArg.Kind == OpImm + if immediate { + if countArg.Imm < -128 || countArg.Imm > 255 { + return true, false, fmt.Errorf("%s %s immediate count is outside Go's signed/unsigned byte classes: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast { + if !isAMD64MemoryOperand(dataArg) { + return true, false, fmt.Errorf("%s %s.BCST requires an immediate count and scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if dataArg.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(dataArg, byteWidth) { + return true, false, fmt.Errorf("%s %s source register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(dataArg) { + return true, false, fmt.Errorf("%s %s immediate form requires matching vector or memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else { + if properties.broadcast { + return true, false, fmt.Errorf("%s %s.BCST is absent from variable-count rows: %q", c.goarch, baseOp, ins.Raw) + } + if countArg.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(countArg, 16) { + return true, false, fmt.Errorf("%s %s variable count must be X/m128: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(countArg) { + return true, false, fmt.Errorf("%s %s variable count must be X/m128: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoPackedVectorMoveRegister(dataArg, byteWidth) { + return true, false, fmt.Errorf("%s %s variable-count data must be a matching vector register: %q", c.goarch, baseOp, ins.Raw) + } + } + + lanes := byteWidth * 8 / laneBits + var data string + var err error + if properties.broadcast { + var scalar string + scalar, err = c.evalIntSized(dataArg, amd64IntegerTypeForBits(laneBits)) + if err == nil { + data = amd64SplatInteger(c, lanes, laneBits, scalar) + } + } else { + var bytesValue string + bytesValue, err = c.loadPackedCompareBytes(dataArg, byteWidth) + if err == nil { + data = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, bytesValue) + } + } + if err != nil { + return true, false, err + } + count, err := c.loadPackedUniformShiftCount(countArg, false, laneBits) + if err != nil { + return true, false, err + } + result := c.emitPackedArithmeticRightShift(lanes, laneBits, data, count) + if masked { + mask, err := c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +// loadPackedUniformShiftCount returns an i64 count clamped to laneBits-1, as +// specified for x86 packed arithmetic shifts when the count is too large. +func (c *amd64Ctx) loadPackedUniformShiftCount(count Operand, mmx bool, laneBits int) (string, error) { + if count.Kind == OpImm { + value := uint64(count.Imm) & 0xff + if value >= uint64(laneBits) { + value = uint64(laneBits - 1) + } + return fmt.Sprintf("%d", value), nil + } + var count64 string + var err error + if mmx { + count64, err = c.loadLegacyMMXPackedSource(count) + } else { + var countBytes string + countBytes, err = c.loadXVecOperand(count) + if err == nil { + countWords := c.newTmp() + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", countWords, countBytes) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, countWords) + count64 = "%" + low + } + } + if err != nil { + return "", err + } + inRange := c.newTmp() + clamped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %d\n", inRange, count64, laneBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 %d\n", clamped, inRange, count64, laneBits-1) + return "%" + clamped, nil +} + +func (c *amd64Ctx) emitPackedArithmeticRightShift(lanes, laneBits int, data, count64 string) string { + count := count64 + if laneBits < 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, count64, laneBits) + count = "%" + narrow + } + counts := amd64SplatInteger(c, lanes, laneBits, count) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ashr <%d x i%d> %s, %s\n", result, lanes, laneBits, data, counts) + return "%" + result +} diff --git a/amd64_lower_arithmetic_shift_test.go b/amd64_lower_arithmetic_shift_test.go new file mode 100644 index 000000000..7823d9c58 --- /dev/null +++ b/amd64_lower_arithmetic_shift_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedArithmeticRightShiftCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast := 15, 22 + if target.goarch == "386" { + legacyLast, zLast = 7, 7 + } + var source strings.Builder + source.WriteString("TEXT packedarithmeticrightshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSRAW", "PSRAL"} { + // Go's Py3 MMX rows are named-source mode64 only. Their + // complete mode32 rejection controls are independently + // exercised by TestX86YMMMode32LegacyShiftActualGoRows. + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, M1\n", op) + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M1\n", op) + } + fmt.Fprintf(&source, "\t%s $-1, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSRAW", "VPSRAD", "VPSRAQ"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s $3, %s20, %s21\n", op, width, width) + fmt.Fprintf(&source, "\t%s $3, 8(AX), %s21\n", op, width) + fmt.Fprintf(&source, "\t%s X0, %s20, %s21\n", op, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20, %s21\n", op, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, %s20, K1, %s22\n", op, width, width) + fmt.Fprintf(&source, "\t%s.Z X0, %s20, K7, %s22\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s $3, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s $3, 8(AX), Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s X0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, Z20, K2, Z22\n", op) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z20, K6, Z22\n", op) + } + if op != "VPSRAW" { + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), X21\n", op) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Y21\n", op) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z $3, 8(AX), K5, Z22\n", op) + } + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedarithmeticrightshiftforms": {Name: "packedarithmeticrightshiftforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-arithmetic-right-shift-"+target.name+".ll", "packed-arithmetic-right-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedArithmeticRightShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSRAW.Z $3, X0", + "PSRAL $3", + "PSRAW M0, X1", + "PSRAL X0, M1", + "PSRAQ $3, X0", + "PSRAW $3, Y0", + "VPSRAW R8, X1, X2", + "VPSRAD Y0, X1, X2", + "VPSRAQ X0, 8(AX), X2", + "VPSRAW.BCST $3, 8(AX), X2", + "VPSRAD.BCST X0, X1, X2", + "VPSRAQ.BCST $3, X1, X2", + "VPSRAW.Z $3, X1, X2", + "VPSRAD $3, X1, K0, X2", + "VPSRAQ $3, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedArithmeticRightShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSRAW X8, X0", + "PSRAL X0, X8", + "VPSRAW $3, Z8, Z1", + "VPSRAD X0, Z1, Z8", + "VPSRAQ $3, X1, K1, X2", + "VPSRAW.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedArithmeticRightShiftRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedArithmeticRightShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed arithmetic-right-shift forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_atomic.go b/amd64_lower_atomic.go index c2783260b..ac38ea87c 100644 --- a/amd64_lower_atomic.go +++ b/amd64_lower_atomic.go @@ -3,240 +3,23 @@ package plan9asm import "fmt" func (c *amd64Ctx) lowerAtomic(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, terminated, err := c.lowerLockScalarRMW(op, ins); known { + return known, terminated, err + } + if spec, recognized := amd64ExchangeSpecs[Op(normalizeInstructionOpcode(op))]; recognized { + return c.lowerExchange(op, spec, ins) + } + if _, _, recognized := amd64CompareExchangeProperties(normalizeInstructionOpcode(op)); recognized { + return c.lowerCompareExchange(op, ins) + } switch op { case "LOCK": - // LOCK is a prefix in Plan 9 syntax. Our lowering emits atomic IR for - // the following memory RMW instruction, so the prefix itself is a no-op. - return true, false, nil - - case "CMPXCHG8B": - if len(ins.Args) != 1 || ins.Args[0].Kind != OpMem { - return true, false, fmt.Errorf("amd64 CMPXCHG8B expects mem: %q", ins.Raw) - } - word := func(r Reg) (string, error) { - return c.evalIntSized(Operand{Kind: OpReg, Reg: r}, I32) - } - ax, err := word(AX) - if err != nil { - return true, false, err - } - dx, err := word(DX) - if err != nil { - return true, false, err - } - bx, err := word(BX) - if err != nil { - return true, false, err - } - cx, err := word(CX) - if err != nil { - return true, false, err - } - pair := func(lo, hi string) string { - lo64 := c.newTmp() - hi64 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", lo64, lo) - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", hi64, hi) - shifted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 32\n", shifted, hi64) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", out, shifted, lo64) - return "%" + out - } - expected := pair(ax, dx) - desired := pair(bx, cx) - ptr, ptrType, err := c.ptrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - if ptrType != "ptr" { - return true, false, fmt.Errorf("amd64 CMPXCHG8B does not support segment-relative memory: %q", ins.Raw) - } - result := c.newTmp() - fmt.Fprintf(c.b, " %%%s = cmpxchg ptr %s, i64 %s, i64 %s seq_cst seq_cst, align 8\n", result, ptr, expected, desired) - old := c.newTmp() - success := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i1 } %%%s, 0\n", old, result) - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i1 } %%%s, 1\n", success, result) - oldLo := c.newTmp() - oldHiShift := c.newTmp() - oldHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", oldLo, old) - fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, 32\n", oldHiShift, old) - fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", oldHi, oldHiShift) - if err := c.storeRegSized(AX, I32, "%"+oldLo); err != nil { - return true, false, err - } - if err := c.storeRegSized(DX, I32, "%"+oldHi); err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", success, c.flagsZSlot) - return true, false, nil - - case "CMPXCHGL", "CMPXCHGQ": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpMem { - return true, false, fmt.Errorf("amd64 %s expects src, mem: %q", op, ins.Raw) - } - ty := I32 - align := 4 - if op == "CMPXCHGQ" { - ty = I64 - align = 8 - } - - expAX, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - exp, err := c.amd64AtomicTruncFromI64(expAX, ty) - if err != nil { - return true, false, err - } - newv64, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - newv, err := c.amd64AtomicTruncFromI64(newv64, ty) - if err != nil { - return true, false, err - } - ptr, err := c.amd64AtomicPtrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - cx := c.newTmp() - fmt.Fprintf(c.b, " %%%s = cmpxchg ptr %s, %s %s, %s %s seq_cst seq_cst, align %d\n", cx, ptr, ty, exp, ty, newv, align) - old := c.newTmp() - okv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue {%s, i1} %%%s, 0\n", old, ty, cx) - fmt.Fprintf(c.b, " %%%s = extractvalue {%s, i1} %%%s, 1\n", okv, ty, cx) - old64, err := c.amd64AtomicExtendToI64("%"+old, ty) - if err != nil { - return true, false, err - } - if err := c.storeReg(AX, old64); err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", okv, c.flagsZSlot) - return true, false, nil - - case "XADDL", "XADDQ": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpMem { - return true, false, fmt.Errorf("amd64 %s expects srcReg, mem: %q", op, ins.Raw) - } - srcReg := ins.Args[0] - if srcReg.Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects register src: %q", op, ins.Raw) - } - ty := I32 - if op == "XADDQ" { - ty = I64 - } - src64, err := c.loadReg(srcReg.Reg) - if err != nil { - return true, false, err - } - src, err := c.amd64AtomicTruncFromI64(src64, ty) - if err != nil { - return true, false, err - } - ptr, err := c.amd64AtomicPtrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - old := c.newTmp() - fmt.Fprintf(c.b, " %%%s = atomicrmw add ptr %s, %s %s seq_cst\n", old, ptr, ty, src) - old64, err := c.amd64AtomicExtendToI64("%"+old, ty) - if err != nil { - return true, false, err - } - return true, false, c.storeReg(srcReg.Reg, old64) - - case "XCHGB", "XCHGL", "XCHGQ": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects srcReg, dst: %q", op, ins.Raw) - } - srcReg := ins.Args[0] - if srcReg.Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects register src: %q", op, ins.Raw) - } - ty := I8 - switch op { - case "XCHGL": - ty = I32 - case "XCHGQ": - ty = I64 - } - src64, err := c.loadReg(srcReg.Reg) - if err != nil { - return true, false, err - } - if ins.Args[1].Kind == OpReg { - dstReg := ins.Args[1].Reg - dst64, err := c.loadReg(dstReg) - if err != nil { - return true, false, err - } - switch ty { - case I64: - if err := c.storeReg(srcReg.Reg, dst64); err != nil { - return true, false, err - } - return true, false, c.storeReg(dstReg, src64) - case I32: - s32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", s32, src64) - d32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", d32, dst64) - sz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", sz, s32) - dz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", dz, d32) - if err := c.storeReg(srcReg.Reg, "%"+dz); err != nil { - return true, false, err - } - return true, false, c.storeReg(dstReg, "%"+sz) - default: - s8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", s8, src64) - d8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", d8, dst64) - sz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", sz, s8) - dz := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", dz, d8) - if err := c.storeReg(srcReg.Reg, "%"+dz); err != nil { - return true, false, err - } - return true, false, c.storeReg(dstReg, "%"+sz) - } - } - if ins.Args[1].Kind != OpMem && ins.Args[1].Kind != OpSym { - return true, false, fmt.Errorf("amd64 %s expects srcReg, mem/reg: %q", op, ins.Raw) - } - src, err := c.amd64AtomicTruncFromI64(src64, ty) - if err != nil { - return true, false, err - } - ptr := "" - if ins.Args[1].Kind == OpMem { - ptr, err = c.amd64AtomicPtrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - } else { - ptr, err = c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - } - old := c.newTmp() - fmt.Fprintf(c.b, " %%%s = atomicrmw xchg ptr %s, %s %s seq_cst\n", old, ptr, ty, src) - old64, err := c.amd64AtomicExtendToI64("%"+old, ty) - if err != nil { - return true, false, err + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%w: LOCK operands cannot substitute for a complete following atomic instruction: %q", ErrProbeNeedsContext, ins.Raw) } - return true, false, c.storeReg(srcReg.Reg, old64) + // Only the whole-source consumer can bind this prefix to an adjacent + // complete native operation. An isolated helper has no such body. + return true, false, fmt.Errorf("%w: standalone LOCK has no source-adjacent complete atomic operation", ErrProbeNeedsContext) case "ORB", "ANDB", "ORL", "ANDL", "ORQ", "ANDQ": if len(ins.Args) != 2 || ins.Args[1].Kind != OpMem { diff --git a/amd64_lower_average.go b/amd64_lower_average.go new file mode 100644 index 000000000..5d8076bc1 --- /dev/null +++ b/amd64_lower_average.go @@ -0,0 +1,117 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerPackedUnsignedAverage(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedAverageProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + ins.Op = op + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + // A virtual FP/SP/BP view is not the source's native aligned m128. + // This family does not add a caller or fixed-local backing permission. + virtual := first.Kind == OpFP + if first.Kind == OpMem { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed average m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.spec.laneBits + lanes := form.byteWidth * 8 / bits + if form.mmx { + var a string + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + // Exact m64 bytes, including independent logical FP fields. Native + // reads stay observable without inventing an x87/tag-state bridge. + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedUnsignedAverage(lanes, bits, c.bitcastI64ToIntegerLanes(lanes, bits, a), c.bitcastI64ToIntegerLanes(lanes, bits, b)) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, lanes, bits, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + var mask, value string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.legacy && form.memory >= 0 { + mnemonic := map[int]string{8: "pavgb", 16: "pavgw"}[bits] + // Keep the original single native m128 instruction, alignment fault + // and segment. Portable arithmetic must not erase these effects. + value, err = c.emitPackedBinaryNativeMemory(first, bits, b, mnemonic) + } else { + // E4.nb memory suppresses inactive byte/word faults. Active loads stay + // volatile if the result is dead; virtual FP needs its full span first. + a, loadErr := c.loadX86PackedShiftLanes(first, form.byteWidth, bits, false, mask) + err = loadErr + if err == nil { + value = c.emitPackedUnsignedAverage(lanes, bits, a, b) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + } + value = amd64ApplyIntegerLaneMask(c, lanes, bits, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, bits, value, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedUnsignedAverage(lanes, laneBits int, first, second string) string { + narrowType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + wideBits := laneBits * 2 + wideType := fmt.Sprintf("<%d x i%d>", lanes, wideBits) + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", firstWide, narrowType, first, wideType) + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", secondWide, narrowType, second, wideType) + sum := c.newTmp() + rounded := c.newTmp() + average := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", sum, wideType, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %s\n", rounded, wideType, sum, llvmSplatSignedInteger(lanes, wideBits, 1)) + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %s\n", average, wideType, rounded, llvmSplatSignedInteger(lanes, wideBits, 1)) + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", result, wideType, average, narrowType) + return "%" + result +} diff --git a/amd64_lower_average_test.go b/amd64_lower_average_test.go new file mode 100644 index 000000000..b0dc20dc5 --- /dev/null +++ b/amd64_lower_average_test.go @@ -0,0 +1,124 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedUnsignedAverageCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast := 15, 22 + if target.goarch == "386" { + legacyLast, zLast = 7, 7 + } + var source strings.Builder + source.WriteString("TEXT packedunsignedaverageforms(SB),4,$0-0\n") + for _, stem := range []string{"PAVGB", "PAVGW"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M1\n", stem) + fmt.Fprintf(&source, "\t%s 8(AX), M2\n", stem) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", stem, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", stem, legacyLast) + vector := "V" + stem + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s6, %s7\n", vector, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 8(AX), %s21, K7, %s22\n", vector, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", vector, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", vector, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K2, Z22\n", vector) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K6, Z22\n", vector) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, target.goarch, source.String(), "", true) + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedunsignedaverageforms": {Name: "packedunsignedaverageforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-unsigned-average-"+target.name+".ll", "packed-unsigned-average-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedUnsignedAverageRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PAVGB.Z X0, X1", + "PAVGW X0", + "PAVGB M0, X1", + "PAVGW X0, M1", + "PAVGB Y0, Y1", + "PAVGW X0, (AX)", + "VPAVGB X0, X1", + "VPAVGW X0, Y1, Y2", + "VPAVGB X0, X1, (AX)", + "VPAVGW.BCST 8(AX), X1, X2", + "VPAVGB.Z X0, X1, X2", + "VPAVGW X0, X1, K0, X2", + "VPAVGB X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedAverageRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PAVGB X8, X0", + "PAVGW X0, X8", + "VPAVGB Z8, Z1, Z2", + "VPAVGW Z0, Z1, Z8", + "VPAVGB X0, X1, K1, X2", + "VPAVGW.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedAverageRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedUnsignedAverageRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed unsigned-average forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_bextr.go b/amd64_lower_bextr.go new file mode 100644 index 000000000..a21afc9cd --- /dev/null +++ b/amd64_lower_bextr.go @@ -0,0 +1,85 @@ +package plan9asm + +import "fmt" + +// lowerBEXTR implements Go 1.27's complete _ybextrl operand row for BEXTRL/Q: +// register control, register/memory source, and register destination. +func (c *amd64Ctx) lowerBEXTR(op Op, ins Instr) (ok bool, terminated bool, err error) { + bits := 0 + switch op { + case "BEXTRL": + bits = 32 + case "BEXTRQ": + bits = 64 + default: + return false, false, nil + } + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects register control, register/memory source, and register destination: %q", c.goarch, op, ins.Raw) + } + control, source, destination := ins.Args[0], ins.Args[1], ins.Args[2] + if control.Kind != OpReg || !isX86YrlRegisterForArch(control.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s control is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) + } + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + controlValue, err := c.evalIntSized(control, typ) + if err != nil { + return true, false, err + } + sourceValue, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + start := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, 255\n", start, typ, controlValue) + lengthShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, 8\n", lengthShift, typ, controlValue) + length := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 255\n", length, typ, lengthShift) + startOK := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %d\n", startOK, typ, start, bits) + safeStart := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %d\n", safeStart, startOK, typ, start, typ, bits-1) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", shifted, typ, sourceValue, safeStart) + rawLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", rawLength, startOK, typ, length, typ) + remaining := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %d, %%%s\n", remaining, typ, bits, safeStart) + useRawLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %%%s\n", useRawLength, typ, rawLength, remaining) + effectiveLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %%%s\n", effectiveLength, useRawLength, typ, rawLength, typ, remaining) + isFullWidth := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %d\n", isFullWidth, typ, effectiveLength, bits) + safeLength := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", safeLength, typ, effectiveLength, bits-1) + one := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", one, typ, safeLength) + partialMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, -1\n", partialMask, typ, one) + mask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s -1, %s %%%s\n", mask, isFullWidth, typ, typ, partialMask) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", result, typ, shifted, mask) + if err := c.storeRegSized(destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + return true, false, nil +} diff --git a/amd64_lower_bextr_test.go b/amd64_lower_bextr_test.go new file mode 100644 index 000000000..92dd4f625 --- /dev/null +++ b/amd64_lower_bextr_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86BEXTRCompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA bextrdata+0(SB)/8, $1\n") + source.WriteString("GLOBL bextrdata(SB), $8\n") + source.WriteString("TEXT bextrforms(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + op := "BEXTR" + width + fmt.Fprintf(&source, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&source, "\t%s DX, 8(BX), SI\n", op) + fmt.Fprintf(&source, "\t%s CX, bextrdata(SB), DX\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12, R13\n", op) + } else { + fmt.Fprintf(&source, "\t%s AX, BX, SP\n", op) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86BEXTRCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86BEXTRCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"bextrforms": {Name: "bextrforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"lshr i32", "lshr i64", `"target-features"="+bmi"`} { + if !strings.Contains(ir, want) { + t.Fatalf("BEXTR lowering omitted %q:\n%s", want, ir) + } + } + if strings.Contains(ir, "+bmi2") { + t.Fatalf("BEXTR is BMI1 but inferred BMI2:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "bextr-"+target.name+".ll", "bextr-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86BEXTRRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "BEXTRL $1, AX, BX"}, + {goarch: "amd64", instruction: "BEXTRQ 0(AX), BX, CX"}, + {goarch: "amd64", instruction: "BEXTRQ AX, $1, CX"}, + {goarch: "amd64", instruction: "BEXTRL AX, BX, 0(CX)"}, + {goarch: "amd64", instruction: "BEXTRQ X0, AX, BX"}, + {goarch: "amd64", instruction: "BEXTRQ AX, X0, BX"}, + {goarch: "amd64", instruction: "BEXTRQ AX, BX, X0"}, + {goarch: "amd64", instruction: "BEXTRQ AX, BX"}, + {goarch: "amd64", instruction: "BEXTRQ.Z AX, BX, CX"}, + {goarch: "386", instruction: "BEXTRL R8, AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _ybextrl table", test.instruction) + } + }) + } +} + +func TestAMD64BEXTRRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT bextrsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVL $0x0804, AX + MOVL $0x1234, BX + BEXTRL AX, BX, CX + MOVQ CX, 0(DI) + SETCS 8(DI) + SETOS 9(DI) + SETEQ 10(DI) + + MOVL $0x0120, AX + MOVL $-1, BX + BEXTRL AX, BX, CX + MOVQ CX, 16(DI) + SETEQ 24(DI) + + MOVQ $0x083c, AX + MOVQ $-1, 32(DI) + BEXTRQ AX, 32(DI), CX + MOVQ CX, 40(DI) + + MOVQ $0x0404, AX + MOVQ $0xf0, BX + BEXTRQ AX, BX, AX + MOVQ AX, 48(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "bextrsemantics": { + Name: "bextrsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void bextrsemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[56] = {0}; + bextrsemantics(out); + if (load64(out + 0) != UINT64_C(0x23)) return 10; + if (out[8] != 0 || out[9] != 0 || out[10] != 0) return 11; + if (load64(out + 16) != 0 || out[24] != 1) return 12; + if (load64(out + 40) != UINT64_C(0xf)) return 13; + if (load64(out + 48) != UINT64_C(0xf)) return 14; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bextr_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_bf16_convert.go b/amd64_lower_bf16_convert.go new file mode 100644 index 000000000..da42e06a2 --- /dev/null +++ b/amd64_lower_bf16_convert.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64BF16ConvertSourceBytes = map[Op]int{ + "VCVTNEPS2BF16X": 16, + "VCVTNEPS2BF16Y": 32, + "VCVTNEPS2BF16": 64, +} + +func (c *amd64Ctx) lowerBF16Convert(op Op, ins Instr) (ok bool, terminated bool, err error) { + base := Op(strings.SplitN(string(op), ".", 2)[0]) + sourceBytes := amd64BF16ConvertSourceBytes[base] + if sourceBytes == 0 { + return false, false, nil + } + if !ins.x86Encoded { + return true, false, fmt.Errorf("%s has no named Go 1.27 encoder form: %q", base, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s expects source, [K mask,] destination: %q", base, ins.Raw) + } + broadcast := strings.Contains(string(op), ".BCST") + zeroing := strings.HasSuffix(string(op), ".Z") + masked := len(ins.Args) == 3 + if zeroing && !masked || broadcast && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s has invalid mask or broadcast operands: %q", base, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s requires K1-K7 for masking: %q", base, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + destinationBytes := sourceBytes / 2 + if destinationBytes < 16 { + destinationBytes = 16 + } + if destination.Kind != OpReg || amd64VectorByteWidth(destination.Reg) != destinationBytes { + return true, false, fmt.Errorf("%s requires a %d-byte destination register: %q", base, destinationBytes, ins.Raw) + } + source := ins.Args[0] + if source.Kind == OpReg && amd64VectorByteWidth(source.Reg) != sourceBytes || + source.Kind != OpReg && !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s requires a %d-byte vector or memory source: %q", base, sourceBytes, ins.Raw) + } + + lanes := sourceBytes / 4 + mask := "" + if masked { + mask, err = c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + } + inputBits, err := c.loadMaskedPackedCompareLanes(source, sourceBytes, 32, broadcast, mask) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x float>\n", input, lanes, inputBits, lanes) + resultLanes := lanes + if resultLanes < 8 { + resultLanes = 8 + } + intrinsic := map[int]string{ + 16: "llvm.x86.vcvtneps2bf16128", + 32: "llvm.x86.vcvtneps2bf16256", + 64: "llvm.x86.avx512bf16.cvtneps2bf16.512", + }[sourceBytes] + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <%d x bfloat> @%s(<%d x float> %%%s)\n", + converted, resultLanes, intrinsic, lanes, input) + computed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x bfloat> %%%s to <%d x i16>\n", + computed, resultLanes, converted, resultLanes) + result := "%" + computed + if lanes < resultLanes { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i16> %s, <%d x i16> poison, <%d x i32> <", + low, resultLanes, result, resultLanes, lanes) + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + c.b.WriteString(", ") + } + fmt.Fprintf(c.b, "i32 %d", lane) + } + c.b.WriteString(">\n") + result = "%" + low + } + if masked { + old, loadErr := c.loadPackedExtendInputs(destination, destinationBytes, lanes, 16) + if loadErr != nil { + return true, false, loadErr + } + result = amd64ApplyIntegerLaneMask(c, lanes, 16, result, old, mask, zeroing) + } + outputBytes := lanes * 2 + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x i8>\n", + bytesValue, lanes, result, outputBytes) + return true, false, c.storePackedHalfResult(destination, destinationBytes, outputBytes, "%"+bytesValue) +} diff --git a/amd64_lower_binary_float.go b/amd64_lower_binary_float.go new file mode 100644 index 000000000..1fdd02806 --- /dev/null +++ b/amd64_lower_binary_float.go @@ -0,0 +1,379 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64BinaryFloatingMode uint8 + +const ( + amd64BinaryFloatingAdd amd64BinaryFloatingMode = iota + amd64BinaryFloatingSub + amd64BinaryFloatingMul + amd64BinaryFloatingDiv + amd64BinaryFloatingMax + amd64BinaryFloatingMin + amd64BinaryFloatingSqrt + amd64BinaryFloatingScale +) + +type amd64BinaryFloatingSpec struct { + laneBits int + mode amd64BinaryFloatingMode + scalar bool + sae bool + rawOnly bool +} + +// amd64BinaryFloatingSpecs covers Go 1.27's complete V-prefixed binary +// floating-point family and the raw-only AVX-512 FP16 packed/scalar binary family. +// Packed Go forms use _yvaddpd, while scalar Go forms and VSQRTSS/VSQRTSD +// use _yvaddsd. SCALEF uses the same source/destination shape through +// _yvscalefpd/_yvgetexpsd. MAX/MIN enable SAE; arithmetic, square-root, and +// scale operations enable explicit rounding. +var amd64BinaryFloatingSpecs = map[Op]amd64BinaryFloatingSpec{ + "VADDPH": {laneBits: 16, mode: amd64BinaryFloatingAdd, rawOnly: true}, + "VSUBPH": {laneBits: 16, mode: amd64BinaryFloatingSub, rawOnly: true}, + "VMULPH": {laneBits: 16, mode: amd64BinaryFloatingMul, rawOnly: true}, + "VDIVPH": {laneBits: 16, mode: amd64BinaryFloatingDiv, rawOnly: true}, + "VMINPH": {laneBits: 16, mode: amd64BinaryFloatingMin, sae: true, rawOnly: true}, + "VMAXPH": {laneBits: 16, mode: amd64BinaryFloatingMax, sae: true, rawOnly: true}, + "VADDSH": {laneBits: 16, mode: amd64BinaryFloatingAdd, scalar: true, rawOnly: true}, + "VSUBSH": {laneBits: 16, mode: amd64BinaryFloatingSub, scalar: true, rawOnly: true}, + "VMULSH": {laneBits: 16, mode: amd64BinaryFloatingMul, scalar: true, rawOnly: true}, + "VDIVSH": {laneBits: 16, mode: amd64BinaryFloatingDiv, scalar: true, rawOnly: true}, + "VMINSH": {laneBits: 16, mode: amd64BinaryFloatingMin, scalar: true, sae: true, rawOnly: true}, + "VMAXSH": {laneBits: 16, mode: amd64BinaryFloatingMax, scalar: true, sae: true, rawOnly: true}, + "VADDPS": {laneBits: 32, mode: amd64BinaryFloatingAdd}, + "VADDPD": {laneBits: 64, mode: amd64BinaryFloatingAdd}, + "VADDSS": {laneBits: 32, mode: amd64BinaryFloatingAdd, scalar: true}, + "VADDSD": {laneBits: 64, mode: amd64BinaryFloatingAdd, scalar: true}, + "VSUBPS": {laneBits: 32, mode: amd64BinaryFloatingSub}, + "VSUBPD": {laneBits: 64, mode: amd64BinaryFloatingSub}, + "VSUBSS": {laneBits: 32, mode: amd64BinaryFloatingSub, scalar: true}, + "VSUBSD": {laneBits: 64, mode: amd64BinaryFloatingSub, scalar: true}, + "VMULPS": {laneBits: 32, mode: amd64BinaryFloatingMul}, + "VMULPD": {laneBits: 64, mode: amd64BinaryFloatingMul}, + "VMULSS": {laneBits: 32, mode: amd64BinaryFloatingMul, scalar: true}, + "VMULSD": {laneBits: 64, mode: amd64BinaryFloatingMul, scalar: true}, + "VDIVPS": {laneBits: 32, mode: amd64BinaryFloatingDiv}, + "VDIVPD": {laneBits: 64, mode: amd64BinaryFloatingDiv}, + "VDIVSS": {laneBits: 32, mode: amd64BinaryFloatingDiv, scalar: true}, + "VDIVSD": {laneBits: 64, mode: amd64BinaryFloatingDiv, scalar: true}, + "VMAXPS": {laneBits: 32, mode: amd64BinaryFloatingMax, sae: true}, + "VMAXPD": {laneBits: 64, mode: amd64BinaryFloatingMax, sae: true}, + "VMAXSS": {laneBits: 32, mode: amd64BinaryFloatingMax, scalar: true, sae: true}, + "VMAXSD": {laneBits: 64, mode: amd64BinaryFloatingMax, scalar: true, sae: true}, + "VMINPS": {laneBits: 32, mode: amd64BinaryFloatingMin, sae: true}, + "VMINPD": {laneBits: 64, mode: amd64BinaryFloatingMin, sae: true}, + "VMINSS": {laneBits: 32, mode: amd64BinaryFloatingMin, scalar: true, sae: true}, + "VMINSD": {laneBits: 64, mode: amd64BinaryFloatingMin, scalar: true, sae: true}, + "VSQRTSS": {laneBits: 32, mode: amd64BinaryFloatingSqrt, scalar: true}, + "VSQRTSD": {laneBits: 64, mode: amd64BinaryFloatingSqrt, scalar: true}, + "VSQRTSH": {laneBits: 16, mode: amd64BinaryFloatingSqrt, scalar: true, rawOnly: true}, + "VSCALEFPS": {laneBits: 32, mode: amd64BinaryFloatingScale}, + "VSCALEFPD": {laneBits: 64, mode: amd64BinaryFloatingScale}, + "VSCALEFSS": {laneBits: 32, mode: amd64BinaryFloatingScale, scalar: true}, + "VSCALEFSD": {laneBits: 64, mode: amd64BinaryFloatingScale, scalar: true}, +} + +type amd64BinaryFloatingSuffix struct { + broadcast bool + zeroing bool + rounding string + sae bool +} + +func parseAMD64BinaryFloatingSuffix(suffix string) (amd64BinaryFloatingSuffix, bool) { + switch suffix { + case "": + return amd64BinaryFloatingSuffix{}, true + case "Z": + return amd64BinaryFloatingSuffix{zeroing: true}, true + case "BCST": + return amd64BinaryFloatingSuffix{broadcast: true}, true + case "BCST.Z": + return amd64BinaryFloatingSuffix{broadcast: true, zeroing: true}, true + case "SAE": + return amd64BinaryFloatingSuffix{sae: true}, true + case "SAE.Z": + return amd64BinaryFloatingSuffix{sae: true, zeroing: true}, true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + return amd64BinaryFloatingSuffix{rounding: suffix}, true + case "RN_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", "RZ_SAE.Z": + return amd64BinaryFloatingSuffix{rounding: strings.TrimSuffix(suffix, ".Z"), zeroing: true}, true + default: + return amd64BinaryFloatingSuffix{}, false + } +} + +func (c *amd64Ctx) lowerBinaryFloating(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ok, term, err := c.lowerFloatMulDiv(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFloatAddSub(op, ins); ok { + return ok, term, err + } + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64BinaryFloatingSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if spec.rawOnly && !ins.x86Encoded { + return true, false, fmt.Errorf("%s has no named Go assembler form: %q", baseOp, ins.Raw) + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's binary floating optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && properties.broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if spec.sae { + if properties.rounding != "" { + return true, false, fmt.Errorf("%s %s supports SAE, not explicit rounding: %q", c.goarch, baseOp, ins.Raw) + } + } else if properties.sae { + return true, false, fmt.Errorf("%s %s supports explicit rounding, not SAE: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects src1, src2, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked && !spec.rawOnly { + return true, false, fmt.Errorf("386 %s mask forms exceed Go 1.27's assembler operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if spec.scalar { + if byteWidth != 16 { + return true, false, fmt.Errorf("%s %s scalar destination must be an X register: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed destination must be an X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoEVEXVectorRegister(destination, byteWidth) || !c.isGoEVEXVectorRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s second source and destination must be same-width Go 1.27 vector registers: %q", c.goarch, baseOp, ins.Raw) + } + if (properties.rounding != "" || properties.sae) && !spec.scalar && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed rounding/SAE is available only for Z registers: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast { + if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory first source: %q", c.goarch, baseOp, ins.Raw) + } + } else if ins.Args[0].Kind == OpReg { + if !c.isGoEVEXVectorRegister(ins.Args[0], byteWidth) { + return true, false, fmt.Errorf("%s %s first source must match the destination width and register class: %q", c.goarch, baseOp, ins.Raw) + } + } else if properties.rounding != "" || properties.sae { + return true, false, fmt.Errorf("%s %s explicit rounding/SAE requires a register first source: %q", c.goarch, baseOp, ins.Raw) + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s first source must be a vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + var mask string + if masked { + if !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + loadedMask, loadErr := c.loadK(ins.Args[2].Reg) + if loadErr != nil { + return true, false, loadErr + } + mask = loadedMask + } + if spec.scalar { + return c.lowerScalarBinaryFloating(spec, properties, ins, destination, mask) + } + return c.lowerPackedBinaryFloating(spec, properties, ins, destination, byteWidth, mask) +} + +func (c *amd64Ctx) lowerPackedBinaryFloating(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, ins Instr, destination Operand, byteWidth int, mask string) (bool, bool, error) { + if spec.mode == amd64BinaryFloatingScale { + return c.lowerPackedScaleF(spec, properties, ins.Args[0], ins.Args[1], destination, byteWidth, mask) + } + lanes := byteWidth * 8 / spec.laneBits + llvmType := amd64FMA3LLVMType(lanes, spec.laneBits) + load := func(operand Operand, allowBroadcast bool) (string, error) { + if allowBroadcast && properties.broadcast { + var scalar string + var err error + switch spec.laneBits { + case 16: + bits, loadErr := c.loadPackedBroadcastScalar(operand, 16, ins.x86Encoded) + if loadErr != nil { + return "", loadErr + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i16 %s to half\n", value, bits) + scalar = "%" + value + case 32: + scalar, err = c.evalF32(operand) + case 64: + scalar, err = c.evalF64(operand) + } + if err != nil { + return "", err + } + return c.splatFMA3Scalar(lanes, spec.laneBits, scalar), nil + } + bytesValue, err := c.loadPackedCompareBytes(operand, byteWidth) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, byteWidth, bytesValue, llvmType) + return "%" + value, nil + } + first, err := load(ins.Args[0], true) + if err != nil { + return true, false, err + } + second, err := load(ins.Args[1], false) + if err != nil { + return true, false, err + } + computed := c.emitBinaryFloating(spec, llvmType, lanes, second, first, properties.rounding) + if mask != "" { + computedBits := c.bitcastFloatingToIntegerLanes(llvmType, lanes, spec.laneBits, computed) + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + oldBits := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + masked := amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computedBits, oldBits, mask, properties.zeroing) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", back, lanes, spec.laneBits, masked, llvmType) + computed = "%" + back + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, llvmType, computed, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarBinaryFloating(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, ins Instr, destination Operand, mask string) (bool, bool, error) { + if spec.mode == amd64BinaryFloatingScale { + return c.lowerScalarScaleF(spec, properties, ins.Args[0], ins.Args[1], destination, mask) + } + var first, second string + var err error + if spec.laneBits == 16 { + var firstBits, secondBits string + if ins.Args[0].Kind == OpReg { + firstBits, err = c.loadXLowInteger(ins.Args[0].Reg, 16) + } else if mask != "" { + firstBits, err = c.loadVectorScalarMemory(ins.Args[0], 16, mask) + } else { + firstBits, err = c.evalIntSized(ins.Args[0], "i16") + } + if err == nil { + secondBits, err = c.loadXLowInteger(ins.Args[1].Reg, 16) + } + if err == nil { + firstTemp, secondTemp := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i16 %s to half\n", firstTemp, firstBits) + fmt.Fprintf(c.b, " %%%s = bitcast i16 %s to half\n", secondTemp, secondBits) + first, second = "%"+firstTemp, "%"+secondTemp + } + } else if spec.laneBits == 32 { + first, err = c.evalF32(ins.Args[0]) + if err == nil { + second, err = c.loadXLowF32(ins.Args[1].Reg) + } + } else { + first, err = c.evalF64(ins.Args[0]) + if err == nil { + second, err = c.loadXLowF64(ins.Args[1].Reg) + } + } + if err != nil { + return true, false, err + } + llvmType := amd64FMA3LLVMType(1, spec.laneBits) + computed := c.emitBinaryFloating(spec, llvmType, 1, second, first, properties.rounding) + computedBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", computedBits, llvmType, computed, spec.laneBits) + secondBytes, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, secondBytes) + if spec.rawOnly { + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.laneBits, "%"+computedBits, base, mask, properties.zeroing) + } + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+computedBits, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitBinaryFloating(spec amd64BinaryFloatingSpec, llvmType string, lanes int, left, right, rounding string) string { + result := c.newTmp() + switch spec.mode { + case amd64BinaryFloatingSqrt: + if rounding == "" { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.sqrt.%s(%s %s)\n", result, llvmType, amd64BinaryFloatingIntrinsicSuffix(lanes, spec.laneBits), llvmType, right) + } else { + intrinsic := fmt.Sprintf("llvm.experimental.constrained.sqrt.%s", amd64BinaryFloatingIntrinsicSuffix(lanes, spec.laneBits)) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", result, llvmType, intrinsic, llvmType, right, amd64FMA3RoundingMetadata(rounding)) + } + case amd64BinaryFloatingMax, amd64BinaryFloatingMin: + predicate := "ogt" + if spec.mode == amd64BinaryFloatingMin { + predicate = "olt" + } + comparison := c.newTmp() + predicateType := "i1" + if lanes > 1 { + predicateType = fmt.Sprintf("<%d x i1>", lanes) + } + fmt.Fprintf(c.b, " %%%s = fcmp %s %s %s, %s\n", comparison, predicate, llvmType, left, right) + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", result, predicateType, comparison, llvmType, left, llvmType, right) + default: + operation := "fadd" + switch spec.mode { + case amd64BinaryFloatingSub: + operation = "fsub" + case amd64BinaryFloatingMul: + operation = "fmul" + case amd64BinaryFloatingDiv: + operation = "fdiv" + } + if rounding == "" { + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", result, operation, llvmType, left, right) + } else { + intrinsic := fmt.Sprintf("llvm.experimental.constrained.%s.%s", operation, amd64BinaryFloatingIntrinsicSuffix(lanes, spec.laneBits)) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", result, llvmType, intrinsic, llvmType, left, llvmType, right, amd64FMA3RoundingMetadata(rounding)) + } + } + return "%" + result +} + +func (c *amd64Ctx) bitcastFloatingToIntegerLanes(llvmType string, lanes, laneBits int, value string) string { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i%d>\n", result, llvmType, value, lanes, laneBits) + return "%" + result +} + +func amd64BinaryFloatingIntrinsicSuffix(lanes, laneBits int) string { + suffix := "f32" + if laneBits == 16 { + suffix = "f16" + } else if laneBits == 64 { + suffix = "f64" + } + if lanes > 1 { + return fmt.Sprintf("v%d%s", lanes, suffix) + } + return suffix +} diff --git a/amd64_lower_bittest.go b/amd64_lower_bittest.go new file mode 100644 index 000000000..897ed806c --- /dev/null +++ b/amd64_lower_bittest.go @@ -0,0 +1,193 @@ +package plan9asm + +import "fmt" + +// lowerBitTestFamily implements the complete Go 1.27 ybtl table. BTW/L/Q, +// BTCW/L/Q, BTRW/L/Q, and BTSW/L/Q all accept exactly either a signed imm8 or +// a Yrl register bit index followed by a Yml register/memory destination. +func (c *amd64Ctx) lowerBitTestFamily(op Op, ins Instr) (ok bool, terminated bool, err error) { + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86BitTestProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + if form.destination.Kind != OpReg && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return true, false, fmt.Errorf("%w: bit string needs its original native address size", ErrProbeNeedsContext) + } + stem, bits := form.stem, form.bits + typ := amd64IntegerTypeForBits(bits) + bitIndex, memoryWordOffset, registerIndex, err := c.bitTestIndex(form.index, bits, typ) + if err != nil { + return true, false, err + } + value, store, err := c.loadBitTestDestination(form.destination, typ, memoryWordOffset, registerIndex) + if err != nil { + return true, false, err + } + + shifted := c.newTmp() + bit := c.newTmp() + carry := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", shifted, typ, value, bitIndex) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", bit, typ, shifted) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", carry, typ, bit) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + if stem == "BT" { + return true, false, nil + } + + bitMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %s\n", bitMask, typ, bitIndex) + result := c.newTmp() + switch stem { + case "BTC": + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", result, typ, value, bitMask) + case "BTR": + inverseMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %%%s, -1\n", inverseMask, typ, bitMask) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, value, inverseMask) + case "BTS": + fmt.Fprintf(c.b, " %%%s = or %s %s, %%%s\n", result, typ, value, bitMask) + } + return true, false, store("%" + result) +} + +func amd64BitTestProperties(op string) (stem string, bits int, ok bool) { + switch op { + case "BTW": + return "BT", 16, true + case "BTL": + return "BT", 32, true + case "BTQ": + return "BT", 64, true + case "BTCW": + return "BTC", 16, true + case "BTCL": + return "BTC", 32, true + case "BTCQ": + return "BTC", 64, true + case "BTRW": + return "BTR", 16, true + case "BTRL": + return "BTR", 32, true + case "BTRQ": + return "BTR", 64, true + case "BTSW": + return "BTS", 16, true + case "BTSL": + return "BTS", 32, true + case "BTSQ": + return "BTS", 64, true + } + return "", 0, false +} + +// bitTestIndex returns the within-word index and, for a register index, the +// signed byte offset used when a memory destination is treated as an unbounded +// bit string. The index register has the same width as the instruction. +// Intel SDM Vol.2A, Bit(BitBase, BitOffset) and BT/BTC/BTR/BTS: +// https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf +// Immediate high bits are ignored by the CPU; unlike some assemblers, actual +// Go does not fold them into a new displacement (tested for every imm8). +func (c *amd64Ctx) bitTestIndex(index Operand, bits int, typ LLVMType) (bitIndex, memoryWordOffset string, registerIndex bool, err error) { + if index.Kind == OpImm { + return fmt.Sprintf("%d", index.Imm&int64(bits-1)), "", false, nil + } + value, err := c.evalIntSized(index, typ) + if err != nil { + return "", "", false, err + } + withinWord := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %d\n", withinWord, typ, value, bits-1) + bitIndex = "%" + withinWord + + signedIndex := value + if bits != 64 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext %s %s to i64\n", extended, typ, value) + signedIndex = "%" + extended + } + wordIndex := c.newTmp() + byteOffset := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ashr i64 %s, %d\n", wordIndex, signedIndex, bitTestLog2(bits)) + fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %d\n", byteOffset, wordIndex, bits/8) + return bitIndex, "%" + byteOffset, true, nil +} + +func bitTestLog2(bits int) int { + switch bits { + case 16: + return 4 + case 32: + return 5 + default: + return 6 + } +} + +func (c *amd64Ctx) loadBitTestDestination(destination Operand, typ LLVMType, memoryWordOffset string, registerIndex bool) (string, func(string) error, error) { + if destination.Kind == OpReg { + return c.loadIntDestination(destination, typ) + } + if destination.Kind == OpFP { + if registerIndex { + return "", nil, fmt.Errorf("%w: a register-indexed FP bit string has no fixed logical byte span", ErrProbeNeedsContext) + } + return c.loadBitTestFPDestination(destination, typ) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return "", nil, err + } + if registerIndex { + adjusted := c.newTmp() + // Signed floor division selects the complete containing word. Keep + // the original pointer address space and its target-width wrapping. + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %s\n", adjusted, pointerType, pointer, memoryWordOffset) + pointer = "%" + adjusted + } + loaded := c.newTmp() + // Intel SDM Vol.2A BT/BTC/BTR/BTS: even a dead CF retains the complete + // native read/fault; modifying forms also retain their write, including + // when the selected bit already has the requested value. This is not LOCK. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", loaded, typ, pointerType, pointer) + return "%" + loaded, func(value string) error { + fmt.Fprintf(c.b, " store volatile %s %s, %s %s, align 1\n", typ, value, pointerType, pointer) + return nil + }, nil +} + +func (c *amd64Ctx) loadBitTestFPDestination(destination Operand, typ LLVMType) (string, func(string) error, error) { + bits, _ := amd64IntegerTypeBits(typ) + width := bits / 8 + if err := c.validateX86PackedShiftFPSpan(destination.FPOffset, width); err != nil { + return "", nil, err + } + result := false + for offset := destination.FPOffset; offset < destination.FPOffset+int64(width); offset++ { + slot, output, _ := c.fpVectorSlot(offset) + if slot.Type == Ptr || slot.Type == I1 { + return "", nil, fmt.Errorf("%w: FP bit string needs declared numerical bytes", ErrProbeNeedsContext) + } + result = result || output + } + if result { + if err := c.scalarFPResultInitialized(); err != nil { + return "", nil, err + } + } + bytes, err := c.loadX86PackedShiftBytes(destination, width, true) + if err != nil { + return "", nil, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, width, bytes, typ) + return "%" + value, func(value string) error { + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, typ, value, width) + return c.storeFPVectorBytes(destination.FPOffset, width, "%"+bytes, "") + }, nil +} diff --git a/amd64_lower_blend_imm.go b/amd64_lower_blend_imm.go new file mode 100644 index 000000000..634e8c364 --- /dev/null +++ b/amd64_lower_blend_imm.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerImmediatePackedBlend implements the complete Go 1.27 immediate +// packed-blend family. The legacy instructions use yxshuf, while all four +// V-prefixed instructions use _yvblendpd and therefore have VEX X/Y forms +// only (no EVEX Z, mask, zeroing, broadcast, or SAE forms). +func (c *amd64Ctx) lowerImmediatePackedBlend(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + laneBits, vector, recognized := amd64ImmediatePackedBlendProperties(baseOp) + if !recognized { + return false, false, nil + } + if suffix != "" { + return true, false, fmt.Errorf("%s %s has a suffix absent from its Go 1.27 optab: %q", c.goarch, baseOp, ins.Raw) + } + if !vector { + return c.lowerLegacyImmediatePackedBlend(baseOp, laneBits, ins) + } + return c.lowerVectorImmediatePackedBlend(baseOp, laneBits, ins) +} + +func amd64ImmediatePackedBlendProperties(op string) (laneBits int, vector, ok bool) { + switch op { + case "PBLENDW": + return 16, false, true + case "BLENDPS": + return 32, false, true + case "BLENDPD": + return 64, false, true + case "VPBLENDW": + return 16, true, true + case "VPBLENDD", "VBLENDPS": + return 32, true, true + case "VBLENDPD": + return 64, true, true + default: + return 0, false, false + } +} + +func (c *amd64Ctx) lowerLegacyImmediatePackedBlend(baseOp string, laneBits int, ins Instr) (bool, bool, error) { + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects $u8, X/m128, X: %q", c.goarch, baseOp, ins.Raw) + } + imm, ok := amd64UnsignedImm8(ins.Args[0]) + if !ok { + return true, false, fmt.Errorf("%s %s immediate is outside Go 1.27's unsigned-imm8 class: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[2].Kind != OpReg || !c.isGoLegacyXReg(ins.Args[2].Reg) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's legacy X class: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[1].Kind == OpReg { + if !c.isGoLegacyXReg(ins.Args[1].Reg) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's legacy X class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s source must be X or memory: %q", c.goarch, baseOp, ins.Raw) + } + firstBytes, err := c.loadPackedCompareBytes(ins.Args[1], 16) + if err != nil { + return true, false, err + } + secondBytes, err := c.loadX(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + result := c.emitImmediatePackedBlend(16, laneBits, imm, firstBytes, secondBytes) + return true, false, c.storeX(ins.Args[2].Reg, result) +} + +func (c *amd64Ctx) lowerVectorImmediatePackedBlend(baseOp string, laneBits int, ins Instr) (bool, bool, error) { + if c.goarch != "amd64" { + return true, false, fmt.Errorf("386 %s exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 4 { + return true, false, fmt.Errorf("amd64 %s expects $u8, X/Y memory-or-register, X/Y register, X/Y destination: %q", baseOp, ins.Raw) + } + imm, ok := amd64UnsignedImm8(ins.Args[0]) + if !ok { + return true, false, fmt.Errorf("amd64 %s immediate is outside Go 1.27's unsigned-imm8 class: %q", baseOp, ins.Raw) + } + dstArg := ins.Args[3] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects a vector-register destination: %q", baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(dstArg.Reg) + if byteWidth != 16 && byteWidth != 32 { + return true, false, fmt.Errorf("amd64 %s expects an X or Y destination: %q", baseOp, ins.Raw) + } + if !amd64VEXVectorRegister(dstArg, byteWidth) || !amd64VEXVectorRegister(ins.Args[2], byteWidth) { + return true, false, fmt.Errorf("amd64 %s second source and destination must be same-width VEX X/Y registers: %q", baseOp, ins.Raw) + } + if ins.Args[1].Kind == OpReg { + if !amd64VEXVectorRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("amd64 %s first source must match its destination's VEX width: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s first source must be a vector register or memory: %q", baseOp, ins.Raw) + } + firstBytes, err := c.loadPackedCompareBytes(ins.Args[1], byteWidth) + if err != nil { + return true, false, err + } + secondBytes, err := c.loadPackedCompareBytes(ins.Args[2], byteWidth) + if err != nil { + return true, false, err + } + result := c.emitImmediatePackedBlend(byteWidth, laneBits, imm, firstBytes, secondBytes) + return true, false, c.storeVectorBytes(dstArg.Reg, byteWidth, result) +} + +// emitImmediatePackedBlend selects the first (r/m) source when the immediate +// bit is one and the second source when it is zero. VPBLENDW repeats the eight +// immediate bits independently in each 128-bit lane of a Y register. +func (c *amd64Ctx) emitImmediatePackedBlend(byteWidth, laneBits int, imm uint8, firstBytes, secondBytes string) string { + lanes := byteWidth * 8 / laneBits + first := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, firstBytes) + second := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, secondBytes) + mask := make([]string, 0, lanes) + for lane := 0; lane < lanes; lane++ { + if imm&(1< %s, <%d x i%d> %s, <%d x i32> <%s>\n", + shuffled, lanes, laneBits, first, lanes, laneBits, second, lanes, strings.Join(mask, ", ")) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <%d x i8>\n", out, lanes, laneBits, shuffled, byteWidth) + return "%" + out +} diff --git a/amd64_lower_blend_mask.go b/amd64_lower_blend_mask.go new file mode 100644 index 000000000..31c0fda54 --- /dev/null +++ b/amd64_lower_blend_mask.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64MaskBlendSpec struct { + laneBits int + broadcast bool +} + +// amd64MaskBlendSpecs is the complete shared _yvblendmpd grammar for the +// floating-point and integer mask-blend families. +var amd64MaskBlendSpecs = map[Op]amd64MaskBlendSpec{ + "VBLENDMPS": {laneBits: 32, broadcast: true}, + "VBLENDMPD": {laneBits: 64, broadcast: true}, + "VPBLENDMB": {laneBits: 8}, + "VPBLENDMW": {laneBits: 16}, + "VPBLENDMD": {laneBits: 32, broadcast: true}, + "VPBLENDMQ": {laneBits: 64, broadcast: true}, +} + +// lowerMaskBlend implements Go 1.27's complete _yvblendmpd operand table for +// VBLENDM{PS,PD} and VPBLENDM{B,W,D,Q}. A set K bit selects the first Plan 9 +// (Intel r/m) source; an unset bit selects the second source, or zero for .Z. +func (c *amd64Ctx) lowerMaskBlend(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, ok := amd64MaskBlendSpecs[Op(baseOp)] + if !ok { + return false, false, nil + } + suffixes, err := amd64ParseTwoSourcePermuteSuffixes(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + if suffixes.broadcast && !spec.broadcast { + return true, false, fmt.Errorf("amd64 %s does not enable broadcast in Go 1.27: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("amd64 %s expects first source, second source, [K mask,] destination: %q", baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if masked && c.goarch == "386" { + return true, false, fmt.Errorf("386 %s masked form exceeds the Go assembler frontend's operand limit: %q", baseOp, ins.Raw) + } + if suffixes.zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s .Z requires a K1-K7 mask: %q", baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + byteWidth := 0 + if destination.Kind == OpReg { + byteWidth = amd64VectorByteWidth(destination.Reg) + } + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 %s expects an in-range X, Y, or Z destination: %q", baseOp, ins.Raw) + } + second := ins.Args[1] + if !c.isGoPackedVectorMoveRegister(second, byteWidth) { + return true, false, fmt.Errorf("amd64 %s second source must match the destination width: %q", baseOp, ins.Raw) + } + first := ins.Args[0] + if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("amd64 %s first source register must match the destination width: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s first source must be a matching vector register or memory: %q", baseOp, ins.Raw) + } + if suffixes.broadcast && !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory first source: %q", baseOp, ins.Raw) + } + + firstValue, err := c.loadPackedCompareLanes(first, byteWidth, spec.laneBits, suffixes.broadcast) + if err != nil { + return true, false, err + } + if !masked { + out := c.newTmp() + lanes := byteWidth * 8 / spec.laneBits + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, firstValue, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) + } + + maskArg := ins.Args[2] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + maskIndex, validMask := amd64ParseKReg(maskArg.Reg) + if !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err := c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + result := amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, firstValue, secondValue, mask, suffixes.zeroing) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} diff --git a/amd64_lower_blend_mask_test.go b/amd64_lower_blend_mask_test.go new file mode 100644 index 000000000..db5ae7e50 --- /dev/null +++ b/amd64_lower_blend_mask_test.go @@ -0,0 +1,233 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64MaskBlendGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64MaskBlendSpec{ + "VBLENDMPS": {laneBits: 32, broadcast: true}, + "VBLENDMPD": {laneBits: 64, broadcast: true}, + "VPBLENDMB": {laneBits: 8}, + "VPBLENDMW": {laneBits: 16}, + "VPBLENDMD": {laneBits: 32, broadcast: true}, + "VPBLENDMQ": {laneBits: 64, broadcast: true}, + } + if len(amd64MaskBlendSpecs) != len(expected) { + t.Fatalf("mask-blend grammar has %d entries, want %d", len(amd64MaskBlendSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64MaskBlendSpecs[op]; !ok { + t.Errorf("mask-blend grammar omitted %s", op) + } else if got != want { + t.Errorf("mask-blend grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86MaskBlendCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 shares _yvblendmpd across VBLENDM{PS,PD} and + // VPBLENDM{B,W,D,Q}: matching X/Y/Z sources and destination, optional + // K1-K7 selection mask, and .Z. PS/PD/D/Q additionally enable + // scalar-memory broadcast on the first Plan 9 source. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT maskblendforms(SB),$0-0\n") + for _, family := range []struct { + op string + broadcast bool + }{ + {op: "VBLENDMPS", broadcast: true}, + {op: "VBLENDMPD", broadcast: true}, + {op: "VPBLENDMB"}, + {op: "VPBLENDMW"}, + {op: "VPBLENDMD", broadcast: true}, + {op: "VPBLENDMQ", broadcast: true}, + } { + op := family.op + fmt.Fprintf(&source, "\t%s X1, X20, X21\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s Z1, Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K1, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z Y1, Y20, K2, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), Z20, K3, Z21\n", op) + } + if family.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 104(AX), X20, X21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 112(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 120(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 128(AX), Z20, K4, Z21\n", op) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"maskblendforms": {Name: "maskblendforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "mask-blend-"+target.name+".ll", "mask-blend-"+target.name+".o", ll) + }) + } +} + +func TestAMD64FloatingMaskBlendRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT floatingmaskblendsemantics(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VBLENDMPS Z0, Z1, K1, Z2 + VMOVDQU64 Z2, 0(AX) + VBLENDMPD.Z Z0, Z1, K1, Z2 + VMOVDQU64 Z2, 64(AX) + VBLENDMPS.BCST 0(BX), Z1, K1, Z2 + VMOVDQU64 Z2, 128(AX) + VBLENDMPD.BCST.Z 0(BX), Z1, K1, Z2 + VMOVDQU64 Z2, 192(AX) + STC + VBLENDMPS Z0, Z1, K1, Z2 + SETCS 256(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"floatingmaskblendsemantics": { + Name: "floatingmaskblendsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatingmaskblendsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +int main(void) { + uint32_t first32[16], second32[16]; uint8_t out[257] = {0}; + const uint64_t mask = UINT64_C(0xa55a); + for (int i=0;i<16;i++) { first32[i]=UINT32_C(0x7f000000)+i; second32[i]=UINT32_C(0x3f000000)+i; } + floatingmaskblendsemantics(out,(const uint8_t *)first32,(const uint8_t *)second32,mask); + uint32_t *ps=(uint32_t *)(out+0), *psbcst=(uint32_t *)(out+128); + for (int i=0;i<16;i++) { + uint32_t want=(mask>>i)&1 ? first32[i] : second32[i]; if (ps[i]!=want) return 10+i; + want=(mask>>i)&1 ? first32[0] : second32[i]; if (psbcst[i]!=want) return 30+i; + } + uint64_t first64[8], second64[8], pdzero[8], pdbczero[8]; + memcpy(first64,first32,64); memcpy(second64,second32,64); + memcpy(pdzero,out+64,64); memcpy(pdbczero,out+192,64); + for (int i=0;i<8;i++) { + uint64_t want=(mask>>i)&1 ? first64[i] : 0; if (pdzero[i]!=want) return 50+i; + want=(mask>>i)&1 ? first64[0] : 0; if (pdbczero[i]!=want) return 70+i; + } + if (out[256]!=1) return 90; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "floating_mask_blend_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86MaskBlendRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VBLENDMPS X0, Y1, Y2", + "VBLENDMPD X0, X1, K0, X2", + "VBLENDMPS.BCST X0, X1, X2", + "VBLENDMPD.RN_SAE X0, X1, X2", + "VPBLENDMQ X0, X1", + "VPBLENDMQ X0, Y1, Y2", + "VPBLENDMQ X0, (AX), X2", + "VPBLENDMQ X0, X1, K0, X2", + "VPBLENDMQ X0, X1, AX", + "VPBLENDMQ.Z X0, X1, X2", + "VPBLENDMQ.BCST X0, X1, X2", + "VPBLENDMB.BCST (AX), X1, X2", + "VPBLENDMW.BCST (AX), X1, X2", + "VPBLENDMQ.Z.BCST (AX), X1, K1, X2", + "VPBLENDMQ.RN_SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86MaskBlendRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\tVPBLENDMQ X0, X1, K1, X2\n\tRET\n", false) + assertX86MaskBlendRejected(t, "386", "i386-unknown-linux-gnu", "VPBLENDMQ X0, X1, K1, X2") + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\tVPBLENDMQ Z0, Z1, Z8\n\tRET\n", false) + assertX86MaskBlendRejected(t, "386", "i386-unknown-linux-gnu", "VPBLENDMQ Z0, Z1, Z8") +} + +func assertX86MaskBlendRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPBLENDM forms", instruction) + } +} diff --git a/amd64_lower_blend_variable.go b/amd64_lower_blend_variable.go new file mode 100644 index 000000000..c97292e1e --- /dev/null +++ b/amd64_lower_blend_variable.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64VariableBlendSpec struct { + laneBits int + vector bool + opcode int +} + +// Go's yblendvpd and _yvblendvpd differ only in implicit/explicit mask, +// destructive/non-destructive destination and the available vector widths. +// The VEX raw decoder consumes the same opcode table as named lowering. +var amd64VariableBlendSpecs = map[Op]amd64VariableBlendSpec{ + "BLENDVPS": {laneBits: 32, opcode: 0x14}, + "BLENDVPD": {laneBits: 64, opcode: 0x15}, + "PBLENDVB": {laneBits: 8, opcode: 0x10}, + "VBLENDVPS": {laneBits: 32, vector: true, opcode: 0x4a}, + "VBLENDVPD": {laneBits: 64, vector: true, opcode: 0x4b}, + "VPBLENDVB": {laneBits: 8, vector: true, opcode: 0x4c}, +} + +// lowerVariableBlend implements Go 1.27's complete yblendvpd and _yvblendvpd +// families. Each opcode tests the high bit of its mask lane: false selects the +// base/destination lane and true selects the register-or-memory source lane. +func (c *amd64Ctx) lowerVariableBlend(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64VariableBlendSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + laneBits, vectorForm := spec.laneBits, spec.vector + if rawOp != baseOp { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", baseOp, ins.Raw) + } + wantArgs := 3 + if vectorForm { + wantArgs = 4 + } + if len(ins.Args) != wantArgs { + return true, false, fmt.Errorf("amd64 %s expects mask, source, %s: %q", baseOp, map[bool]string{false: "X destination", true: "base, destination"}[vectorForm], ins.Raw) + } + + maskArg, source := ins.Args[0], ins.Args[1] + base, destination := ins.Args[2], ins.Args[2] + if vectorForm { + destination = ins.Args[3] + } + if maskArg.Kind != OpReg || base.Kind != OpReg || destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects vector mask/base/destination registers: %q", baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(base.Reg) + if !vectorForm { + if maskArg.Reg != Reg("X0") || byteWidth != 16 || destination.Reg != base.Reg || !c.isGoLegacyXReg(base.Reg) { + return true, false, fmt.Errorf("%s %s expects implicit X0 mask and an in-range X destination: %q", c.goarch, baseOp, ins.Raw) + } + } else { + if byteWidth != 16 && byteWidth != 32 { + return true, false, fmt.Errorf("amd64 %s only accepts X or Y forms: %q", baseOp, ins.Raw) + } + if !amd64VEXVectorRegister(maskArg, byteWidth) || !amd64VEXVectorRegister(base, byteWidth) || !amd64VEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 %s operands must be same-width VEX registers: %q", baseOp, ins.Raw) + } + } + if source.Kind == OpReg { + valid := false + if vectorForm { + valid = amd64VEXVectorRegister(source, byteWidth) + } else { + valid = c.isGoLegacyXReg(source.Reg) + } + if !valid { + return true, false, fmt.Errorf("%s %s source register has the wrong width or range: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("amd64 %s source must be a same-width vector register or memory: %q", baseOp, ins.Raw) + } + + maskBytes, err := c.loadPackedCompareBytes(maskArg, byteWidth) + if err != nil { + return true, false, err + } + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + baseBytes, err := c.loadPackedCompareBytes(base, byteWidth) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / laneBits + maskLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, maskBytes) + sourceLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, sourceBytes) + baseLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, baseBytes) + selectedMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i%d> %s, zeroinitializer\n", selectedMask, lanes, laneBits, maskLanes) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %s, <%d x i%d> %s\n", result, lanes, selectedMask, lanes, laneBits, sourceLanes, lanes, laneBits, baseLanes) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + if vectorForm { + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) + } + return true, false, c.storeLegacyXViews(destination.Reg, "%"+out) +} diff --git a/amd64_lower_blend_variable_test.go b/amd64_lower_blend_variable_test.go new file mode 100644 index 000000000..541be136a --- /dev/null +++ b/amd64_lower_blend_variable_test.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86VariableBlendCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's yblendvpd and _yvblendvpd tables are shared by the + // PS (dword lanes), PD (qword lanes), and PBLENDVB (byte lanes) variants. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + var src strings.Builder + src.WriteString("TEXT variableblendforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"BLENDVPS", "BLENDVPD", "PBLENDVB"} { + fmt.Fprintf(&src, "\t%s X0, (AX), X1\n", op) + fmt.Fprintf(&src, "\t%s X0, X2, X%d\n", op, legacyLast) + } + for _, op := range []string{"VBLENDVPS", "VBLENDVPD", "VPBLENDVB"} { + fmt.Fprintf(&src, "\t%s X0, (AX), X1, X2\n", op) + fmt.Fprintf(&src, "\t%s X12, X13, X14, X15\n", op) + fmt.Fprintf(&src, "\t%s Y0, 32(AX), Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s Y12, Y13, Y14, Y15\n", op) + } + src.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "variableblendforms": {Name: "variableblendforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "variable-blend-"+target.name+".ll", "variable-blend-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86VariableBlendRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "BLENDVPS X1, X2, X3", + "BLENDVPD X0, Y1, X2", + "PBLENDVB X0, X1, Y2", + "BLENDVPS.Z X0, X1, X2", + "VBLENDVPS X0, X1, X2", + "VBLENDVPD X0, X1, X2, X3, X4", + "VPBLENDVB (AX), X1, X2, X3", + "VBLENDVPS X0, Y1, X2, X3", + "VBLENDVPD Y0, Y1, X2, Y3", + "VPBLENDVB Y0, Y1, Y2, X3", + "VBLENDVPS.Z X0, X1, X2, X3", + "VBLENDVPD X16, X1, X2, X3", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86VariableBlendRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "BLENDVPS X0, X8, X1", + "BLENDVPD X0, X1, X8", + "PBLENDVB X8, X1, X2", + } { + assertX86VariableBlendRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86VariableBlendRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's variable blend tables", instruction) + } +} diff --git a/amd64_lower_bls.go b/amd64_lower_bls.go new file mode 100644 index 000000000..58f957b50 --- /dev/null +++ b/amd64_lower_bls.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerLowestSetBitManipulation implements Go 1.27's complete _yblsil table: +// BLSI, BLSMSK, and BLSR at L/Q widths, with a Yml register-or-memory source +// and a Yrl destination register. +func (c *amd64Ctx) lowerLowestSetBitManipulation(op Op, ins Instr) (ok bool, terminated bool, err error) { + name := strings.ToUpper(string(op)) + family := "" + bits := 0 + switch name { + case "BLSIL": + family, bits = "BLSI", 32 + case "BLSIQ": + family, bits = "BLSI", 64 + case "BLSMSKL": + family, bits = "BLSMSK", 32 + case "BLSMSKQ": + family, bits = "BLSMSK", 64 + case "BLSRL": + family, bits = "BLSR", 32 + case "BLSRQ": + family, bits = "BLSR", 64 + default: + return false, false, nil + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects source, destination register: %q", c.goarch, name, ins.Raw) + } + source, destination := ins.Args[0], ins.Args[1] + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, name, ins.Raw) + } + } else if source.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(source.Sym), "$") || !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, name, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, name, ins.Raw) + } + + // VEX.W is ignored for this family in 32-bit mode. Go 1.27 therefore + // accepts the Q spellings for 386 and emits the same 32-bit operation. + typ := I32 + if bits == 64 && c.goarch != "386" { + typ = I64 + } + value, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + minusOne := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %s, 1\n", minusOne, typ, value) + result := c.newTmp() + carry := "" + switch family { + case "BLSI": + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", negative, typ, value) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, value, negative) + carry = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %s, 0\n", carry, typ, value) + case "BLSMSK": + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", result, typ, value, minusOne) + carry = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", carry, typ, value) + case "BLSR": + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, value, minusOne) + carry = c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", carry, typ, value) + } + + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + if family == "BLSMSK" { + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsZSlot) + } else { + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + } + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + return true, false, c.storeRegSized(destination.Reg, typ, "%"+result) +} diff --git a/amd64_lower_bls_test.go b/amd64_lower_bls_test.go new file mode 100644 index 000000000..c2328460f --- /dev/null +++ b/amd64_lower_bls_test.go @@ -0,0 +1,374 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86RawBLSMSKReportedEncoding(t *testing.T) { + const source = `TEXT rawBLSMSK(SB), NOSPLIT, $0-0 + BYTE $0xc4 + BYTE $0xe2 + BYTE $0x88 + BYTE $0xf3 + BYTE $0xd2 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawBLSMSK": {Name: "rawBLSMSK", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("raw BLSMSK did not infer BMI1 target feature:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-blsmsk-reported.ll", "raw-blsmsk-reported.o", ir) +} + +func TestTranslateX86BLSCompleteGoAssemblerForms(t *testing.T) { + registers386 := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + registersAMD64 := append(append([]string{}, registers386...), "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + targets := []struct { + name string + goarch string + triple string + widths []string + registers []string + }{ + {name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", widths: []string{"L", "Q"}, registers: registers386}, + {name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", widths: []string{"L", "Q"}, registers: registers386}, + {name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", widths: []string{"L", "Q"}, registers: registersAMD64}, + {name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", widths: []string{"L", "Q"}, registers: registersAMD64}, + {name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", widths: []string{"L", "Q"}, registers: registersAMD64}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT blsforms(SB),$0-0\n") + for _, family := range []string{"BLSI", "BLSMSK", "BLSR"} { + for _, width := range target.widths { + op := family + width + for index, register := range target.registers { + other := target.registers[(index+5)%len(target.registers)] + fmt.Fprintf(&source, "\t%s %s, %s\n", op, register, other) + fmt.Fprintf(&source, "\t%s %s, %s\n", op, other, register) + } + fmt.Fprintf(&source, "\t%s 8(AX), DX\n", op) + fmt.Fprintf(&source, "\t%s 16(AX)(CX*4), BX\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"blsforms": {Name: "blsforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `"target-features"="+bmi"`) { + t.Fatalf("BLS family did not infer BMI1 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "bls-family.ll", "bls-family.o", ir) + }) + } +} + +func TestTranslateX86BLSRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "386", instruction: "BLSIL R8, AX"}, + {goarch: "386", instruction: "BLSIQ R8, AX"}, + {goarch: "386", instruction: "BLSIL AX, R8"}, + {goarch: "386", instruction: "BLSIQ AX, R8"}, + {goarch: "amd64", instruction: "BLSIW AX, DX"}, + {goarch: "amd64", instruction: "BLSIL AL, DX"}, + {goarch: "amd64", instruction: "BLSIL X0, DX"}, + {goarch: "amd64", instruction: "BLSIL $1, DX"}, + {goarch: "amd64", instruction: "BLSIL AX, 0(DX)"}, + {goarch: "amd64", instruction: "BLSIL AX"}, + {goarch: "amd64", instruction: "BLSIL AX, DX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: test.goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yblsil table", test.instruction) + } + }) + } +} + +func TestX86RawBLSCompleteRegisterFields(t *testing.T) { + families := []struct { + digit byte + name string + }{ + {digit: 1, name: "BLSR"}, + {digit: 2, name: "BLSMSK"}, + {digit: 3, name: "BLSI"}, + } + for _, mode := range []int{32, 64} { + registerCount := 8 + widths := []bool{false, true} + if mode == 64 { + registerCount = 16 + } + for _, family := range families { + for _, width64 := range widths { + for destinationNumber := 0; destinationNumber < registerCount; destinationNumber++ { + for sourceNumber := 0; sourceNumber < registerCount; sourceNumber++ { + vex1 := byte(0xe2) + if sourceNumber >= 8 { + vex1 &^= 0x20 + } + vex2 := byte((^destinationNumber & 15) << 3) + if width64 { + vex2 |= 0x80 + } + modRM := byte(0xc0 | int(family.digit)<<3 | sourceNumber&7) + code := []byte{0xc4, vex1, vex2, 0xf3, modRM} + instruction, length, ok, err := decodedX86BLSInstruction(code, mode) + if err != nil || !ok || length != len(code) { + t.Fatalf("mode=%d %x: instruction=%#v length=%d ok=%v err=%v", mode, code, instruction, length, ok, err) + } + width := "L" + if width64 && mode == 64 { + width = "Q" + } + source, _ := decodedX86GeneralRegister(sourceNumber) + destination, _ := decodedX86GeneralRegister(destinationNumber) + if instruction.Op != Op(family.name+width) || len(instruction.Args) != 2 || instruction.Args[0].Reg != source || instruction.Args[1].Reg != destination { + t.Fatalf("mode=%d %x decoded as %#v, want %s%s %s, %s", mode, code, instruction, family.name, width, source, destination) + } + } + } + } + } + } +} + +func TestX86RawBLSCompleteMemoryFormsAndTargets(t *testing.T) { + targets := []struct { + name string + goarch string + triple string + encodings [][]byte + }{ + { + name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", + encodings: [][]byte{ + {0xc4, 0xe2, 0x70, 0xf3, 0x18}, // BLSIL 0(AX), CX + {0xc4, 0xe2, 0x68, 0xf3, 0x54, 0x48, 0x08}, // BLSMSKL 8(AX)(CX*2), DX + {0xc4, 0xe2, 0x60, 0xf3, 0x0d, 0x78, 0x56, 0x34, 0x12}, // BLSRL 0x12345678, BX + }, + }, + { + name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", + encodings: [][]byte{ + {0xc4, 0xe2, 0x70, 0xf3, 0x18}, + {0xc4, 0xe2, 0x68, 0xf3, 0x54, 0x48, 0x08}, + {0xc4, 0xe2, 0x60, 0xf3, 0x0d, 0x78, 0x56, 0x34, 0x12}, + }, + }, + { + name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", + encodings: [][]byte{ + {0xc4, 0xe2, 0x88, 0xf3, 0xd2}, // reported BLSMSKQ DX, R14 + {0xc4, 0x82, 0x88, 0xf3, 0x5c, 0x88, 0x10}, // BLSIQ 16(R8)(R9*4), R14 + {0x64, 0xc4, 0xe2, 0x80, 0xf3, 0x4c, 0x48, 0x08}, // BLSRQ FS:8(AX)(CX*2), R15 + {0xc4, 0xe2, 0x80, 0xf3, 0x1c, 0x25, 0x78, 0x56, 0x34, 0x12}, // BLSIQ 0x12345678, R15 + }, + }, + { + name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", + encodings: [][]byte{ + {0xc4, 0xe2, 0x88, 0xf3, 0xd2}, + {0xc4, 0x82, 0x88, 0xf3, 0x5c, 0x88, 0x10}, + {0x64, 0xc4, 0xe2, 0x80, 0xf3, 0x4c, 0x48, 0x08}, + {0xc4, 0xe2, 0x80, 0xf3, 0x1c, 0x25, 0x78, 0x56, 0x34, 0x12}, + }, + }, + { + name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", + encodings: [][]byte{ + {0xc4, 0xe2, 0x88, 0xf3, 0xd2}, + {0xc4, 0x82, 0x88, 0xf3, 0x5c, 0x88, 0x10}, + {0x64, 0xc4, 0xe2, 0x80, 0xf3, 0x4c, 0x48, 0x08}, + {0xc4, 0xe2, 0x80, 0xf3, 0x1c, 0x25, 0x78, 0x56, 0x34, 0x12}, + }, + }, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, encoding := range target.encodings { + name := fmt.Sprintf("rawBLS_%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + source.WriteString("\tRET\n") + sigs[name] = FuncSig{Name: name, Ret: Void} + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-bls.ll", "raw-bls.o", ir) + }) + } +} + +func TestX86RawBLSRejectsInvalidOrLayoutDependentForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex_l", mode: 64, encoding: []byte{0xc4, 0xe2, 0x8c, 0xf3, 0xd2}}, + {name: "vex_prefix", mode: 64, encoding: []byte{0xc4, 0xe2, 0x89, 0xf3, 0xd2}}, + {name: "invalid_digit", mode: 64, encoding: []byte{0xc4, 0xe2, 0x88, 0xf3, 0xc2}}, + {name: "extended_destination_386", mode: 32, encoding: []byte{0xc4, 0xe2, 0x38, 0xf3, 0xd2}}, + {name: "address_size_override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe2, 0x88, 0xf3, 0x54, 0x48, 0x08}}, + {name: "rip_relative", mode: 64, encoding: []byte{0xc4, 0xe2, 0x88, 0xf3, 0x15, 0, 0, 0, 0}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86BLSInstruction(test.encoding, test.mode) + if test.name == "vex_l" || test.name == "vex_prefix" { + if ok { + t.Fatalf("reserved VEX encoding %x was recognized: %v", test.encoding, err) + } + return + } + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} + +func TestAMD64BLSRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT blssemantics(SB),$0-8 + MOVQ out+0(FP), DI + MOVQ $0, AX + BLSIQ AX, BX + MOVQ BX, 0(DI) + SETCS 48(DI) + SETEQ 49(DI) + SETMI 50(DI) + SETOS 51(DI) + MOVQ $0x8000000000000000, AX + BLSIQ AX, BX + MOVQ BX, 8(DI) + MOVQ $0, AX + BLSMSKQ AX, BX + MOVQ BX, 16(DI) + SETCS 52(DI) + SETEQ 53(DI) + SETMI 54(DI) + SETOS 55(DI) + MOVQ $10, AX + BLSRQ AX, BX + MOVQ BX, 24(DI) + SETCS 56(DI) + SETEQ 57(DI) + SETMI 58(DI) + SETOS 59(DI) + MOVL $0x80000000, AX + BLSIL AX, BX + MOVQ BX, 32(DI) + MOVL $0, AX + BLSMSKL AX, BX + MOVQ BX, 40(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "blssemantics": {Name: "blssemantics", Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void blssemantics(uint8_t *); +int main(void) { + uint8_t out[64] = {0}; + blssemantics(out); + const uint64_t want[6] = {0, UINT64_C(0x8000000000000000), UINT64_MAX, 8, UINT64_C(0x80000000), UINT64_C(0xffffffff)}; + for (int i = 0; i < 6; i++) { + uint64_t got; + memcpy(&got, out + i * 8, sizeof(got)); + if (got != want[i]) return 10 + i; + } + const uint8_t flags[12] = {0,1,0,0, 1,0,1,0, 0,0,0,0}; + for (int i = 0; i < 12; i++) if (out[48+i] != flags[i]) return 30+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bls_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_bmi2_shift.go b/amd64_lower_bmi2_shift.go new file mode 100644 index 000000000..0a5c5fed2 --- /dev/null +++ b/amd64_lower_bmi2_shift.go @@ -0,0 +1,74 @@ +package plan9asm + +import "fmt" + +type amd64BMI2ShiftSpec struct { + bits int + prefix int + kind string +} + +var amd64BMI2ShiftSpecs = map[string]amd64BMI2ShiftSpec{ + "SARXL": {bits: 32, prefix: 2, kind: "ashr"}, + "SARXQ": {bits: 64, prefix: 2, kind: "ashr"}, + "SHLXL": {bits: 32, prefix: 1, kind: "shl"}, + "SHLXQ": {bits: 64, prefix: 1, kind: "shl"}, + "SHRXL": {bits: 32, prefix: 3, kind: "lshr"}, + "SHRXQ": {bits: 64, prefix: 3, kind: "lshr"}, +} + +// lowerBMI2VariableShift implements the six SARX/SHLX/SHRX L/Q opcodes. Go +// 1.27 gives all of them the single _ybextrl row: Yrl count, Yml source, Yrl +// destination. Unlike legacy shifts, the count must be a register and flags +// are not modified. +func (c *amd64Ctx) lowerBMI2VariableShift(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, supported := amd64BMI2ShiftSpecs[string(op)] + if !supported { + return false, false, nil + } + if _, _, err := x86ScalarShiftProperties(op, ins, c.goarch); err != nil { + return true, false, err + } + bits := spec.bits + if c.goarch == "386" { + // Go accepts Q spellings, but VEX.W is ignored outside 64-bit mode. + // Both the memory access and the count mask must remain 32-bit. + bits = 32 + } + + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects count register, register/memory source, and destination register: %q", c.goarch, op, ins.Raw) + } + count, source, destination := ins.Args[0], ins.Args[1], ins.Args[2] + if count.Kind != OpReg || !isX86YrlRegisterForArch(count.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s count is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) + } + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + countValue, err := c.evalIntSized(count, typ) + if err != nil { + return true, false, err + } + sourceValue, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + maskedCount := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %d\n", maskedCount, typ, countValue, bits-1) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s, %%%s\n", result, spec.kind, typ, sourceValue, maskedCount) + if err := c.storeRegSized(destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + return true, false, nil +} diff --git a/amd64_lower_bmi2_shift_test.go b/amd64_lower_bmi2_shift_test.go new file mode 100644 index 000000000..f94d53c6c --- /dev/null +++ b/amd64_lower_bmi2_shift_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86BMI2VariableShiftFormsSource(goarch string) string { + var source strings.Builder + symbol := "bmi2shiftdata" + size := 8 + if goarch == "386" { + symbol = "bmi2shiftdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT bmi2shiftforms(SB),$0-0\n") + for _, stem := range []string{"SARX", "SHLX", "SHRX"} { + for _, width := range []string{"L", "Q"} { + op := stem + width + fmt.Fprintf(&source, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&source, "\t%s DX, 8(BX), SI\n", op) + fmt.Fprintf(&source, "\t%s CX, %s(SB), DX\n", op, symbol) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12, R13\n", op) + } + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86BMI2VariableShiftCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86BMI2VariableShiftFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"bmi2shiftforms": {Name: "bmi2shiftforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + operations := []string{"ashr i32", "shl i32", "lshr i32", `"target-features"="+bmi2"`} + if target.goarch == "amd64" { + operations = append(operations, "ashr i64", "shl i64", "lshr i64") + } else if strings.Contains(ll, "ashr i64") || strings.Contains(ll, "shl i64") || strings.Contains(ll, "lshr i64") { + t.Fatal("386 BMI2 shift used 64-bit arithmetic") + } + for _, want := range operations { + if !strings.Contains(ll, want) { + t.Fatalf("BMI2 variable-shift lowering omitted %q:\n%s", want, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "bmi2-shift-"+target.name+".ll", "bmi2-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86BMI2VariableShiftRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "SHLXQ $1, AX, BX"}, + {goarch: "amd64", instruction: "SHRXQ 0(BX), AX, CX"}, + {goarch: "amd64", instruction: "SARXL AX, BX, 0(CX)"}, + {goarch: "amd64", instruction: "SHLXL X0, AX, BX"}, + {goarch: "amd64", instruction: "SHRXQ AX, X0, BX"}, + {goarch: "amd64", instruction: "SARXQ AX, BX"}, + {goarch: "amd64", instruction: "SHLXQ.Z AX, BX, CX"}, + {goarch: "386", instruction: "SHLXL R8, AX, BX"}, + {goarch: "386", instruction: "SHRXQ AX, R8, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _ybextrl table", test.instruction) + } + }) + } +} + +func TestAMD64BMI2VariableShiftRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT bmi2shiftsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $36, AX + MOVQ $0x12345678, BX + SHLXL AX, BX, CX + MOVQ CX, 0(DI) + MOVQ $-81985529216486896, 8(DI) + MOVQ $72, AX + SHRXQ AX, 8(DI), CX + MOVQ CX, 8(DI) + MOVQ $0x80000000, BX + SARXL AX, BX, CX + MOVQ CX, 16(DI) + MOVQ $-9223372036854775808, BX + SARXQ AX, BX, CX + MOVQ CX, 24(DI) + MOVQ $1, AX + MOVQ $3, BX + CMPQ BX, BX + STC + SHLXQ AX, BX, AX + MOVQ AX, 32(DI) + SETCS 40(DI) + SETEQ 41(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "bmi2shiftsemantics": { + Name: "bmi2shiftsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void bmi2shiftsemantics(uint8_t *); +int main(void) { + uint8_t out[48] = {0}; + bmi2shiftsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0x23456780)) return 10; + if (*(uint64_t *)(out+8) != UINT64_C(0x00fedcba98765432)) return 11; + if (*(uint64_t *)(out+16) != UINT64_C(0xff800000)) return 12; + if (*(uint64_t *)(out+24) != UINT64_C(0xff80000000000000)) return 13; + if (*(uint64_t *)(out+32) != 6) return 14; + if (out[40] != 1 || out[41] != 1) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bmi2_shift_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_bound.go b/amd64_lower_bound.go new file mode 100644 index 000000000..bd5f55a4c --- /dev/null +++ b/amd64_lower_bound.go @@ -0,0 +1,63 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64BoundOperandBits = map[Op]int{ + "BOUNDW": 16, + "BOUNDL": 32, +} + +// lowerBound implements Go 1.27's complete yrl_m family. BOUND is valid only +// in 16/32-bit x86 modes, so plan9asm preserves the exact faulting instruction +// as constrained inline assembly rather than approximating its exception. +func (c *amd64Ctx) lowerBound(op Op, ins Instr) (ok bool, terminated bool, err error) { + raw := strings.ToUpper(string(op)) + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + bits, recognized := amd64BoundOperandBits[Op(base)] + if !recognized { + return false, false, nil + } + if raw != base { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, base, ins.Raw) + } + if c.goarch != "386" { + return true, false, fmt.Errorf("%s %s is unavailable outside 32-bit x86 mode: %q", c.goarch, base, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("386 %s expects GP register and bounds memory: %q", base, ins.Raw) + } + if ins.Args[0].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[0].Reg, c.goarch) { + return true, false, fmt.Errorf("386 %s source is outside Go 1.27's Yrl class: %q", base, ins.Raw) + } + if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s second operand must be Go 1.27 Ym memory: %q", base, ins.Raw) + } + typeName := I32 + if bits == 16 { + typeName = I16 + } + value, err := c.evalIntSized(ins.Args[0], typeName) + if err != nil { + return true, false, err + } + pointer, pointerType, err := c.x86DescriptorMemoryPointer(ins.Args[1]) + if err != nil { + return true, false, err + } + // LLVM 22's integrated assembler no longer recognizes the BOUND mnemonic, + // even for an i386 triple. Constrain the value and bounds pointer to AX/CX + // and emit 62 /r with ModRM 00:000:001. The 66 prefix selects BOUNDW. + encoding := ".byte 0x62, 0x01" + if bits == 16 { + encoding = ".byte 0x66, 0x62, 0x01" + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s, %s %s)\n", + encoding, "{ax},{cx},~{memory},~{dirflag},~{fpsr},~{flags}", typeName, value, pointerType, pointer) + return true, false, nil +} diff --git a/amd64_lower_bound_test.go b/amd64_lower_bound_test.go new file mode 100644 index 000000000..329d1d487 --- /dev/null +++ b/amd64_lower_bound_test.go @@ -0,0 +1,79 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslate386BoundCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + triple string + }{ + {name: "linux-386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := "TEXT boundforms(SB),NOSPLIT,$16-0\n" + for _, op := range []string{"BOUNDW", "BOUNDL"} { + for _, reg := range []string{"AX", "BX", "CX", "DX", "SP", "BP", "SI", "DI"} { + source += "\t" + op + " " + reg + ", 0(SP)\n" + } + source += "\t" + op + " AX, 8(BX)\n" + } + source += "\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "386", + Sigs: map[string]FuncSig{"boundforms": {Name: "boundforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{".byte 0x66, 0x62, 0x01", ".byte 0x62, 0x01", `"{ax},{cx},~{memory},~{dirflag},~{fpsr},~{flags}"`} { + if !strings.Contains(ll, want) { + t.Fatalf("translated BOUND family missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "bound-"+target.name+".ll", "bound-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86BoundRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "BOUNDW AX, 0(BX)"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "BOUNDL AX, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDL X0, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDW $1, 0(BX)"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDL AX, BX"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "BOUNDL.Z AX, 0(BX)"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+test.instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's BOUND tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_branch.go b/amd64_lower_branch.go index 345ec225a..33837c90f 100644 --- a/amd64_lower_branch.go +++ b/amd64_lower_branch.go @@ -34,6 +34,16 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi } return true, false, nil case OpSym: + if strings.HasPrefix(strings.TrimSpace(ins.Args[0].Sym), "*") { + addr, err := c.loadIndirectSymbolAddr(ins.Args[0].Sym) + if err != nil { + return true, false, err + } + if err := c.callIndirectAddr(addr); err != nil { + return true, false, err + } + return true, false, nil + } if err := c.callSym(ins.Args[0]); err != nil { return true, false, err } @@ -42,16 +52,22 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi return true, false, fmt.Errorf("amd64 CALL expects reg or symbol(SB) target: %q", ins.Raw) } - case "JMP", - "JE", "JEQ", "JZ", "JNE", "JNZ", - "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JS", "JNS", - "JA", "JHI", "JAE", "JHS", "JB", "JLO", "JBE", "JLS", "JNA", - "JC", "JNC", "JCC": - // ok default: - return false, false, nil + if op != "JMP" && op != "LOOP" && !isAMD64ConditionalBranch(op) { + return false, false, nil + } } - if len(ins.Args) != 1 { + args := ins.Args + // The Go assembler accepts a legacy x86 branch-hint operand such as + // JEQ $1, target. It is not part of the encoded instruction on current + // toolchains, so preserve control flow and ignore the hint value. + if op != "JMP" && op != "LOOP" && !isAMD64CounterZeroBranch(op) && len(args) == 2 && args[0].Kind == OpImm { + if args[0].Imm != 0 && args[0].Imm != 1 { + return true, false, fmt.Errorf("amd64 %s branch hint must be $0 or $1: %q", op, ins.Raw) + } + args = args[1:] + } + if len(args) != 1 { return true, false, fmt.Errorf("amd64 %s expects 1 operand: %q", op, ins.Raw) } target := "" @@ -65,19 +81,19 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi } return false } - switch ins.Args[0].Kind { + switch args[0].Kind { case OpIdent: - target = ins.Args[0].Ident + target = args[0].Ident case OpReg: // Labels like V1 may be tokenized as registers. Prefer block labels; // otherwise treat JMP reg as an indirect tail jump. - name := string(ins.Args[0].Reg) + name := string(args[0].Reg) if knownBlock(name) { target = name break } if op == "JMP" { - addr, err := c.loadReg(ins.Args[0].Reg) + addr, err := c.loadReg(args[0].Reg) if err != nil { return true, false, err } @@ -88,10 +104,20 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi } return true, false, fmt.Errorf("amd64 %s invalid register target: %q", op, ins.Raw) case OpSym: - s := strings.TrimSpace(ins.Args[0].Sym) + s := strings.TrimSpace(args[0].Sym) + if op == "JMP" && strings.HasPrefix(s, "*") { + addr, err := c.loadIndirectSymbolAddr(s) + if err != nil { + return true, false, err + } + if err := c.tailCallIndirectAddrAndRet(addr); err != nil { + return true, false, err + } + return true, true, nil + } // Treat JMP foo(SB) as a tailcall to another TEXT (common in stdlib asm). if op == "JMP" && strings.HasSuffix(s, "(SB)") { - if err := c.tailCallAndRet(ins.Args[0]); err != nil { + if err := c.tailCallAndRet(args[0]); err != nil { return true, false, err } return true, true, nil @@ -103,13 +129,13 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi // PC-relative branches like "JEQ 2(PC)" are used as a compact way to skip // the next instruction. Handle them by mapping the offset to our block // sequence (blocks are split at terminators, so the pattern works well). - if strings.EqualFold(string(ins.Args[0].Mem.Base), "PC") { + if strings.EqualFold(string(args[0].Mem.Base), "PC") { pcRel = true - pcOff = ins.Args[0].Mem.Off + pcOff = args[0].Mem.Off break } if op == "JMP" { - addr, err := c.addrFromMem(ins.Args[0].Mem) + addr, err := c.addrFromMem(args[0].Mem) if err != nil { return true, false, err } @@ -150,70 +176,102 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi fall := c.blocks[bi+1].name cond := "" switch op { + case "LOOP": + counter, err := c.loadReg(CX) + if err != nil { + return true, false, err + } + if c.goarch == "386" { + old32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", old32, counter) + next32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, 1\n", next32, old32) + next64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", next64, next32) + if err := c.storeReg(CX, "%"+next64); err != nil { + return true, false, err + } + test := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", test, next32) + cond = "%" + test + } else { + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %s, 1\n", next, counter) + if err := c.storeReg(CX, "%"+next); err != nil { + return true, false, err + } + test := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", test, next) + cond = "%" + test + } + case "JCXZW", "JCXZL", "JCXZQ": + // cmd/internal/obj/x86 emits an address-size override only for + // JCXZL. Therefore Go's historical spellings test these widths: + // + // JCXZW JCXZL JCXZQ + // amd64 RCX ECX RCX + // 386 ECX CX ECX + // + // Preserve that behavior instead of inferring the width from the + // mnemonic suffix. + counter, err := c.loadReg(CX) + if err != nil { + return true, false, err + } + bits := 64 + if c.goarch == "386" { + bits = 32 + } + if op == "JCXZL" { + bits /= 2 + } + value := counter + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", truncated, counter, bits) + value = "%" + truncated + } + test := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %s, 0\n", test, bits, value) + cond = "%" + test case "JE", "JEQ", "JZ": - cond = c.loadFlag(c.flagsZSlot) + cond, err = c.x86Condition("EQ") case "JNE", "JNZ": - z := c.loadFlag(c.flagsZSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, z) - cond = "%" + t - case "JL", "JLT": - cond = c.loadFlag(c.flagsSltSlot) - case "JGE": - slt := c.loadFlag(c.flagsSltSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, slt) - cond = "%" + t - case "JS": - cond = c.loadFlag(c.flagsSltSlot) - case "JNS": - slt := c.loadFlag(c.flagsSltSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, slt) - cond = "%" + t - case "JLE": - slt := c.loadFlag(c.flagsSltSlot) - z := c.loadFlag(c.flagsZSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t, slt, z) - cond = "%" + t - case "JG", "JGT": - slt := c.loadFlag(c.flagsSltSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, slt, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 - case "JB", "JLO", "JC": - cond = c.loadFlag(c.flagsCFSlot) - case "JNC", "JCC": - cf := c.loadFlag(c.flagsCFSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, cf) - cond = "%" + t - case "JAE", "JHS": - cf := c.loadFlag(c.flagsCFSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", t, cf) - cond = "%" + t + cond, err = c.x86Condition("NE") + case "JL", "JLT", "JNGE": + cond, err = c.x86Condition("LT") + case "JGE", "JNL": + cond, err = c.x86Condition("GE") + case "JS", "JMI": + cond, err = c.x86Condition("MI") + case "JNS", "JPL": + cond, err = c.x86Condition("PL") + case "JLE", "JNG": + cond, err = c.x86Condition("LE") + case "JG", "JGT", "JNLE": + cond, err = c.x86Condition("GT") + case "JB", "JLO", "JC", "JCS", "JNAE": + cond, err = c.x86Condition("CS") + case "JNC", "JCC", "JAE", "JHS", "JNB": + cond, err = c.x86Condition("CC") case "JBE", "JLS", "JNA": - cf := c.loadFlag(c.flagsCFSlot) - z := c.loadFlag(c.flagsZSlot) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t, cf, z) - cond = "%" + t - case "JA", "JHI": - cf := c.loadFlag(c.flagsCFSlot) - z := c.loadFlag(c.flagsZSlot) - t1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", t1, cf, z) - t2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", t2, t1) - cond = "%" + t2 + cond, err = c.x86Condition("LS") + case "JA", "JHI", "JNBE": + cond, err = c.x86Condition("HI") + case "JO", "JOS": + cond, err = c.x86Condition("OS") + case "JNO", "JOC": + cond, err = c.x86Condition("OC") + case "JP", "JPE", "JPS": + cond, err = c.x86Condition("PS") + case "JNP", "JPO", "JPC": + cond, err = c.x86Condition("PC") default: return true, false, fmt.Errorf("amd64: unsupported branch %s", op) } + if err != nil { + return true, false, err + } if err := emitCondBr(cond, target, fall); err != nil { return true, false, err @@ -221,9 +279,42 @@ func (c *amd64Ctx) lowerBranch(bi int, ii int, op Op, ins Instr, emitBr amd64Emi return true, true, nil } +func (c *amd64Ctx) loadIndirectSymbolAddr(sym string) (string, error) { + sym = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(sym), "*")) + if !strings.HasSuffix(sym, "(SB)") { + return "", fmt.Errorf("amd64 indirect branch expects register, memory, or *symbol(SB), got %q", sym) + } + ptr, err := c.ptrFromSB(sym) + if err != nil { + return "", err + } + value := c.newTmp() + if c.goarch == "386" { + fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", value, ptr) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, value) + return "%" + wide, nil + } + fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", value, ptr) + return "%" + value, nil +} + func (c *amd64Ctx) callIndirectAddr(addr string) error { fptr := c.newTmp() fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", fptr, addr) + if sig := c.indirectABI0; sig != nil { + args, err := c.abi0CallArgs("indirect callback", *sig) + if err != nil { + return err + } + if sig.Ret == Void { + fmt.Fprintf(c.b, " call void %%%s(%s)\n", fptr, strings.Join(args, ", ")) + return nil + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s %%%s(%s)\n", result, sig.Ret, fptr, strings.Join(args, ", ")) + return c.storeABI0CallResult("indirect callback", *sig, "%"+result) + } di, _ := c.loadReg(DI) si, _ := c.loadReg(SI) dx, _ := c.loadReg(DX) @@ -237,6 +328,15 @@ func (c *amd64Ctx) callIndirectAddr(addr string) error { } func (c *amd64Ctx) tailCallIndirectAddrAndRet(addr string) error { + if len(c.indirectLabels) != 0 { + ptr := c.ptrFromAddrI64(addr) + labels := make([]string, len(c.indirectLabels)) + for i, label := range c.indirectLabels { + labels[i] = "label %" + amd64LLVMBlockName(label) + } + fmt.Fprintf(c.b, " indirectbr ptr %s, [%s]\n", ptr, strings.Join(labels, ", ")) + return nil + } if err := c.callIndirectAddr(addr); err != nil { return err } @@ -294,6 +394,7 @@ func (c *amd64Ctx) callSym(symOp Operand) error { if !strings.HasSuffix(s, "(SB)") { return fmt.Errorf("amd64 call expects (SB) symbol, got %q", s) } + internalABI := strings.HasSuffix(strings.TrimSuffix(s, "(SB)"), "") s = strings.TrimSuffix(s, "(SB)") callee := c.resolve(s) // Syscall stubs invoke runtime entersyscall/exitsyscall around SYSCALL. @@ -310,6 +411,25 @@ func (c *amd64Ctx) callSym(symOp Operand) error { } callee = funcSigSymbol(callee, csig) + stackABI := !internalABI && len(csig.ArgRegs) == 0 && + (len(csig.Frame.Params) != 0 || len(csig.Frame.Results) != 0) + if stackABI { + args, err := c.abi0CallArgs(callee, csig) + if err != nil { + return err + } + if csig.Ret == Void { + fmt.Fprintf(c.b, " call void %s(%s)\n", llvmGlobal(callee), strings.Join(args, ", ")) + return nil + } + t := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s %s(%s)\n", t, csig.Ret, llvmGlobal(callee), strings.Join(args, ", ")) + if len(csig.Frame.Results) != 0 { + return c.storeABI0CallResult(callee, csig, "%"+t) + } + return fmt.Errorf("amd64 call %q: ABI0 result %s has no frame slot", callee, csig.Ret) + } + args := make([]string, 0, len(csig.Args)) goABI := []Reg{AX, BX, CX, DI, SI, Reg("R8"), Reg("R9"), Reg("R10"), Reg("R11")} regCursor := 0 @@ -418,12 +538,13 @@ func (c *amd64Ctx) tailCallAndRet(symOp Operand) error { // If ArgRegs is empty, default to register-based passing (ABIInternal-ish) // because most intra-asm tailcalls depend on explicit register setup. // - // Exception: for tailcalls to Go functions with an identical signature, + // Exception: for tailcalls to Go functions with identical argument types, // use the current function's LLVM args. This matches stdlib patterns like // "JMP ·countGeneric(SB)" that happen before any register shuffling and - // are stack-ABI tailcalls in the original asm. + // are stack-ABI tailcalls in the original asm. The return types may differ + // while using ABI-equivalent physical result slots. useLLVMArgs := false - if len(csig.ArgRegs) == 0 && len(csig.Args) == len(c.sig.Args) && csig.Ret == c.sig.Ret { + if len(csig.ArgRegs) == 0 && len(csig.Args) == len(c.sig.Args) { same := true for j := 0; j < len(csig.Args); j++ { if csig.Args[j] != c.sig.Args[j] { @@ -527,8 +648,22 @@ func (c *amd64Ctx) tailCallAndRet(symOp Operand) error { return nil } if c.sig.Ret != csig.Ret { + if adapted, ok, err := c.adaptTailCallAggregateReturn("%"+t, csig.Ret, c.sig.Ret); err != nil { + return fmt.Errorf("amd64 tailcall return type mismatch for %q: %w", callee, err) + } else if ok { + fmt.Fprintf(c.b, " ret %s %s\n", c.sig.Ret, adapted) + return nil + } conv := c.newTmp() switch { + case c.goarch == "386" && csig.Ret == I32 && c.sig.Ret == Ptr: + fmt.Fprintf(c.b, " %%%s = inttoptr i32 %%%s to ptr\n", conv, t) + fmt.Fprintf(c.b, " ret ptr %%%s\n", conv) + return nil + case c.goarch == "386" && csig.Ret == Ptr && c.sig.Ret == I32: + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %%%s to i32\n", conv, t) + fmt.Fprintf(c.b, " ret i32 %%%s\n", conv) + return nil case csig.Ret == I64 && (c.sig.Ret == I1 || c.sig.Ret == I8 || c.sig.Ret == I16 || c.sig.Ret == I32): fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to %s\n", conv, t, c.sig.Ret) fmt.Fprintf(c.b, " ret %s %%%s\n", c.sig.Ret, conv) @@ -552,3 +687,47 @@ func (c *amd64Ctx) tailCallAndRet(symOp Operand) error { fmt.Fprintf(c.b, " ret %s %%%s\n", c.sig.Ret, t) return nil } + +func (c *amd64Ctx) adaptTailCallAggregateReturn(value string, fromTy, toTy LLVMType) (string, bool, error) { + fromFields, fromAggregate := parseLiteralStructFields(fromTy) + toFields, toAggregate := parseLiteralStructFields(toTy) + if !fromAggregate || !toAggregate { + return "", false, nil + } + if len(fromFields) != len(toFields) { + return "", false, fmt.Errorf("caller %s and callee %s use different aggregate field counts", toTy, fromTy) + } + + wordTy := I64 + if c.goarch == "386" { + wordTy = I32 + } + for i := range fromFields { + from, to := fromFields[i], toFields[i] + if from == to || (from == wordTy && to == Ptr) || (from == Ptr && to == wordTy) { + continue + } + return "", false, fmt.Errorf("caller field %d has type %s but callee field has ABI-incompatible type %s", i, to, from) + } + + result := "undef" + for i := range fromFields { + from, to := fromFields[i], toFields[i] + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", extracted, fromTy, value, i) + field := "%" + extracted + if from != to { + converted := c.newTmp() + if from == wordTy && to == Ptr { + fmt.Fprintf(c.b, " %%%s = inttoptr %s %s to ptr\n", converted, wordTy, field) + } else { + fmt.Fprintf(c.b, " %%%s = ptrtoint ptr %s to %s\n", converted, field, wordTy) + } + field = "%" + converted + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertvalue %s %s, %s %s, %d\n", inserted, toTy, result, to, field, i) + result = "%" + inserted + } + return result, true, nil +} diff --git a/amd64_lower_branch_test.go b/amd64_lower_branch_test.go index da0c48225..ebf12e273 100644 --- a/amd64_lower_branch_test.go +++ b/amd64_lower_branch_test.go @@ -1,6 +1,3 @@ -//go:build !llgo -// +build !llgo - package plan9asm import ( diff --git a/amd64_lower_broadcast.go b/amd64_lower_broadcast.go new file mode 100644 index 000000000..15f0b289f --- /dev/null +++ b/amd64_lower_broadcast.go @@ -0,0 +1,119 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedScalarBroadcast implements the complete Go 1.27 +// _yvpbroadcastb operand table shared by VPBROADCASTB/W/D/Q. +func (c *amd64Ctx) lowerPackedScalarBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastInteger { + return false, false, nil + } + if err != nil { + return true, false, err + } + baseOp, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + laneBits, byteWidth := f.laneBits, f.byteWidth + dstArg := ins.Args[f.destination] + masked, zeroing := f.mask >= 0, f.zeroing + var mask string + if masked { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + + load := func() (string, error) { + return c.loadPackedBroadcastScalar(ins.Args[0], laneBits, ins.x86Encoded) + } + var scalar string + if masked && ins.Args[0].Kind != OpReg { + // Any active destination lane enables the one scalar read. Irrelevant + // high K bits must not turn a fully masked memory source into a load. + lanes := byteWidth * 8 / laneBits + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(dstArg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadPackedBroadcastScalar(src Operand, laneBits int, rawEncoded bool) (string, error) { + if src.Kind == OpImm && rawEncoded { + value := src.Imm + switch laneBits { + case 8: + value = int64(int8(value)) + case 16: + value = int64(int16(value)) + case 32: + value = int64(int32(value)) + } + return fmt.Sprintf("%d", value), nil + } + if src.Kind == OpReg { + if isAMD64XReg(src.Reg) { + bytesValue, err := c.loadX(src.Reg) + if err != nil { + return "", err + } + lanes := 128 / laneBits + laneVector := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, bytesValue) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 0\n", low, lanes, laneBits, laneVector) + return "%" + low, nil + } + if !isAMD64YrlRegister(src.Reg) { + return "", fmt.Errorf("expected an X register, GP register, or memory, got %s", src.String()) + } + return c.evalIntSized(src, amd64IntegerTypeForBits(laneBits)) + } + if src.Kind == OpSym { + // A bare integer in Go's x86 syntax is absolute memory, not an + // immediate (immediates carry a '$'). The parser keeps this spelling + // as OpSym so distinguish it here before the generic symbol loader. + if address, parseErr := parseInt(src.Sym); parseErr == nil { + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i%d, ptr %s, align 1\n", loaded, laneBits, c.ptrFromAddrI64(fmt.Sprintf("%d", address))) + return "%" + loaded, nil + } + } + if !isAMD64MemoryOperand(src) { + return "", fmt.Errorf("expected an X register, GP register, or memory, got %s", src.String()) + } + return c.evalIntSized(src, amd64IntegerTypeForBits(laneBits)) +} + +// isAMD64YrlRegister mirrors Go's amd64 Yrl register class. Byte aliases, +// vector, mask, MMX, and x87 registers are deliberately excluded. +func isAMD64YrlRegister(r Reg) bool { + switch strings.ToUpper(string(r)) { + case "AX", "BX", "CX", "DX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return true + default: + return false + } +} diff --git a/amd64_lower_broadcast_block.go b/amd64_lower_broadcast_block.go new file mode 100644 index 000000000..1945c8c26 --- /dev/null +++ b/amd64_lower_broadcast_block.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedBlockBroadcastSpec struct { + sourceBytes int + laneBits int + allowXSource bool + vexMemoryOnly bool + allowDestinations map[int]bool +} + +var amd64PackedBlockBroadcastSpecs = map[string]amd64PackedBlockBroadcastSpec{ + "VBROADCASTF128": {sourceBytes: 16, laneBits: 64, vexMemoryOnly: true, allowDestinations: map[int]bool{32: true}}, + "VBROADCASTI128": {sourceBytes: 16, laneBits: 64, vexMemoryOnly: true, allowDestinations: map[int]bool{32: true}}, + "VBROADCASTF32X2": {sourceBytes: 8, laneBits: 32, allowXSource: true, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTI32X2": {sourceBytes: 8, laneBits: 32, allowXSource: true, allowDestinations: map[int]bool{16: true, 32: true, 64: true}}, + "VBROADCASTF32X4": {sourceBytes: 16, laneBits: 32, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTF64X2": {sourceBytes: 16, laneBits: 64, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTI32X4": {sourceBytes: 16, laneBits: 32, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTI64X2": {sourceBytes: 16, laneBits: 64, allowDestinations: map[int]bool{32: true, 64: true}}, + "VBROADCASTF32X8": {sourceBytes: 32, laneBits: 32, allowDestinations: map[int]bool{64: true}}, + "VBROADCASTF64X4": {sourceBytes: 32, laneBits: 64, allowDestinations: map[int]bool{64: true}}, + "VBROADCASTI32X8": {sourceBytes: 32, laneBits: 32, allowDestinations: map[int]bool{64: true}}, + "VBROADCASTI64X4": {sourceBytes: 32, laneBits: 64, allowDestinations: map[int]bool{64: true}}, +} + +// lowerPackedBlockBroadcast implements all Go 1.27 EVEX packed-block +// broadcast tables. The X2 forms copy the low 64 source bits; the X4 and X8 +// forms accept only 128- and 256-bit memory sources respectively. +func (c *amd64Ctx) lowerPackedBlockBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastBlock { + return false, false, nil + } + if err != nil { + return true, false, err + } + baseOp, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec := amd64PackedBlockBroadcastSpecs[baseOp] + source, destination := ins.Args[0], ins.Args[f.destination] + destinationBytes := f.byteWidth + masked, zeroing := f.mask >= 0, f.zeroing + mask := "" + if masked { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + + block, err := c.loadPackedBlockBroadcastSource(source, spec.sourceBytes) + if err != nil { + return true, false, err + } + broadcast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %s, <%d x i32> %s\n", + broadcast, spec.sourceBytes, block, spec.sourceBytes, block, destinationBytes, llvmRepeatI8Mask(spec.sourceBytes, destinationBytes)) + result := "%" + broadcast + + if masked { + lanes := destinationBytes * 8 / spec.laneBits + computed := c.bitcastVectorBytesToIntegerLanes(destinationBytes, lanes, spec.laneBits, result) + oldBytes, err := c.loadPackedCompareBytes(destination, destinationBytes) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(destinationBytes, lanes, spec.laneBits, oldBytes) + maskedResult := amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, zeroing) + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", cast, lanes, spec.laneBits, maskedResult, destinationBytes) + result = "%" + cast + } + return true, false, c.storePackedMoveOperand(destination, destinationBytes, result) +} + +func amd64PackedBlockXSourceHint(allowed bool) string { + if allowed { + return " or X-register source" + } + return "" +} + +func (c *amd64Ctx) loadPackedBlockBroadcastSource(source Operand, sourceBytes int) (string, error) { + if source.Kind == OpReg { + value, err := c.loadX(source.Reg) + if err != nil { + return "", err + } + words := c.newTmp() + low := c.newTmp() + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", words, value) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, words) + fmt.Fprintf(c.b, " %%%s = bitcast i64 %%%s to <8 x i8>\n", bytesValue, low) + return "%" + bytesValue, nil + } + + switch source.Kind { + case OpMem: + address, err := c.addrFromMem(source.Mem) + if err != nil { + return "", err + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load <%d x i8>, ptr %s, align 1\n", loaded, sourceBytes, c.ptrFromAddrI64(address)) + return "%" + loaded, nil + case OpSym: + pointer, err := c.ptrFromSB(source.Sym) + if err != nil { + return "", err + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load <%d x i8>, ptr %s, align 1\n", loaded, sourceBytes, pointer) + return "%" + loaded, nil + case OpFP: + chunks := sourceBytes / 8 + words := "zeroinitializer" + for chunk := 0; chunk < chunks; chunk++ { + word, err := c.evalFPToI64(source.FPOffset + int64(chunk*8)) + if err != nil { + return "", err + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %s, i32 %d\n", inserted, chunks, words, word, chunk) + words = "%" + inserted + } + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytesValue, chunks, words, sourceBytes) + return "%" + bytesValue, nil + default: + return "", fmt.Errorf("expected packed-block broadcast memory source, got %s", source.String()) + } +} diff --git a/amd64_lower_broadcast_block_test.go b/amd64_lower_broadcast_block_test.go new file mode 100644 index 000000000..3335b2ea8 --- /dev/null +++ b/amd64_lower_broadcast_block_test.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedBlockBroadcastCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 exposes the complete VEX/EVEX packed-block broadcast family as: + // + // VBROADCASTF128 m128 -> Y + // VBROADCASTI128 m128 -> Y + // VBROADCASTF32X2 X/m64 -> Y/Z + // VBROADCASTI32X2 X/m64 -> X/Y/Z + // VBROADCASTF32X4 m128 -> Y/Z + // VBROADCASTF64X2 m128 -> Y/Z + // VBROADCASTI32X4 m128 -> Y/Z + // VBROADCASTI64X2 m128 -> Y/Z + // VBROADCASTF32X8 m256 -> Z + // VBROADCASTF64X4 m256 -> Z + // VBROADCASTI32X8 m256 -> Z + // VBROADCASTI64X4 m256 -> Z + // + // Every form also accepts K1-K7 merge masking and .Z zero masking. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastY := 15 + lastZ := 21 + if target.goarch == "386" { + lastY, lastZ = 7, 7 + } + var src strings.Builder + src.WriteString("TEXT packedblockbroadcastforms(SB),$0-0\n") + fmt.Fprintf(&src, "\tVBROADCASTF128 0(AX), Y%d\n", lastY) + fmt.Fprintf(&src, "\tVBROADCASTI128 16(AX), Y%d\n", lastY) + src.WriteString("\tVBROADCASTF32X2 X20, Y21\n") + src.WriteString("\tVBROADCASTF32X2 8(AX), K1, Y20\n") + fmt.Fprintf(&src, "\tVBROADCASTF32X2.Z X20, K2, Z%d\n", lastZ) + src.WriteString("\tVBROADCASTI32X2 X20, X21\n") + src.WriteString("\tVBROADCASTI32X2 16(AX), K3, Y20\n") + fmt.Fprintf(&src, "\tVBROADCASTI32X2.Z X20, K4, Z%d\n", lastZ) + for _, op := range []string{ + "VBROADCASTF32X4", "VBROADCASTF64X2", + "VBROADCASTI32X4", "VBROADCASTI64X2", + } { + fmt.Fprintf(&src, "\t%s 24(AX), Y20\n", op) + fmt.Fprintf(&src, "\t%s 40(AX), K5, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z 56(AX), K6, Z%d\n", op, lastZ) + } + for _, op := range []string{ + "VBROADCASTF32X8", "VBROADCASTF64X4", + "VBROADCASTI32X8", "VBROADCASTI64X4", + } { + fmt.Fprintf(&src, "\t%s 72(AX), Z%d\n", op, lastZ) + fmt.Fprintf(&src, "\t%s.Z 104(AX), K7, Z%d\n", op, lastZ) + } + src.WriteString("\tRET\n") + source := src.String() + if target.goarch == "386" { + // High Go spellings have no physical mode32 binding. The + // independent authority controls retain their Context cases. + source = strings.NewReplacer("X20", "X3", "X21", "X4", "Y20", "Y3", "Y21", "Y4").Replace(source) + } + requireX86GoAssemblerResult(t, target.goarch, source, true) + + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedblockbroadcastforms": {Name: "packedblockbroadcastforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-block-broadcast-"+target.name+".ll", "packed-block-broadcast-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedBlockBroadcastRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VBROADCASTF128 (AX), Y16", + "VBROADCASTI128 (AX), Y16", + "VBROADCASTF128 X0, Y1", + "VBROADCASTI128 X0, Y1", + "VBROADCASTF128 (AX), X1", + "VBROADCASTI128 (AX), Z1", + "VBROADCASTF128 (AX), K1, Y1", + "VBROADCASTI128.Z (AX), K1, Y1", + "VBROADCASTF32X2 X0, X1", + "VBROADCASTF32X2 Y0, Y1", + "VBROADCASTI32X2 Y0, Y1", + "VBROADCASTF32X4 X0, Y1", + "VBROADCASTF64X2 X0, Y1", + "VBROADCASTI32X4 X0, Y1", + "VBROADCASTI64X2 X0, Y1", + "VBROADCASTF32X4 (AX), X1", + "VBROADCASTF32X8 (AX), Y1", + "VBROADCASTF64X4 Y0, Z1", + "VBROADCASTI32X8 Y0, Z1", + "VBROADCASTI64X4 (AX), Y1", + "VBROADCASTI32X4 (AX), K0, Y1", + "VBROADCASTI32X4.Z (AX), Y1", + "VBROADCASTI32X4.BCST (AX), Y1", + "VBROADCASTI32X4 (AX), K1, K2, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86PackedBlockBroadcastRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VBROADCASTF128 (AX), Y16", + "VBROADCASTI128 (AX), Y16", + "VBROADCASTI32X2 X20, Z8", + "VBROADCASTI32X4 (AX), Z20", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86PackedBlockBroadcastRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedBlockBroadcastRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed-block broadcast forms", instruction) + } +} + +func TestAMD64LegacyBlockBroadcastRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT legacyblockbroadcast(SB),$0-16 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + VBROADCASTF128 (BX), Y0 + VMOVDQU Y0, 0(AX) + VBROADCASTI128 (BX), Y1 + VMOVDQU Y1, 32(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "legacyblockbroadcast": {Name: "legacyblockbroadcast", Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +extern void legacyblockbroadcast(uint8_t *, const uint8_t *); +int main(void) { + const uint8_t source[16] = {0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0xaa,0xbb,0xcc,0xdd,0xee,0xff}; + uint8_t out[64] = {0}; + legacyblockbroadcast(out, source); + for (int block = 0; block < 2; block++) { + if (memcmp(out + 32*block, source, 16) || memcmp(out + 32*block + 16, source, 16)) return block + 1; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_block_broadcast", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_broadcast_float.go b/amd64_lower_broadcast_float.go new file mode 100644 index 000000000..ef0653057 --- /dev/null +++ b/amd64_lower_broadcast_float.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" +) + +// lowerScalarFloatBroadcast implements all Go 1.27 _yvbroadcastss and +// _yvbroadcastsd operand forms, including EVEX mask merge and zeroing. +func (c *amd64Ctx) lowerScalarFloatBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastFloat { + return false, false, nil + } + if err != nil { + return true, false, err + } + laneBits, byteWidth := f.laneBits, f.byteWidth + source, dst := ins.Args[0], ins.Args[f.destination] + masked, zeroing := f.mask >= 0, f.zeroing + var mask string + if masked { + mask, err = c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + } + + var scalar string + if source.Kind == OpReg { + scalar, err = c.loadXLowInteger(source.Reg, laneBits) + } else if source.Kind == OpImm { + scalar, err = c.evalIntSized(source, amd64IntegerTypeForBits(laneBits)) + } else { + if masked { + // Broadcasting reads the scalar only if any destination lane is + // active. High K bits outside this vector's lane count do not + // enable the load; checking only mask bit zero is also incorrect. + lanes := byteWidth * 8 / laneBits + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(dst, byteWidth, "%"+out) +} diff --git a/amd64_lower_broadcast_mask.go b/amd64_lower_broadcast_mask.go new file mode 100644 index 000000000..5fbdec76a --- /dev/null +++ b/amd64_lower_broadcast_mask.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64MaskBroadcastSpec struct { + sourceBits int + laneBits int +} + +// amd64MaskBroadcastSpecs is the complete Go 1.27 _yvpbroadcastmb2q grammar. +// The low byte/word of a K register is zero-extended and broadcast into every +// qword/dword lane of an EVEX X, Y, or Z destination. +var amd64MaskBroadcastSpecs = map[Op]amd64MaskBroadcastSpec{ + "VPBROADCASTMB2Q": {sourceBits: 8, laneBits: 64}, + "VPBROADCASTMW2D": {sourceBits: 16, laneBits: 32}, +} + +func (c *amd64Ctx) lowerMaskRegisterBroadcast(op Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86BroadcastProperties(op, ins, c.goarch) + if !known || f.kind != x86BroadcastMask { + return false, false, nil + } + if err != nil { + return true, false, err + } + spec := amd64MaskBroadcastSpecs[Op(strings.ToUpper(string(op)))] + byteWidth := f.byteWidth + mask, err := c.loadK(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, mask, spec.sourceBits) + scalar := "%" + narrow + if spec.sourceBits != spec.laneBits { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i%d\n", wide, spec.sourceBits, scalar, spec.laneBits) + scalar = "%" + wide + } + lanes := byteWidth * 8 / spec.laneBits + result := amd64SplatInteger(c, lanes, spec.laneBits, scalar) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(ins.Args[1], byteWidth, "%"+out) +} diff --git a/amd64_lower_broadcast_mask_test.go b/amd64_lower_broadcast_mask_test.go new file mode 100644 index 000000000..d7c718662 --- /dev/null +++ b/amd64_lower_broadcast_mask_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64MaskBroadcastGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64MaskBroadcastSpec{ + "VPBROADCASTMB2Q": {sourceBits: 8, laneBits: 64}, + "VPBROADCASTMW2D": {sourceBits: 16, laneBits: 32}, + } + if len(amd64MaskBroadcastSpecs) != len(expected) { + t.Fatalf("mask-broadcast grammar has %d entries, want %d", len(amd64MaskBroadcastSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64MaskBroadcastSpecs[op]; !ok { + t.Errorf("mask-broadcast grammar omitted %s", op) + } else if got != want { + t.Errorf("mask-broadcast grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86MaskBroadcastCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + xyLast, zLast := 31, 31 + if target.goarch == "386" { + xyLast, zLast = 7, 7 + } + var source strings.Builder + source.WriteString("TEXT maskbroadcastforms(SB),$0-0\n") + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + fmt.Fprintf(&source, "\t%s K0, X%d\n", op, xyLast) + fmt.Fprintf(&source, "\t%s K7, Y%d\n", op, xyLast) + fmt.Fprintf(&source, "\t%s K1, Z%d\n", op, zLast) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"maskbroadcastforms": {Name: "maskbroadcastforms", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"<2 x i64>", "<8 x i32>", "<8 x i64>", "<16 x i32>"} { + if !strings.Contains(ir, want) { + t.Fatalf("mask broadcast lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "mask-broadcast-"+target.name+".ll", "mask-broadcast-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86MaskBroadcastRejectsFormsOutsideGoTable(t *testing.T) { + for _, instruction := range []string{ + "VPBROADCASTMB2Q AX, X0", + "VPBROADCASTMW2D K8, X0", + "VPBROADCASTMB2Q K1, K2", + "VPBROADCASTMW2D.Z K1, X0", + "VPBROADCASTMB2Q K1, K2, X0", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvpbroadcastmb2q table", instruction) + } + }) + } +} + +func TestAMD64MaskBroadcastRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT maskbroadcastsemantics(SB),$0-16 + MOVQ out+0(FP), CX + MOVQ mask+8(FP), AX + KMOVQ AX, K1 + STC + VPBROADCASTMB2Q K1, X0 + MOVUPS X0, 0(CX) + VPBROADCASTMW2D K1, X1 + MOVUPS X1, 16(CX) + SETCS 32(CX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"maskbroadcastsemantics": { + Name: "maskbroadcastsemantics", Args: []LLVMType{Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void maskbroadcastsemantics(uint8_t *, uint64_t); +int main(void) { + uint8_t out[33] = {0}; + uint64_t q[2]; uint32_t d[4]; + maskbroadcastsemantics(out, UINT64_C(0x1234beefa5)); + memcpy(q, out, 16); memcpy(d, out + 16, 16); + if (q[0] != UINT64_C(0xa5) || q[1] != UINT64_C(0xa5)) return 10; + for (int i = 0; i < 4; i++) if (d[i] != UINT32_C(0xefa5)) return 11 + i; + if (out[32] != 1) return 20; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_broadcast_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_byte_shift.go b/amd64_lower_byte_shift.go new file mode 100644 index 000000000..2bb931b42 --- /dev/null +++ b/amd64_lower_byte_shift.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "fmt" +) + +// lowerPackedByteShift implements Go 1.27's PSLLO/PSRLO aliases and +// PSLLDQ/PSRLDQ vector family. Every operation shifts bytes independently +// within each 128-bit lane and fills vacated bytes with zero. +func (c *amd64Ctx) lowerPackedByteShift(op Op, ins Instr) (bool, bool, error) { + return c.lowerX86PackedShift(op, ins) +} + +func amd64SignedImmediate(operand Operand, bits int) bool { + if operand.Kind != OpImm || bits <= 0 || bits >= 63 { + return false + } + limit := int64(1) << (bits - 1) + return operand.Imm >= -limit && operand.Imm < limit +} + +func amd64VectorByteShiftImmediate(operand Operand) bool { + return operand.Kind == OpImm && operand.Imm >= -128 && operand.Imm <= 255 +} + +func amd64PackedByteShiftRequiresEVEX(source, destination Operand, byteWidth int) bool { + if byteWidth == 64 || isAMD64MemoryOperand(source) { + return true + } + for _, operand := range []Operand{source, destination} { + index, ok := amd64VectorRegisterIndex(operand.Reg, byteWidth) + if !ok || index >= 16 { + return true + } + } + return false +} + +func (c *amd64Ctx) emitPackedByteShift(byteWidth int, source string, count uint8, left bool) string { + if count >= 16 { + return "zeroinitializer" + } + result := "zeroinitializer" + for laneBase := 0; laneBase < byteWidth; laneBase += 16 { + for destinationByte := 0; destinationByte < 16; destinationByte++ { + sourceByte := destinationByte + int(count) + if left { + sourceByte = destinationByte - int(count) + } + if sourceByte < 0 || sourceByte >= 16 { + continue + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", extracted, byteWidth, source, laneBase+sourceByte) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, result, extracted, laneBase+destinationByte) + result = "%" + inserted + } + } + return result +} diff --git a/amd64_lower_byte_shift_test.go b/amd64_lower_byte_shift_test.go new file mode 100644 index 000000000..10062d5c7 --- /dev/null +++ b/amd64_lower_byte_shift_test.go @@ -0,0 +1,180 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedByteShiftCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zSource, zDestination := 16, 31 + if target.goarch == "386" { + legacyLast = 7 + zSource, zDestination = 6, 7 + } + var source strings.Builder + source.WriteString("TEXT packedbyteshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSLLO", "PSLLDQ", "PSRLO", "PSRLDQ"} { + fmt.Fprintf(&source, "\t%s $-128, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $127, X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSLLDQ", "VPSRLDQ"} { + source.WriteString("\t" + op + " $-128, X0, X1\n") + source.WriteString("\t" + op + " $255, Y0, Y1\n") + source.WriteString("\t" + op + " $1, 8(AX), X2\n") + source.WriteString("\t" + op + " $2, 16(AX), Y2\n") + fmt.Fprintf(&source, "\t%s $3, X%d, X%d\n", op, zSource, zDestination) + fmt.Fprintf(&source, "\t%s $4, Z%d, Z%d\n", op, zSource, zDestination) + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedbyteshiftforms": {Name: "packedbyteshiftforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-byte-shift-"+target.name+".ll", "packed-byte-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedByteShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSLLO $-129, X0", + "PSLLDQ $128, X0", + "PSRLO $1, X16", + "PSRLDQ $1, Y0", + "PSLLDQ.Z $1, X0", + "VPSLLDQ $-129, X0, X1", + "VPSRLDQ $256, X0, X1", + "VPSLLDQ $-1, 8(AX), X1", + "VPSRLDQ $-1, X16, X1", + "VPSLLDQ $-1, Z0, Z1", + "VPSRLDQ $1, X0, Y1", + "VPSLLDQ $1, X0, K1, X1", + "VPSRLDQ.Z $1, X0, X1", + "VPSLLDQ $1, X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSLLO $1, X8", + "PSRLO $1, X15", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShiftRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedByteShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed byte-shift tables for %s", instruction, goarch) + } +} + +func TestAMD64PackedByteShiftRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedbyteshift(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), AX + MOVQ in+8(FP), BX + VMOVDQU (BX), Y0 + VPSLLDQ $3, Y0, Y1 + VMOVDQU Y1, 0(AX) + VPSRLDQ $5, Y0, Y1 + VMOVDQU Y1, 32(AX) + VPSLLDQ $-1, Y0, Y1 + VMOVDQU Y1, 64(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedbyteshift": {Name: "packedbyteshift", Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void packedbyteshift(uint8_t *, const uint8_t *); +int main(void) { + uint8_t in[32], out[96]; + for (int i = 0; i < 32; i++) in[i] = (uint8_t)(i+1); + for (int i = 0; i < 96; i++) out[i] = 0xff; + packedbyteshift(out, in); + for (int lane = 0; lane < 2; lane++) for (int i = 0; i < 16; i++) { + int base = lane*16; + uint8_t left = i < 3 ? 0 : in[base+i-3]; + uint8_t right = i+5 >= 16 ? 0 : in[base+i+5]; + if (out[base+i] != left) return 10+base+i; + if (out[32+base+i] != right) return 50+base+i; + if (out[64+base+i] != 0) return 90+base+i; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_byte_shift", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_bzhi.go b/amd64_lower_bzhi.go new file mode 100644 index 000000000..7e696c61c --- /dev/null +++ b/amd64_lower_bzhi.go @@ -0,0 +1,72 @@ +package plan9asm + +import "fmt" + +// lowerBZHI implements Go 1.27's complete _ybextrl operand row for BZHIL/Q: +// register index, register/memory source, and register destination. +func (c *amd64Ctx) lowerBZHI(op Op, ins Instr) (ok bool, terminated bool, err error) { + bits := 0 + switch op { + case "BZHIL": + bits = 32 + case "BZHIQ": + bits = 64 + default: + return false, false, nil + } + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects register index, register/memory source, and register destination: %q", c.goarch, op, ins.Raw) + } + index, source, destination := ins.Args[0], ins.Args[1], ins.Args[2] + if index.Kind != OpReg || !isX86YrlRegisterForArch(index.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s index is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) + } + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + indexValue, err := c.evalIntSized(index, typ) + if err != nil { + return true, false, err + } + sourceValue, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + indexLowByte := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, 255\n", indexLowByte, typ, indexValue) + indexInRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %%%s, %d\n", indexInRange, typ, indexLowByte, bits) + safeIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", safeIndex, typ, indexLowByte, bits-1) + one := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", one, typ, safeIndex) + partialMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %%%s, -1\n", partialMask, typ, one) + mask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s -1\n", mask, indexInRange, typ, partialMask, typ) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", result, typ, sourceValue, mask) + if err := c.storeRegSized(destination.Reg, typ, "%"+result); err != nil { + return true, false, err + } + carry := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", carry, indexInRange) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", sign, typ, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + return true, false, nil +} diff --git a/amd64_lower_bzhi_test.go b/amd64_lower_bzhi_test.go new file mode 100644 index 000000000..5562ca55e --- /dev/null +++ b/amd64_lower_bzhi_test.go @@ -0,0 +1,204 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86BZHICompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA bzhidata+0(SB)/8, $1\n") + source.WriteString("GLOBL bzhidata(SB), $8\n") + source.WriteString("TEXT bzhiforms(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + op := "BZHI" + width + fmt.Fprintf(&source, "\t%s AX, BX, CX\n", op) + fmt.Fprintf(&source, "\t%s DX, 8(BX), SI\n", op) + fmt.Fprintf(&source, "\t%s CX, bzhidata(SB), DX\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12, R13\n", op) + } else { + fmt.Fprintf(&source, "\t%s AX, BX, SP\n", op) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86BZHICompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86BZHICompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"bzhiforms": {Name: "bzhiforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"shl i32", "shl i64", `"target-features"="+bmi2"`} { + if !strings.Contains(ir, want) { + t.Fatalf("BZHI lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "bzhi-"+target.name+".ll", "bzhi-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86BZHIRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "BZHIL $1, AX, BX"}, + {goarch: "amd64", instruction: "BZHIQ 0(AX), BX, CX"}, + {goarch: "amd64", instruction: "BZHIQ AX, $1, CX"}, + {goarch: "amd64", instruction: "BZHIL AX, BX, 0(CX)"}, + {goarch: "amd64", instruction: "BZHIQ X0, AX, BX"}, + {goarch: "amd64", instruction: "BZHIQ AX, X0, BX"}, + {goarch: "amd64", instruction: "BZHIQ AX, BX, X0"}, + {goarch: "amd64", instruction: "BZHIQ AX, BX"}, + {goarch: "amd64", instruction: "BZHIQ.Z AX, BX, CX"}, + {goarch: "386", instruction: "BZHIL R8, AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _ybextrl table", test.instruction) + } + }) + } +} + +func TestAMD64BZHIRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT bzhisemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + + MOVL $0x7fffffff, R8 + ADDL $1, R8 + MOVL $8, AX + MOVL $0x1234, BX + BZHIL AX, BX, CX + MOVQ CX, 0(DI) + SETCS 8(DI) + SETOS 9(DI) + SETEQ 10(DI) + SETMI 11(DI) + + MOVL $31, AX + MOVL $0x80000000, BX + BZHIL AX, BX, CX + MOVQ CX, 16(DI) + SETEQ 24(DI) + + MOVL $32, AX + MOVL $0x80000000, BX + BZHIL AX, BX, CX + MOVQ CX, 32(DI) + SETCS 40(DI) + SETMI 41(DI) + + MOVQ $0x1000000000000004, AX + MOVQ $-1, 48(DI) + BZHIQ AX, 48(DI), CX + MOVQ CX, 56(DI) + + MOVQ $64, AX + MOVQ $0x1234, BX + BZHIQ AX, BX, AX + MOVQ AX, 64(DI) + SETCS 72(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "bzhisemantics": { + Name: "bzhisemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void bzhisemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[80] = {0}; + bzhisemantics(out); + if (load64(out + 0) != UINT64_C(0x34)) return 10; + if (out[8] != 0 || out[9] != 0 || out[10] != 0 || out[11] != 0) return 11; + if (load64(out + 16) != 0 || out[24] != 1) return 12; + if (load64(out + 32) != UINT64_C(0x80000000) || out[40] != 1 || out[41] != 1) return 13; + if (load64(out + 56) != UINT64_C(0xf)) return 14; + if (load64(out + 64) != UINT64_C(0x1234) || out[72] != 1) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "bzhi_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_cache_line.go b/amd64_lower_cache_line.go new file mode 100644 index 000000000..57dbc2054 --- /dev/null +++ b/amd64_lower_cache_line.go @@ -0,0 +1,42 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86CacheLineWritebackOps = map[string]struct{}{ + "CLFLUSH": {}, + "CLFLUSHOPT": {}, + "CLWB": {}, +} + +// lowerCacheLineWriteback covers Go 1.27's complete x86 yclflush operand +// table. All three instructions accept exactly one Ym memory operand on both +// 386 and amd64. Preserve the architectural cache operation as inline +// assembly: replacing it with an ordinary load would lose ordering and +// persistence semantics. +func (c *amd64Ctx) lowerCacheLineWriteback(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if _, ok := x86CacheLineWritebackOps[baseOp]; !ok { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 1 || !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s expects one Go 1.27 Ym memory operand: %q", c.goarch, baseOp, ins.Raw) + } + + ptr, ptrType, err := c.x86DescriptorMemoryPointer(ins.Args[0]) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype(i8) %s)\n", + strings.ToLower(baseOp)+" $0", "*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, ptr) + return true, false, nil +} diff --git a/amd64_lower_cache_line_test.go b/amd64_lower_cache_line_test.go new file mode 100644 index 000000000..52f173bdb --- /dev/null +++ b/amd64_lower_cache_line_test.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86CacheLineWritebackCompleteGoAssemblerForms(t *testing.T) { + const source = ` +TEXT cachelinewriteback(SB),$16-0 + CLFLUSH (AX) + CLFLUSHOPT 8(BX) + CLWB local+0(SP) + RET +` + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "386", triple: "i686-pc-windows-msvc"}, + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.goarch+"/"+target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: target.triple, Goarch: target.goarch, + Sigs: map[string]FuncSig{"cachelinewriteback": {Name: "cachelinewriteback", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, mnemonic := range []string{"clflush $0", "clflushopt $0", "clwb $0"} { + if !strings.Contains(ir, mnemonic) { + t.Fatalf("cache-line lowering omitted %q:\n%s", mnemonic, ir) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-cache-line-writeback.ll", "x86-cache-line-writeback.o", ir) + }) + } +} + +func TestTranslateX86CacheLineWritebackRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + triple := "i386-unknown-linux-gnu" + if goarch == "amd64" { + triple = "x86_64-unknown-linux-gnu" + } + for _, instruction := range []string{ + "CLFLUSH", + "CLFLUSH AX", + "CLFLUSH $1", + "CLFLUSH (AX), (BX)", + "CLFLUSH.X (AX)", + "CLFLUSHOPT AX", + "CLWB AX", + } { + t.Run(goarch+"/"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + TargetTriple: triple, Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + if err == nil { + t.Fatalf("Translate accepted form rejected by Go 1.27: %s", instruction) + } + }) + } + } +} diff --git a/amd64_lower_call_abi0_test.go b/amd64_lower_call_abi0_test.go new file mode 100644 index 000000000..8346f9523 --- /dev/null +++ b/amd64_lower_call_abi0_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86ABI0CallUsesOutgoingStackFrame(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + mov string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", mov: "MOVQ"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", mov: "MOVL"}, + } { + t.Run(target.goarch, func(t *testing.T) { + source := "TEXT ·abi0Caller(SB),$48-0\n" + + "\t" + target.mov + " $1, AX\n" + + "\t" + target.mov + " AX, 0(SP)\n" + + "\t" + target.mov + " $2, AX\n" + + "\t" + target.mov + " AX, 8(SP)\n" + + "\t" + target.mov + " $3, AX\n" + + "\t" + target.mov + " AX, 16(SP)\n" + + "\tMOVL $0x3fc00000, 24(SP)\n" + + "\t" + target.mov + " $4, AX\n" + + "\t" + target.mov + " AX, 32(SP)\n" + + "\tCALL ·abi0Callee(SB)\n" + + "\t" + target.mov + " 40(SP), AX\n" + + "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + word := I64 + if target.goarch == "386" { + word = I32 + } + aggregate := LLVMType("{ " + string(Ptr) + ", " + string(word) + ", " + string(word) + " }") + callee := FuncSig{ + Name: "abi0Callee", + Args: []LLVMType{aggregate, LLVMType("float"), LLVMType("double")}, + Ret: word, + Frame: FrameLayout{ + Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: 0}, + {Offset: 8, Type: word, Index: 0, Field: 1}, + {Offset: 16, Type: word, Index: 0, Field: 2}, + {Offset: 24, Type: LLVMType("float"), Index: 1, Field: -1}, + {Offset: 32, Type: LLVMType("double"), Index: 2, Field: -1}, + }, + Results: []FrameSlot{{Offset: 40, Type: word, Index: 0, Field: -1}}, + }, + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + ResolveSym: func(sym string) string { return strings.TrimPrefix(sym, "·") }, + Sigs: map[string]FuncSig{ + "abi0Caller": {Name: "abi0Caller", Ret: Void}, + "abi0Callee": callee, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"load ptr", "load float", "load double", "call " + string(word) + " @abi0Callee", "store " + string(word)} { + if !strings.Contains(ll, want) { + t.Fatalf("ABI0 call omitted %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "abi0-call-"+target.goarch+".ll", "abi0-call-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86IndirectBranchCompatibilitySyntax(t *testing.T) { + const source = ` +TEXT ·indirectCallRegister(SB),$0-0 + CALL *AX + RET + +TEXT ·indirectCallSymbol(SB),$0-0 + CALL *·pointer(SB) + RET + +TEXT ·indirectJumpRegister(SB),$0-0 + JMP *AX +` + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + ResolveSym: func(sym string) string { return strings.TrimPrefix(sym, "·") }, + Sigs: map[string]FuncSig{ + "indirectCallRegister": {Name: "indirectCallRegister", Ret: Void}, + "indirectCallSymbol": {Name: "indirectCallSymbol", Ret: Void}, + "indirectJumpRegister": {Name: "indirectJumpRegister", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if got := strings.Count(ll, "call i64 %"); got != 3 { + t.Fatalf("indirect CALL/JMP count = %d, want 3:\n%s", got, ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "indirect-branch-"+target.goarch+".ll", "indirect-branch-"+target.goarch+".o", ll) + }) + } +} diff --git a/amd64_lower_carry_flag_test.go b/amd64_lower_carry_flag_test.go new file mode 100644 index 000000000..186758247 --- /dev/null +++ b/amd64_lower_carry_flag_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestTranslateX86CarryFlagControlCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT carryflagforms(SB),NOSPLIT,$0-0\n\tCLC\n\tSTC\n\tCMC\n\tRET\n") + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"carryflagforms": {Name: "carryflagforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"store i1 false, ptr %flags_cf", "store i1 true, ptr %flags_cf", "xor i1"} { + if !strings.Contains(ll, want) { + t.Fatalf("carry flag lowering missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "carry-flag-"+target.name+".ll", "carry-flag-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86CarryFlagControlRejectsOperands(t *testing.T) { + for _, instruction := range []string{"CLC AX", "STC $1", "CMC 0(BX)", "CLC.Z"} { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's carry-control ynone table", instruction) + } + }) + } +} + +func TestAMD64CarryFlagControlRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT carryflagsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + CLC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 0(DI) + STC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 8(DI) + CLC + CMC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 16(DI) + STC + CMC + MOVQ $0, AX + ADCQ $0, AX + MOVQ AX, 24(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "carryflagsemantics": {Name: "carryflagsemantics", Args: []LLVMType{Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void carryflagsemantics(uint64_t *); +int main(void) { + uint64_t out[4] = {9, 9, 9, 9}; + const uint64_t want[4] = {0, 1, 1, 0}; + carryflagsemantics(out); + for (int i = 0; i < 4; i++) if (out[i] != want[i]) return 10+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "carry_flag_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_carry_rotate_test.go b/amd64_lower_carry_rotate_test.go new file mode 100644 index 000000000..e9cc175f5 --- /dev/null +++ b/amd64_lower_carry_rotate_test.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86CarryRotateCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + widths := []string{"B", "W", "L"} + if target.goarch == "amd64" { + widths = append(widths, "Q") + } + var source strings.Builder + source.WriteString("TEXT carryrotateforms(SB),NOSPLIT,$0-0\n") + for _, stem := range []string{"RCL", "RCR"} { + for _, width := range widths { + op := stem + width + reg := "AX" + if width == "B" { + reg = "AH" + } + fmt.Fprintf(&source, "\t%s $1, %s\n", op, reg) + fmt.Fprintf(&source, "\t%s $255, 8(BX)\n", op) + fmt.Fprintf(&source, "\t%s CL, %s\n", op, reg) + fmt.Fprintf(&source, "\t%s CX, 16(BX)\n", op) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "carryrotateforms": {Name: "carryrotateforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "carry-rotate-"+target.name+".ll", "carry-rotate-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86CarryRotateRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "RCLB $-1, AL", + "RCRW $256, AX", + "RCLL AX, BX", + "RCRQ (AX), BX", + "RCLQ $1, X0", + "RCRB.Z $1, AL", + "RCLL $1, AX, BX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86CarryRotateRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "RCLQ $1, AX", + "RCRQ CL, (BX)", + "RCLB $1, SP", + "RCRL $1, R8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86CarryRotateRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86CarryRotateRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's RCL/RCR tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_clmul.go b/amd64_lower_clmul.go new file mode 100644 index 000000000..e30aad292 --- /dev/null +++ b/amd64_lower_clmul.go @@ -0,0 +1,90 @@ +package plan9asm + +import "fmt" + +// lowerPackedCarrylessMultiply implements Go 1.27's complete legacy and +// vector PCLMULQDQ tables. VPCLMULQDQ applies one independent 64x64->128 +// carryless multiplication to every 128-bit lane. +func (c *amd64Ctx) lowerPackedCarrylessMultiply(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + f, known, err := x86CLMULProperties(rawOp, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if f.legacy && f.memory >= 0 { + return true, false, c.lowerLegacyCLMULMemory(ins, f) + } + second, err := c.loadX86PackedShiftBytes(ins.Args[f.source], f.byteWidth, true) + if err != nil { + return true, false, err + } + first, err := c.loadPackedCompareBytes(ins.Args[f.second], f.byteWidth) + if err != nil { + return true, false, err + } + result := c.emitPackedCarrylessMultiply(f.byteWidth, first, second, f.immediate) + if f.legacy { + return true, false, c.storeX(ins.Args[f.destination].Reg, result) + } + return true, false, c.storePackedMoveOperand(ins.Args[f.destination], f.byteWidth, result) +} + +// The legacy m128 opcode, not its selected numerical qword, determines the +// native memory/alignment/fault effect. Retain the whole original instruction +// even when its product is dead; virtual FP/SP/BP storage is not that address. +func (c *amd64Ctx) lowerLegacyCLMULMemory(ins Instr, f x86CLMULForm) error { + source := ins.Args[f.source] + virtual := source.Kind == OpFP + if source.Kind == OpMem && !x86CallerMemoryCertified(ins, f.source) && !c.fixedLocalMemoryCertified(ins, f.source) { + virtual = !x86OrdinaryNativeMemoryOperand(source, c.goarch) + } + if virtual { + return fmt.Errorf("%w: legacy carryless multiply m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + first, err := c.loadPackedCompareLanes(ins.Args[f.second], 16, 64, false) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + assembly := fmt.Sprintf("pclmulqdq $$%d, %s($2), $0", f.immediate, segment) + result, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <2 x i64> asm sideeffect %q, %q(<2 x i64> %s, %s %s)\n", + result, assembly, "=x,0,r,~{memory}", first, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, result) + return c.storeX(ins.Args[f.destination].Reg, "%"+bytes) +} + +func (c *amd64Ctx) emitPackedCarrylessMultiply(byteWidth int, firstBytes, secondBytes string, immediate uint8) string { + qwords := byteWidth / 8 + first := c.newTmp() + second := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", first, byteWidth, firstBytes, qwords) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", second, byteWidth, secondBytes, qwords) + result := "zeroinitializer" + for lane := 0; lane < byteWidth/16; lane++ { + firstLane := c.newTmp() + secondLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", firstLane, qwords, first, qwords, lane*2, lane*2+1) + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i64> %%%s, <%d x i64> poison, <2 x i32> \n", secondLane, qwords, second, qwords, lane*2, lane*2+1) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %%%s, <2 x i64> %%%s, i8 %d)\n", product, firstLane, secondLane, immediate) + for word := 0; word < 2; word++ { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 %d\n", extracted, product, word) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, qwords, result, extracted, lane*2+word) + result = "%" + inserted + } + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytes, qwords, result, byteWidth) + return "%" + bytes +} diff --git a/amd64_lower_clmul_test.go b/amd64_lower_clmul_test.go new file mode 100644 index 000000000..b92772964 --- /dev/null +++ b/amd64_lower_clmul_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedCarrylessMultiplyCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + fmt.Fprintf(&source, "TEXT carrylessmultiplyforms(SB),NOSPLIT,$0-0\n\tPCLMULQDQ $0, X0, X%d\n\tPCLMULQDQ $255, 8(AX), X%d\n", last, last) + if target.goarch == "amd64" { + source.WriteString("\tVPCLMULQDQ $0, X0, X1, X2\n") + source.WriteString("\tVPCLMULQDQ $1, 8(AX), X1, X2\n") + source.WriteString("\tVPCLMULQDQ $16, Y0, Y1, Y2\n") + source.WriteString("\tVPCLMULQDQ $17, 8(AX), Y1, Y2\n") + source.WriteString("\tVPCLMULQDQ $127, Z0, Z1, Z2\n") + source.WriteString("\tVPCLMULQDQ $128, 8(AX), Z1, Z2\n") + source.WriteString("\tVPCLMULQDQ $255, X16, X31, X0\n") + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "carrylessmultiplyforms": {Name: "carrylessmultiplyforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-carryless-multiply-"+target.name+".ll", "packed-carryless-multiply-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedCarrylessMultiplyRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PCLMULQDQ $-1, X0, X1", + "PCLMULQDQ $256, X0, X1", + "PCLMULQDQ $1, X16, X1", + "PCLMULQDQ $1, Y0, Y1", + "PCLMULQDQ.Z $1, X0, X1", + "PCLMULQDQ $1, X0, (AX)", + "VPCLMULQDQ $-1, X0, X1, X2", + "VPCLMULQDQ $256, X0, X1, X2", + "VPCLMULQDQ $1, X0, Y1, Y2", + "VPCLMULQDQ $1, X0, X1, K1, X2", + "VPCLMULQDQ.Z $1, X0, X1, X2", + "VPCLMULQDQ $1, X0, (AX), X2", + "VPCLMULQDQ $1, X0, X1, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedCarrylessMultiplyRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PCLMULQDQ $1, X8, X0", + "PCLMULQDQ $1, X0, X8", + "VPCLMULQDQ $1, X0, X1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedCarrylessMultiplyRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedCarrylessMultiplyRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PCLMULQDQ/VPCLMULQDQ tables for %s", instruction, goarch) + } +} + +func TestAMD64PackedCarrylessMultiplyRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT carrylessmultiply(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), BX + MOVQ b+16(FP), CX + VMOVDQU (BX), Y0 + VMOVDQU (CX), Y1 + VPCLMULQDQ $0x10, Y0, Y1, Y2 + VMOVDQU Y2, (AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "carrylessmultiply": {Name: "carrylessmultiply", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void carrylessmultiply(uint64_t *, const uint64_t *, const uint64_t *); +static void clmul(uint64_t a, uint64_t b, uint64_t *lo, uint64_t *hi) { + uint64_t l = 0, h = 0; + for (unsigned i = 0; i < 64; i++) if ((b >> i) & 1) { + l ^= a << i; + if (i) h ^= a >> (64-i); + } + *lo = l; *hi = h; +} +int main(void) { + uint64_t a[4] = {0x123456789abcdef0ULL, 0xfedcba9876543210ULL, 0x1111222233334444ULL, 0x5555666677778888ULL}; + uint64_t b[4] = {0x0123456789abcdefULL, 0x0f1e2d3c4b5a6978ULL, 0x9999aaaabbbbccccULL, 0xddddeeeeffff0001ULL}; + uint64_t out[4] = {0}, want[4] = {0}; + clmul(b[0], a[1], &want[0], &want[1]); + clmul(b[2], a[3], &want[2], &want[3]); + carrylessmultiply(out, a, b); + for (int i = 0; i < 4; i++) if (out[i] != want[i]) return 10+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_carryless_multiply", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_cmov.go b/amd64_lower_cmov.go new file mode 100644 index 000000000..1fe2a59d1 --- /dev/null +++ b/amd64_lower_cmov.go @@ -0,0 +1,137 @@ +package plan9asm + +import "fmt" + +// lowerConditionalMove implements the complete Go 1.27 yml_rl conditional +// move table: every x86 condition code at W/L/Q width, with a register or +// memory source and a GP-register destination. +func (c *amd64Ctx) lowerConditionalMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86ConditionalMoveProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + ty := amd64IntegerTypeForBits(form.bits) + src, err := c.loadConditionalMoveSource(form, ins) + if err != nil { + return true, false, err + } + cur, err := c.evalIntSized(form.destination, ty) + if err != nil { + return true, false, err + } + condition, err := c.x86Condition(form.condition) + if err != nil { + return true, false, err + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, %s %s, %s %s\n", selected, condition, ty, src, ty, cur) + // Select only the operand width, then apply the architectural writeback. + // In particular a false L predicate still zeroes GP bits 63:32. + return true, false, c.storeRegSized(form.destination.Reg, ty, "%"+selected) +} + +func (c *amd64Ctx) loadConditionalMoveSource(form x86ConditionalMoveForm, ins Instr) (string, error) { + source := form.source + if source.Kind == OpReg { + return c.evalIntSized(source, amd64IntegerTypeForBits(form.bits)) + } + if source.Kind == OpMem && !x86OrdinaryNativeMemoryOperand(source, c.goarch) && + !x86CallerMemoryCertified(ins, 0) && !c.fixedLocalMemoryCertified(ins, 0) { + return "", fmt.Errorf("%w: CMOV native stack source lacks its original storage certificate", ErrProbeNeedsContext) + } + if source.Kind == OpFP { + // Only existing declared numerical ABI0 fields supply logical bytes. + // Neither pointer fields, holes nor an unknown native frame are inputs. + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, form.bits/8); err != nil { + return "", err + } + result := false + for offset := source.FPOffset; offset < source.FPOffset+int64(form.bits/8); offset++ { + slot, output, _ := c.fpVectorSlot(offset) + if slot.Type != I8 && slot.Type != I16 && slot.Type != I32 && slot.Type != I64 && slot.Type != LLVMType("float") && slot.Type != LLVMType("double") { + return "", fmt.Errorf("%w: CMOV FP byte is not declared numerical storage", ErrProbeNeedsContext) + } + result = result || output + } + if result { + if err := c.scalarFPResultInitialized(); err != nil { + return "", err + } + } + } + // The shared FP loader bitcasts floating slots before extracting bytes; + // NaN payloads and signed zero are not floating-to-integer conversions. + // Intel CMOVcc reads the complete memory source unconditionally, even for + // a false predicate or dead result. Preserve the original width and fault. + bytes, err := c.loadX86PackedShiftBytes(source, form.bits/8, true) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, form.bits/8, bytes, form.bits) + if source.Kind != OpFP || c.classicFrame != "" { + // LLVM22's x86 select expansion can sink even a volatile load into + // one branch. An unconditional register use keeps the actual source + // read before that expansion without emitting an instruction or + // granting a new memory/FP permission. A false CMOV must still fault. + fmt.Fprintf(c.b, " call void asm sideeffect \"\", \"r\"(i%d %%%s)\n", form.bits, value) + } + return "%" + value, nil +} + +func (c *amd64Ctx) x86Condition(code string) (string, error) { + invert := func(value string) string { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", out, value) + return "%" + out + } + combine := func(instruction, left, right string) string { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i1 %s, %s\n", out, instruction, left, right) + return "%" + out + } + + switch code { + case "CC": + return invert(c.loadFlag(c.flagsCFSlot)), nil + case "CS": + return c.loadFlag(c.flagsCFSlot), nil + case "EQ": + return c.loadFlag(c.flagsZSlot), nil + case "GE", "GT", "LE", "LT": + signedLess := combine("xor", c.loadFlag(c.flagsSltSlot), c.loadFlag(c.flagsOFSlot)) + switch code { + case "LT": + return signedLess, nil + case "GE": + return invert(signedLess), nil + case "LE": + return combine("or", c.loadFlag(c.flagsZSlot), signedLess), nil + case "GT": + return invert(combine("or", c.loadFlag(c.flagsZSlot), signedLess)), nil + } + case "HI": + return combine("and", invert(c.loadFlag(c.flagsCFSlot)), invert(c.loadFlag(c.flagsZSlot))), nil + case "LS": + return combine("or", c.loadFlag(c.flagsCFSlot), c.loadFlag(c.flagsZSlot)), nil + case "MI": + return c.loadFlag(c.flagsSltSlot), nil + case "NE": + return invert(c.loadFlag(c.flagsZSlot)), nil + case "OC": + return invert(c.loadFlag(c.flagsOFSlot)), nil + case "OS": + return c.loadFlag(c.flagsOFSlot), nil + case "PC": + return invert(c.loadFlag(c.flagsPFSlot)), nil + case "PL": + return invert(c.loadFlag(c.flagsSltSlot)), nil + case "PS": + return c.loadFlag(c.flagsPFSlot), nil + } + return "", fmt.Errorf("unsupported x86 condition code %q", code) +} diff --git a/amd64_lower_cmpbt.go b/amd64_lower_cmpbt.go index 3b9369859..25def1522 100644 --- a/amd64_lower_cmpbt.go +++ b/amd64_lower_cmpbt.go @@ -6,179 +6,80 @@ import ( ) func (c *amd64Ctx) lowerCmpBt(op Op, ins Instr) (ok bool, terminated bool, err error) { - switch op { - case "CMPB", "CMPW", "CMPL", "CMPQ": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 CMPQ expects 2 operands: %q", ins.Raw) - } - ty := I64 - switch op { - case "CMPB": - ty = I8 - case "CMPW": - ty = I16 - case "CMPL": - ty = I32 - case "CMPQ": - ty = I64 - } - a, err := c.evalIntSized(ins.Args[0], ty) - if err != nil { - return true, false, err - } - b, err := c.evalIntSized(ins.Args[1], ty) - if err != nil { - return true, false, err - } - c.setCmpFlagsSized(ty, a, b) - return true, false, nil - - case "TESTB", "TESTW", "TESTL", "TESTQ": - // TEST* a,b sets flags based on (a & b) without storing the result. - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects 2 operands: %q", op, ins.Raw) - } - ty := I64 - switch op { - case "TESTB": - ty = I8 - case "TESTW": - ty = I16 - case "TESTL": - ty = I32 - case "TESTQ": - ty = I64 - } - a, err := c.evalIntSized(ins.Args[0], ty) - if err != nil { - return true, false, err - } - b, err := c.evalIntSized(ins.Args[1], ty) - if err != nil { - return true, false, err - } - c.setTestFlagsSized(ty, a, b) - return true, false, nil - - case "BTQ": - // BTQ $imm, reg (sets CF to selected bit) - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 BTQ expects $imm, reg: %q", ins.Raw) - } - v, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - amt := ins.Args[0].Imm & 63 - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", sh, v, amt) - and := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 1\n", and, sh) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", cf, and) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - return true, false, nil - - case "BTSQ", "BTRQ": - // BTSQ/BTRQ src, dstReg|dstMem: CF = old bit, then set or - // clear the selected destination bit. - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) - } - var amt string - var registerIndex string - switch ins.Args[0].Kind { - case OpImm: - amt = fmt.Sprintf("%d", ins.Args[0].Imm&63) - case OpReg: - av, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - m := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, 63\n", m, av) - amt = "%" + m - registerIndex = av - default: - return true, false, fmt.Errorf("amd64 %s expects imm/reg bit index: %q", op, ins.Raw) + if ok, terminated, err := c.lowerBitTestFamily(op, ins); ok { + return ok, terminated, err + } + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86ScalarCompareProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + for index, operand := range form.args { + if operand.Kind == OpReg && operand.Reg == SP && ins.Args[index].Reg != SP { + return true, false, fmt.Errorf("%w: Go REX byte view needs the original physical SP", ErrProbeNeedsContext) } - - var dst string - var storeDst func(string) error - switch ins.Args[1].Kind { - case OpReg: - dv, err := c.loadReg(ins.Args[1].Reg) - if err != nil { - return true, false, err + } + native := form.memory >= 0 && (x86OrdinaryNativeMemoryOperand(form.args[form.memory], arch) || + x86CallerMemoryCertified(ins, form.memory) || c.fixedLocalMemoryCertified(ins, form.memory)) + if native && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return true, false, fmt.Errorf("%w: scalar comparison needs its original native address size", ErrProbeNeedsContext) + } + typ := LLVMType(fmt.Sprintf("i%d", form.bits)) + var values [2]string + for index, operand := range form.args { + if index == form.memory && native { + pointer, pointerType, _, pointerErr := c.packedVectorMemoryPointer(operand) + if pointerErr != nil { + return true, false, pointerErr } - dst = dv - storeDst = func(v string) error { return c.storeReg(ins.Args[1].Reg, v) } - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) + loaded := c.newTmp() + // A compare whose flags are dead must still perform the exact + // native read (and fault). No backing/alignment assumption follows. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", loaded, typ, pointerType, pointer) + values[index] = "%" + loaded + } else { + values[index], err = c.evalIntSized(operand, typ) if err != nil { return true, false, err } - // A register bit index on a memory operand selects a bit in an - // unbounded bit string, not merely within the first qword. Adjust - // the address by floor(index/64)*8 and retain index&63 within it. - if registerIndex != "" { - word := c.newTmp() - fmt.Fprintf(c.b, " %%%s = ashr i64 %s, 6\n", word, registerIndex) - byteOff := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, 3\n", byteOff, word) - adjusted := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i64 %s, %%%s\n", adjusted, addr, byteOff) - addr = "%" + adjusted - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - dst = "%" + ld - storeDst = func(v string) error { - fmt.Fprintf(c.b, " store i64 %s, ptr %s, align 1\n", v, p) - return nil - } - default: - return true, false, fmt.Errorf("amd64 %s expects reg/mem dst: %q", op, ins.Raw) } - - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %s\n", sh, dst, amt) - and := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 1\n", and, sh) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", cf, and) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - one := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i64 1, %s\n", one, amt) - out := c.newTmp() - if op == "BTSQ" { - fmt.Fprintf(c.b, " %%%s = or i64 %s, %%%s\n", out, dst, one) - } else { - mask := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %%%s, -1\n", mask, one) - fmt.Fprintf(c.b, " %%%s = and i64 %s, %%%s\n", out, dst, mask) - } - if err := storeDst("%" + out); err != nil { - return true, false, err - } - return true, false, nil } - return false, false, nil + if form.test { + c.setTestFlagsSized(typ, values[0], values[1]) + } else { + c.setCmpFlagsSized(typ, values[0], values[1]) + } + return true, false, nil } func (c *amd64Ctx) setCmpFlagsSized(ty LLVMType, a, b string) { // Unlike most Plan 9 x86 instructions, CMP spells its operands in // comparison order: CMP a,b sets flags for a-b. + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", result, ty, a, b) + c.setAuxiliaryCarryFlag(ty, a, b, "%"+result) zt := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, %s\n", zt, ty, a, b) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zt, c.flagsZSlot) - slt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, %s\n", slt, ty, a, b) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", sign, ty, result) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) ult := c.newTmp() fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", ult, ty, a, b) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", ult, c.flagsCFSlot) + xorOperands := c.newTmp() + xorResult := c.newTmp() + overflowBits := c.newTmp() + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", xorOperands, ty, a, b) + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", xorResult, ty, a, result) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", overflowBits, ty, xorOperands, xorResult) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", overflow, ty, overflowBits) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + c.setParityFlagSized(ty, "%"+result) } func (c *amd64Ctx) setTestFlagsSized(ty LLVMType, a, b string) { @@ -192,9 +93,14 @@ func (c *amd64Ctx) setTestFlagsSized(ty LLVMType, a, b string) { fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, 0\n", slt, ty, and) fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", slt, c.flagsSltSlot) fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + c.setParityFlagSized(ty, "%"+and) } func (c *amd64Ctx) evalIntSized(op Operand, ty LLVMType) (string, error) { + if err := unresolvedSymbolicImmediateErrorForArch(op, ArchAMD64); err != nil { + return "", err + } switch op.Kind { case OpImm: // LLVM will interpret the literal in the destination integer width. @@ -233,6 +139,18 @@ func (c *amd64Ctx) evalIntSized(op Operand, ty LLVMType) (string, error) { s := strings.TrimSpace(op.Sym) if strings.HasPrefix(s, "$") { address := strings.TrimSpace(strings.TrimPrefix(s, "$")) + if memory, matched, err := parseX86SymbolIndexedMemory(address); matched { + if err != nil { + return "", err + } + wide, err := c.addrFromSymbolIndexedAddress(memory) + if err != nil || ty == I64 { + return wide, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", value, wide, ty) + return "%" + value, nil + } if strings.HasSuffix(address, "(SB)") { p, err := c.ptrFromSB(address) if err != nil { @@ -247,10 +165,14 @@ func (c *amd64Ctx) evalIntSized(op Operand, ty LLVMType) (string, error) { fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to %s\n", value, wide, ty) return "%" + value, nil } - // Macro-style symbolic immediates (e.g. $const_avxSupported) may - // survive preprocessing when include constants are unavailable. - // Keep translation progressing with a conservative zero value. - return "0", nil + return "", fmt.Errorf("unresolved symbolic immediate %q", op.Sym) + } + // Go's x86 syntax treats a bare integer as absolute memory. Keep + // this distinct from the '$' immediate spelling above. + if address, parseErr := parseInt(s); parseErr == nil { + t := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", t, ty, c.ptrFromAddrI64(fmt.Sprintf("%d", address))) + return "%" + t, nil } if strings.HasSuffix(s, "(SB)") { p, err := c.ptrFromSB(s) diff --git a/amd64_lower_cmps_test.go b/amd64_lower_cmps_test.go new file mode 100644 index 000000000..a7ba9a617 --- /dev/null +++ b/amd64_lower_cmps_test.go @@ -0,0 +1,231 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86CMPSCompleteFormsSource(goarch string) string { + widths := []string{"B", "W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + var source strings.Builder + source.WriteString("TEXT cmpsforms(SB),$0-0\n") + for _, width := range widths { + op := "CMPS" + width + fmt.Fprintf(&source, "\t%s\n", op) + fmt.Fprintf(&source, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&source, "\tSTD; REPN; %s\n", op) + source.WriteString("\tCLD\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86CMPSCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86CMPSCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"cmpsforms": {Name: "cmpsforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, width := range []string{"b", "w", "l"} { + if want := "@__plan9asm_rep_cmps" + width; !strings.Contains(ir, want) { + t.Fatalf("CMPS lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@__plan9asm_rep_cmpsq") { + t.Fatalf("CMPSQ lowering omitted qword helper:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "cmps-"+target.name+".ll", "cmps-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86CMPSRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CMPSB AX"}, + {goarch: "amd64", instruction: "CMPSW AX, BX"}, + {goarch: "amd64", instruction: "CMPSL.Z"}, + {goarch: "386", instruction: "CMPSQ"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's CMPS ynone table", test.instruction) + } + }) + } +} + +func TestAMD64CMPSRuntimeSemanticsPrefixesDirectionAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT cmpssemantics(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), R8 + MOVQ left+8(FP), R9 + MOVQ right+16(FP), R10 + + MOVQ R9, SI + MOVQ R10, DI + MOVQ $4, CX + CLD + REP; CMPSB + MOVQ SI, 0(R8) + MOVQ DI, 8(R8) + MOVQ CX, 16(R8) + SETCS 24(R8) + SETEQ 25(R8) + SETMI 26(R8) + SETOS 27(R8) + SETPS 28(R8) + + MOVQ R9, SI + ADDQ $8, SI + MOVQ R10, DI + ADDQ $8, DI + MOVQ $3, CX + REPN; CMPSW + MOVQ SI, 32(R8) + MOVQ DI, 40(R8) + MOVQ CX, 48(R8) + SETCS 56(R8) + SETEQ 57(R8) + + MOVQ R9, SI + ADDQ $28, SI + MOVQ R10, DI + ADDQ $28, DI + STD + CMPSL + CLD + MOVQ SI, 64(R8) + MOVQ DI, 72(R8) + SETCS 80(R8) + SETEQ 81(R8) + SETMI 82(R8) + SETOS 83(R8) + + MOVQ R9, SI + ADDQ $40, SI + MOVQ R10, DI + ADDQ $40, DI + CMPSQ + MOVQ SI, 88(R8) + MOVQ DI, 96(R8) + SETEQ 104(R8) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "cmpssemantics": { + Name: "cmpssemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void cmpssemantics(uint8_t *, uint8_t *, uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t left[56] = {1,2,9,4}; + uint8_t right[56] = {1,2,3,4}; + const uint16_t leftw[3] = {5,7,8}, rightw[3] = {6,7,9}; + memcpy(left + 8, leftw, sizeof(leftw)); + memcpy(right + 8, rightw, sizeof(rightw)); + const uint32_t leftl[2] = {0,1}, rightl[2] = {0,2}; + memcpy(left + 24, leftl, sizeof(leftl)); + memcpy(right + 24, rightl, sizeof(rightl)); + const uint64_t q = UINT64_C(0x0123456789abcdef); + memcpy(left + 40, &q, sizeof(q)); + memcpy(right + 40, &q, sizeof(q)); + uint8_t out[112] = {0}; + cmpssemantics(out, left, right); + if (load64(out+0) != (uintptr_t)(left+3)) return 10; + if (load64(out+8) != (uintptr_t)(right+3)) return 11; + if (load64(out+16) != 1) return 12; + if (out[24] != 0 || out[25] != 0 || out[26] != 0 || out[27] != 0 || out[28] != 1) return 11; + if (load64(out+32) != (uintptr_t)(left+12) || load64(out+40) != (uintptr_t)(right+12) || load64(out+48) != 1) return 12; + if (out[56] != 0 || out[57] != 1) return 13; + if (load64(out+64) != (uintptr_t)(left+24) || load64(out+72) != (uintptr_t)(right+24)) return 14; + if (out[80] != 1 || out[81] != 0 || out[82] != 1 || out[83] != 0) return 15; + if (load64(out+88) != (uintptr_t)(left+48) || load64(out+96) != (uintptr_t)(right+48) || out[104] != 1) return 16; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "cmps_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_cmpxchg.go b/amd64_lower_cmpxchg.go new file mode 100644 index 000000000..3cf59ce4b --- /dev/null +++ b/amd64_lower_cmpxchg.go @@ -0,0 +1,266 @@ +package plan9asm + +import "fmt" + +// lowerCompareExchange implements every operand row in Go 1.27's yrb_mb, +// yrl_ml, and yscond tables for the CMPXCHG family. +func (c *amd64Ctx) lowerCompareExchange(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, recognized, err := x86CompareExchangeProperties(op, ins, c.goarch) + if !recognized { + return false, false, nil + } + if err != nil { + return true, false, err + } + if form.spec.pair { + return c.lowerPairCompareExchange(string(op), form.spec.bits, ins) + } + return c.lowerScalarCompareExchange(string(op), form.spec.bits, ins) +} + +func amd64CompareExchangeProperties(op string) (kind string, bits int, ok bool) { + spec, recognized := x86CompareExchangeSpecs[Op(op)] + if !recognized { + return "", 0, false + } + kind = "scalar" + if spec.pair { + kind = "pair" + } + return kind, spec.bits, true +} + +func (c *amd64Ctx) lowerScalarCompareExchange(baseOp string, bits int, ins Instr) (bool, bool, error) { + form, recognized, err := x86CompareExchangeProperties(Op(baseOp), ins, c.goarch) + if err != nil { + return true, false, err + } + if !recognized || form.spec.pair || form.spec.bits != bits { + return true, false, fmt.Errorf("%s %s has no matching scalar compare-exchange width", c.goarch, baseOp) + } + source, destination := form.source, form.destination + if err := c.validateCompareExchangeFP(destination, bits); err != nil { + return true, false, err + } + + typ := amd64IntegerTypeForBits(bits) + accumulator, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + expected, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) + if err != nil { + return true, false, err + } + desired, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + + var old, success string + if destination.Kind == OpReg { + old, err = c.evalIntSized(destination, typ) + if err != nil { + return true, false, err + } + equal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, %s\n", equal, typ, expected, old) + success = "%" + equal + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, %s %s, %s %s\n", selected, success, typ, desired, typ, old) + if err := c.storeRegSized(destination.Reg, typ, "%"+selected); err != nil { + return true, false, err + } + } else { + ptr, ptrType, err := c.compareExchangePointer(destination) + if err != nil { + return true, false, err + } + old, success = c.emitScalarCompareExchangeNative(bits, destination, ptr, ptrType, expected, desired) + } + + // On failure the accumulator receives the destination. On success it is not + // written at all. In particular, CMPXCHGL zero-extends EAX only on failure; + // a successful compare must preserve the original high half of RAX. + if destination.Kind != OpReg || !amd64CompareExchangeDestinationIsAccumulator(destination.Reg, bits) { + if err := c.storeScalarCompareExchangeAccumulator(accumulator, expected, old, success, typ); err != nil { + return true, false, err + } + } + difference := c.newTmp() + // CMPXCHG compares the accumulator with the destination, so its arithmetic + // flags come from accumulator - destination (Intel's temporary result). + fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", difference, typ, expected, old) + c.setScalarAddSubFlags(typ, false, expected, old, "%"+difference) + // Keep the equality result authoritative even for unusual aliasing forms. + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", success, c.flagsZSlot) + return true, false, nil +} + +func (c *amd64Ctx) storeScalarCompareExchangeAccumulator(accumulator, expected, failureValue, success string, typ LLVMType) error { + if typ == I32 { + failure64 := c.newTmp() + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", failure64, failureValue) + fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %s, i64 %%%s\n", selected, success, accumulator, failure64) + return c.storeReg(AX, "%"+selected) + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, %s %s, %s %s\n", selected, success, typ, expected, typ, failureValue) + return c.storeRegSized(AX, typ, "%"+selected) +} + +func amd64CompareExchangeDestinationIsAccumulator(reg Reg, bits int) bool { + if bits == 8 { + return reg == AX || reg == AL + } + return reg == AX +} + +func (c *amd64Ctx) lowerPairCompareExchange(baseOp string, bits int, ins Instr) (bool, bool, error) { + form, recognized, err := x86CompareExchangeProperties(Op(baseOp), ins, c.goarch) + if err != nil { + return true, false, err + } + if !recognized || !form.spec.pair || form.spec.bits != bits { + return true, false, fmt.Errorf("%s %s has no matching pair compare-exchange width", c.goarch, baseOp) + } + destination := form.destination + if err := c.validateCompareExchangeFP(destination, bits); err != nil { + return true, false, err + } + + ax, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + dx, err := c.loadReg(DX) + if err != nil { + return true, false, err + } + bx, err := c.loadReg(BX) + if err != nil { + return true, false, err + } + cx, err := c.loadReg(CX) + if err != nil { + return true, false, err + } + ptr, ptrType, err := c.compareExchangePointer(destination) + if err != nil { + return true, false, err + } + old, success := c.emitPairCompareExchangeNative(bits, destination, ptr, ptrType, ax, dx, bx, cx) + + oldLow, oldHigh := c.compareExchangeSplitPair(old, bits) + if err := c.storeCompareExchangePairAccumulator(AX, ax, oldLow, success, bits); err != nil { + return true, false, err + } + if err := c.storeCompareExchangePairAccumulator(DX, dx, oldHigh, success, bits); err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", success, c.flagsZSlot) + return true, false, nil +} + +func (c *amd64Ctx) validateCompareExchangeFP(destination Operand, bits int) error { + if destination.Kind != OpFP { + return nil + } + context := func(reason string) error { + return fmt.Errorf("%w: %s CMPXCHG FP native write at +%d(FP): %s", ErrProbeNeedsContext, c.goarch, destination.FPOffset, reason) + } + if bits == 128 { + return context("logical FP storage has no original CMPXCHG16B native address/alignment identity") + } + width := bits / 8 + if err := c.validateFPVectorSpan(destination.FPOffset, width); err != nil { + return context(err.Error()) + } + if c.classicFrame != "" { + return nil + } + slot, _, ok := c.fpVectorSlot(destination.FPOffset) + if !ok || slot.Offset != destination.FPOffset || x86FrameTypeSize(slot.Type) < int64(width) { + return context("native access is not an existing complete contiguous shadow slot") + } + return nil +} + +func (c *amd64Ctx) compareExchangeJoinPair(low, high string, bits int) string { + partType := LLVMType("i32") + pairType := LLVMType("i64") + shift := 32 + if bits == 128 { + partType = I64 + pairType = LLVMType("i128") + shift = 64 + } + lowPart, highPart := low, high + if partType == I32 { + lowPart = c.truncI64(low, I32) + highPart = c.truncI64(high, I32) + } + lowWide := c.newTmp() + highWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", lowWide, partType, lowPart, pairType) + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", highWide, partType, highPart, pairType) + shifted := c.newTmp() + joined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %d\n", shifted, pairType, highWide, shift) + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", joined, pairType, shifted, lowWide) + return "%" + joined +} + +func (c *amd64Ctx) compareExchangeSplitPair(pair string, bits int) (low, high string) { + pairType := LLVMType("i64") + partType := LLVMType("i32") + shift := 32 + if bits == 128 { + pairType = LLVMType("i128") + partType = I64 + shift = 64 + } + lowName := c.newTmp() + highShift := c.newTmp() + highName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", lowName, pairType, pair, partType) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %d\n", highShift, pairType, pair, shift) + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", highName, pairType, highShift, partType) + return "%" + lowName, "%" + highName +} + +func (c *amd64Ctx) storeCompareExchangePairAccumulator(reg Reg, original, failurePart, success string, bits int) error { + failure64 := failurePart + if bits == 64 { + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", widened, failurePart) + failure64 = "%" + widened + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i64 %s, i64 %s\n", selected, success, original, failure64) + return c.storeReg(reg, "%"+selected) +} + +func (c *amd64Ctx) compareExchangePointer(destination Operand) (ptr, ptrType string, err error) { + switch destination.Kind { + case OpMem: + return c.ptrFromMem(destination.Mem) + case OpSym: + ptr, err := c.ptrFromSB(destination.Sym) + return ptr, "ptr", err + case OpFP: + if c.classicFrame != "" { + return c.classicFramePtr(destination.FPOffset), "ptr", nil + } + if ptr := c.fpParamAlloca[destination.FPOffset]; ptr != "" { + return ptr, "ptr", nil + } + if ptr, _, ok := c.fpResultAlloca(destination.FPOffset); ok { + return ptr, "ptr", nil + } + return "", "", fmt.Errorf("CMPXCHG FP operand has no mutable slot at +%d(FP)", destination.FPOffset) + default: + return "", "", fmt.Errorf("CMPXCHG expected memory destination, got %s", destination.String()) + } +} diff --git a/amd64_lower_compare_float.go b/amd64_lower_compare_float.go new file mode 100644 index 000000000..e5aad5888 --- /dev/null +++ b/amd64_lower_compare_float.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" +) + +// lowerFloatingCompare implements the complete Go 1.27 floating-point +// immediate-compare family: legacy CMP{PD,PS,SD,SS}, the VEX vector-result +// forms, and the EVEX K-result forms of VCMP{PD,PS,SD,SS}. +func (c *amd64Ctx) lowerFloatingCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86FloatCompareSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("floating compare dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86FloatCompareProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + return true, false, c.emitNativeFloatingCompare(ins, form) +} + +func amd64FloatingCompareProperties(op string) (elemBits int, packed, vector, ok bool) { + switch op { + case "CMPPD": + return 64, true, false, true + case "CMPPS": + return 32, true, false, true + case "CMPSD": + return 64, false, false, true + case "CMPSS": + return 32, false, false, true + case "VCMPPD": + return 64, true, true, true + case "VCMPPS": + return 32, true, true, true + case "VCMPSD": + return 64, false, true, true + case "VCMPSS": + return 32, false, true, true + default: + return 0, false, false, false + } +} + +func amd64LegacyFloatingCompareImmediate(arg Operand) (int64, bool) { + if arg.Kind == OpImm { + return arg.Imm, true + } + // The pre-Go-1.5 x86 assembler wrote CMP{PD,PS,SD,SS} predicates as a + // bare decimal operand. Keep this compatibility confined to the legacy + // compare family's predicate position instead of accepting bare integers + // as immediates throughout the parser. + var text string + switch arg.Kind { + case OpIdent: + text = arg.Ident + case OpSym: + text = arg.Sym + default: + return 0, false + } + value, err := strconv.ParseInt(strings.TrimSpace(text), 10, 64) + return value, err == nil +} + +func (c *amd64Ctx) isGoLegacyXReg(r Reg) bool { + index, ok := amd64ParseXReg(r) + if !ok { + return false + } + if c.goarch == "386" { + return index < 8 + } + return index < 16 +} + +func dstArgOperand(r Reg) Operand { + return Operand{Kind: OpReg, Reg: r} +} + +func amd64VEXVectorRegister(op Operand, byteWidth int) bool { + if op.Kind != OpReg || amd64VectorByteWidth(op.Reg) != byteWidth { + return false + } + index, ok := amd64VectorRegisterIndex(op.Reg, byteWidth) + return ok && index < 16 +} + +func amd64EVEXVectorRegister(op Operand, byteWidth int) bool { + if op.Kind != OpReg || amd64VectorByteWidth(op.Reg) != byteWidth { + return false + } + index, ok := amd64VectorRegisterIndex(op.Reg, byteWidth) + return ok && index < 32 +} + +func amd64VectorRegisterIndex(r Reg, byteWidth int) (int, bool) { + switch byteWidth { + case 16: + return amd64ParseXReg(r) + case 32: + return amd64ParseYReg(r) + case 64: + return amd64ParseZReg(r) + default: + return 0, false + } +} + +func amd64FloatingComparePredicate(immediate uint8) string { + return [...]string{ + "oeq", "olt", "ole", "uno", "une", "uge", "ugt", "ord", + "ueq", "ult", "ule", "false", "one", "oge", "ogt", "true", + "oeq", "olt", "ole", "uno", "une", "uge", "ugt", "ord", + "ueq", "ult", "ule", "false", "one", "oge", "ogt", "true", + }[immediate&31] +} + +func (c *amd64Ctx) loadFloatingVectorOperand(op Operand, byteWidth, lanes, elemBits int) (string, error) { + bytesValue, err := c.loadPackedCompareBytes(op, byteWidth) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", value, byteWidth, bytesValue, amd64FloatingVectorType(lanes, elemBits)) + return "%" + value, nil +} + +func (c *amd64Ctx) loadFloatingScalarOperand(op Operand, elemBits int) (string, error) { + if elemBits == 64 { + return c.evalF64(op) + } + return c.evalF32(op) +} + +func (c *amd64Ctx) splatFloatingScalar(value string, lanes, elemBits int) string { + vectorType := amd64FloatingVectorType(lanes, elemBits) + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %s, i32 %d\n", inserted, vectorType, result, amd64FloatingScalarType(elemBits), value, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) packFloatingCompareMask(compared string, lanes int) string { + if lanes == 1 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i64\n", wide, compared) + return "%" + wide + } + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %s to %s\n", packed, lanes, compared, packedType) + if lanes == 64 { + return "%" + packed + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %%%s to i64\n", wide, packedType, packed) + return "%" + wide +} + +func amd64FloatingScalarType(elemBits int) string { + if elemBits == 64 { + return "double" + } + return "float" +} + +func amd64FloatingVectorType(lanes, elemBits int) string { + return fmt.Sprintf("<%d x %s>", lanes, amd64FloatingScalarType(elemBits)) +} diff --git a/amd64_lower_compare_float_historical_test.go b/amd64_lower_compare_float_historical_test.go new file mode 100644 index 000000000..f7b50dad3 --- /dev/null +++ b/amd64_lower_compare_float_historical_test.go @@ -0,0 +1,52 @@ +package plan9asm + +import "testing" + +func TestTranslateAMD64LegacyFloatingCompareHistoricalBarePredicate(t *testing.T) { + source := "TEXT historicalcompare(SB),NOSPLIT,$0-0\n" + + "\tCMPPD X1, X0, 1\n" + + "\tCMPPS 8(AX), X0, 2\n" + + "\tCMPSD X1, X0, 5\n" + + "\tCMPSS 8(AX), X0, -1\n" + + "\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "historicalcompare": {Name: "historicalcompare", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "historical-floating-compare.ll", "historical-floating-compare.o", ll) +} + +func TestTranslateAMD64LegacyFloatingCompareRejectsInvalidBarePredicate(t *testing.T) { + for _, instruction := range []string{ + "CMPPD X1, X0, predicate", + "CMPPS X1, X0, 128", + "CMPSD X1, X0, -129", + "CMPSS X1, X0, 1+1", + } { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + continue + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted invalid historical predicate %q", instruction) + } + } +} diff --git a/amd64_lower_compare_greater_test.go b/amd64_lower_compare_greater_test.go new file mode 100644 index 000000000..d1d660505 --- /dev/null +++ b/amd64_lower_compare_greater_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedSignedGreaterThanCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, vectorLeft, vectorRight, zLast := 15, 20, 21, 22 + if target.goarch == "386" { + legacyLast, vectorLeft, vectorRight, zLast = 7, 6, 7, 7 + } + var source strings.Builder + source.WriteString("TEXT packedsignedgreaterthanforms(SB),4,$0-0\n") + for _, op := range []string{"PCMPGTB", "PCMPGTW", "PCMPGTL"} { + // Py1's MMX encoder row is amd64-only. A frontend-accepted + // high vector spelling is likewise not a physical 386 bank. + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M2\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + } + fmt.Fprintf(&source, "\tPCMPGTQ X0, X%d\n", legacyLast) + fmt.Fprintf(&source, "\tPCMPGTQ 8(AX), X%d\n", legacyLast) + for _, op := range []string{"VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s1, %s2\n", op, width, width) + fmt.Fprintf(&source, "\t%s %s%d, %s%d, K3\n", op, width, vectorLeft, width, vectorRight) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, K3\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, K3\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, K0\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K2, K3\n", op) + } else { + fmt.Fprintf(&source, "\t%s Z0, Z6, K%d\n", op, zLast) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s.BCST 8(AX), %s%d, K3\n", op, width, vectorRight) + } + } + } + source.WriteString("\tRET\n") + x86LEAActualTargetGoObject(t, target.goarch, target.triple, source.String()) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedsignedgreaterthanforms": {Name: "packedsignedgreaterthanforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-signed-greater-than-"+target.name+".ll", "packed-signed-greater-than-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedSignedGreaterThanRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PCMPGTB.Z X0, X1", + "PCMPGTW X0", + "PCMPGTL M0, X1", + "PCMPGTQ M0, M1", + "PCMPGTB Y0, Y1", + "PCMPGTW X0, (AX)", + "VPCMPGTB X0, X1", + "VPCMPGTW X0, Y1, Y2", + "VPCMPGTD Z0, Z1, Z2", + "VPCMPGTQ X0, X1, K0, K2", + "VPCMPGTB.Z X0, X1, K2", + "VPCMPGTW.BCST 8(AX), X1, K2", + "VPCMPGTD.BCST X0, X1, K2", + "VPCMPGTQ.BCST 8(AX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedSignedGreaterThanRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PCMPGTW X8, X0", + "PCMPGTL X0, X8", + "VPCMPGTB X16, X1, X2", + "VPCMPGTW X0, X1, X16", + "VPCMPGTD Z8, Z1, K2", + "VPCMPGTQ Z0, Z1, K1, K2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedSignedGreaterThanRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } + for _, op := range []string{"PCMPGTB", "PCMPGTW", "PCMPGTL"} { + for _, operands := range []string{"M0, M1", "8(AX), M2"} { + instruction := op + " " + operands + goAssemblerOracleOutput(t, "386", "TEXT bad(SB),4,$0-0\n"+instruction+"\nRET\n", "", false) + assertX86PackedSignedGreaterThanRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } + } +} + +func assertX86PackedSignedGreaterThanRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed signed greater-than forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_compare_legacy_equal_test.go b/amd64_lower_compare_legacy_equal_test.go new file mode 100644 index 000000000..709e481b6 --- /dev/null +++ b/amd64_lower_compare_legacy_equal_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86LegacyPackedEqualityActualGoEncoderRows(t *testing.T) { + body, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + actual := strings.Join(strings.Fields(string(body)), "") + for _, definition := range []string{ + "var ymm = []ytab{ {Zm_r_xm, 1, argList{Ymm, Ymr}}, {Zm_r_xm, 2, argList{Yxm, Yxr}}, }", + "var yxm_q4 = []ytab{ {Zm_r, 1, argList{Yxm, Yxr}}, }", + "{APCMPEQB, ymm, Py1, opBytes{0x74, Pe, 0x74}}", + "{APCMPEQW, ymm, Py1, opBytes{0x75, Pe, 0x75}}", + "{APCMPEQL, ymm, Py1, opBytes{0x76, Pe, 0x76}}", + "{APCMPEQQ, yxm_q4, Pq4, opBytes{0x29}}", + } { + if !strings.Contains(actual, strings.Join(strings.Fields(definition), "")) { + t.Fatalf("selected Go SDK packed-equality encoding row changed: %s", definition) + } + } +} + +func TestTranslateX86LegacyPackedEqualityCompleteGoAssemblerForms(t *testing.T) { + // PCMPEQ{B,W,L} use Go's ymm table (MMX/memory -> MMX and X/memory + // -> X). Py1 admits the MMX row only in amd64; 386 must use X0-X7. + // PCMPEQQ uses yxm_q4, which has only the X/memory -> X row. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastX := 15 + if target.goarch == "386" { + lastX = 7 + } + var source strings.Builder + source.WriteString("TEXT legacypackedequalityforms(SB),4,$0-0\n") + for _, op := range []string{"PCMPEQB", "PCMPEQW", "PCMPEQL"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M2\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, lastX) + fmt.Fprintf(&source, "\t%s 16(AX), X%d\n", op, lastX) + } + fmt.Fprintf(&source, "\tPCMPEQQ X0, X%d\n", lastX) + fmt.Fprintf(&source, "\tPCMPEQQ 24(AX), X%d\n", lastX) + source.WriteString("\tRET\n") + x86LEAActualTargetGoObject(t, target.goarch, target.triple, source.String()) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "legacypackedequalityforms": {Name: "legacypackedequalityforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "legacy-packed-equality-"+target.name+".ll", "legacy-packed-equality-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86LegacyPackedEqualityRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PCMPEQQ M0, M1", + "PCMPEQQ X0, M1", + "PCMPEQW M0, X1", + "PCMPEQL X0, M1", + "PCMPEQB Y0, X1", + "PCMPEQQ X0, Y1", + "PCMPEQQ.Z X0, X1", + "PCMPEQQ X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86LegacyPackedEqualityRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86LegacyPackedEqualityRejected(t, "386", "i386-unknown-linux-gnu", "PCMPEQQ X8, X0") + assertX86LegacyPackedEqualityRejected(t, "386", "i386-unknown-linux-gnu", "PCMPEQQ X0, X8") + for _, op := range []string{"PCMPEQB", "PCMPEQW", "PCMPEQL"} { + for _, operands := range []string{"M0, M1", "8(AX), M2"} { + assertX86LegacyPackedEqualityRejected(t, "386", "i386-unknown-linux-gnu", op+" "+operands) + } + } +} + +func assertX86LegacyPackedEqualityRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + source := "TEXT bad(SB),4,$0-0\n\t" + instruction + "\n\tRET\n" + goAssemblerOracleOutput(t, goarch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil || ir != "" { + t.Fatalf("Translate accepted %q outside Go 1.27's legacy packed equality forms", instruction) + } +} diff --git a/amd64_lower_compare_mask.go b/amd64_lower_compare_mask.go new file mode 100644 index 000000000..5605d35a4 --- /dev/null +++ b/amd64_lower_compare_mask.go @@ -0,0 +1,156 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ImmediatePackedCompareSpec struct { + laneBits int + unsigned bool +} + +var amd64ImmediatePackedCompareSpecs = map[string]amd64ImmediatePackedCompareSpec{ + "VPCMPB": {laneBits: 8}, + "VPCMPW": {laneBits: 16}, + "VPCMPD": {laneBits: 32}, + "VPCMPQ": {laneBits: 64}, + "VPCMPUB": {laneBits: 8, unsigned: true}, + "VPCMPUW": {laneBits: 16, unsigned: true}, + "VPCMPUD": {laneBits: 32, unsigned: true}, + "VPCMPUQ": {laneBits: 64, unsigned: true}, +} + +// lowerImmediatePackedCompare implements the complete Go 1.27 _yvpcmpb +// operand table and all eight signed/unsigned element-width mnemonics. +func (c *amd64Ctx) lowerImmediatePackedCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + broadcast := false + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + if rawOp[dot+1:] != "BCST" { + if _, known := amd64ImmediatePackedCompareSpecs[baseOp]; known { + return true, false, fmt.Errorf("amd64 %s accepts only the .BCST suffix: %q", baseOp, ins.Raw) + } + return false, false, nil + } + broadcast = true + } + spec, ok := amd64ImmediatePackedCompareSpecs[baseOp] + if !ok { + return false, false, nil + } + // cmd/asm's 386 parser cannot represent the four- and five-operand + // spellings used by this shared x86 table. + if c.goarch == "386" { + return true, false, fmt.Errorf("386 %s exceeds the Go assembler frontend's operand limit: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 4 && len(ins.Args) != 5 { + return true, false, fmt.Errorf("amd64 %s expects $imm8, first source, second source, [K mask,] K destination: %q", baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].ImmRaw != "" || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("amd64 %s expects an unsigned-byte immediate: %q", baseOp, ins.Raw) + } + + second := ins.Args[2] + byteWidth, validWidth := amd64PackedCompareWidth(second) + if !validWidth || !c.isGoPackedVectorMoveRegister(second, byteWidth) { + return true, false, fmt.Errorf("amd64 %s second source must be an in-range X, Y, or Z register: %q", baseOp, ins.Raw) + } + first := ins.Args[1] + if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("amd64 %s register sources must have matching widths: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s first source must be a matching vector register or memory: %q", baseOp, ins.Raw) + } + if broadcast { + if spec.laneBits != 32 && spec.laneBits != 64 { + return true, false, fmt.Errorf("amd64 %s does not enable broadcast in Go 1.27: %q", baseOp, ins.Raw) + } + if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory first source: %q", baseOp, ins.Raw) + } + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s destination must be K0-K7: %q", baseOp, ins.Raw) + } + if _, validDestination := amd64ParseKReg(destination.Reg); !validDestination { + return true, false, fmt.Errorf("amd64 %s destination must be K0-K7: %q", baseOp, ins.Raw) + } + + writeMask := "" + if len(ins.Args) == 5 { + mask := ins.Args[3] + if mask.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s write mask must be K1-K7: %q", baseOp, ins.Raw) + } + maskIndex, validMask := amd64ParseKReg(mask.Reg) + if !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s write mask must be K1-K7: %q", baseOp, ins.Raw) + } + writeMask, err = c.loadK(mask.Reg) + if err != nil { + return true, false, err + } + } + + firstValue, err := c.loadPackedCompareLanes(first, byteWidth, spec.laneBits, broadcast) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + compared := c.emitImmediatePackedComparePredicate(lanes, spec.laneBits, secondValue, firstValue, uint8(ins.Args[0].Imm), spec.unsigned) + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %s to %s\n", packed, lanes, compared, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if writeMask != "" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, writeMask) + result = "%" + masked + } + return true, false, c.storeK(destination.Reg, result) +} + +func (c *amd64Ctx) emitImmediatePackedComparePredicate(lanes, laneBits int, left, right string, immediate uint8, unsigned bool) string { + switch immediate & 7 { + case 3: + return "zeroinitializer" + case 7: + return llvmAllTrueI1VectorValue(lanes) + } + predicates := [8]string{"eq", "slt", "sle", "", "ne", "sge", "sgt", ""} + if unsigned { + predicates = [8]string{"eq", "ult", "ule", "", "ne", "uge", "ugt", ""} + } + compared := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicates[immediate&7], lanes, laneBits, left, right) + return "%" + compared +} + +func llvmAllTrueI1VectorValue(lanes int) string { + var value strings.Builder + value.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + value.WriteString(", ") + } + value.WriteString("i1 true") + } + value.WriteByte('>') + return value.String() +} diff --git a/amd64_lower_compare_mask_test.go b/amd64_lower_compare_mask_test.go new file mode 100644 index 000000000..b4a078102 --- /dev/null +++ b/amd64_lower_compare_mask_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ImmediatePackedCompareCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives VPCMP{B,W,D,Q,UB,UW,UD,UQ} the same _yvpcmpb + // table: unsigned-byte immediate, X/Y/Z r/m first source, matching vector + // second source, optional K1-K7 write mask, and K destination. D/Q forms + // additionally enable scalar-memory broadcast. + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT immediatepackedcompareforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "VPCMPB", "VPCMPW", "VPCMPD", "VPCMPQ", + "VPCMPUB", "VPCMPUW", "VPCMPUD", "VPCMPUQ", + } { + fmt.Fprintf(&source, "\t%s $0, X1, X20, K0\n", op) + fmt.Fprintf(&source, "\t%s $1, 8(AX), X20, K1, K2\n", op) + fmt.Fprintf(&source, "\t%s $2, Y1, Y20, K3\n", op) + fmt.Fprintf(&source, "\t%s $3, 40(AX), Y20, K4, K5\n", op) + fmt.Fprintf(&source, "\t%s $7, Z1, Z20, K6\n", op) + fmt.Fprintf(&source, "\t%s $255, 104(AX), Z20, K7, K0\n", op) + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST $4, 168(AX), X20, K1\n", op) + fmt.Fprintf(&source, "\t%s.BCST $5, 176(AX), Y20, K2, K3\n", op) + fmt.Fprintf(&source, "\t%s.BCST $6, 184(AX), Z20, K4\n", op) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "immediatepackedcompareforms": {Name: "immediatepackedcompareforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "immediate-packed-compare-"+target.name+".ll", "immediate-packed-compare-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ImmediatePackedCompareRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPCMPUD X0, X1, K1", + "VPCMPUD $256, X0, X1, K1", + "VPCMPUD $1, X0, Y1, K1", + "VPCMPUD $1, X0, (AX), K1", + "VPCMPUD $1, X0, X1, K0, K1", + "VPCMPUD $1, X0, X1, AX", + "VPCMPUD.Z $1, X0, X1, K1", + "VPCMPUB.BCST $1, (AX), X1, K1", + "VPCMPUW.BCST $1, (AX), X1, K1", + "VPCMPUD.BCST $1, X0, X1, K1", + "VPCMPUQ $1, X32, X1, K1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ImmediatePackedCompareRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + // cmd/asm's 386 frontend rejects this table's four- and five-operand + // spellings before its shared x86 optab is reached. + assertX86ImmediatePackedCompareRejected(t, "386", "i386-unknown-linux-gnu", "VPCMPUD $1, Y2, Y0, K2") +} + +func assertX86ImmediatePackedCompareRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's immediate packed compare forms", instruction) + } +} diff --git a/amd64_lower_compare_vec.go b/amd64_lower_compare_vec.go new file mode 100644 index 000000000..244885f6e --- /dev/null +++ b/amd64_lower_compare_vec.go @@ -0,0 +1,354 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedIntegerCompareSpec struct { + laneBits int + greater bool + mapNumber, opcode int +} + +// Every Go _yvpcmpeqb entry: X/Y VEX vector results, X/Y/Z EVEX K +// results, optional K1-7 mask, and memory-only D/Q broadcast. Raw decode +// consumes the same lane and encoding axes as the named lowerer. +var amd64PackedIntegerCompareSpecs = map[Op]amd64PackedIntegerCompareSpec{ + "VPCMPEQB": {8, false, 1, 0x74}, + "VPCMPEQW": {16, false, 1, 0x75}, + "VPCMPEQD": {32, false, 1, 0x76}, + "VPCMPEQQ": {64, false, 2, 0x29}, + "VPCMPGTB": {8, true, 1, 0x64}, + "VPCMPGTW": {16, true, 1, 0x65}, + "VPCMPGTD": {32, true, 1, 0x66}, + "VPCMPGTQ": {64, true, 2, 0x37}, +} + +// lowerPackedIntegerCompare implements the legacy PCMPGT{B,W,L,Q} family and +// the complete Go 1.27 _yvpcmpeqb operand family shared by VPCMPEQ{B,W,D,Q} +// and VPCMPGT{B,W,D,Q}. +// VEX forms write X/Y vectors. EVEX forms compare X/Y/Z lanes into a K +// register, optionally gated by a non-K0 mask; D/Q memory operands can be +// broadcast for the EVEX forms. +func (c *amd64Ctx) lowerPackedIntegerCompare(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedIntegerCompareProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + ins.Op = op + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if form.legacy { + local := ins + local.Args = append([]Operand(nil), ins.Args...) + local.Args[form.first] = form.source + if form.memory >= 0 && !form.mmx { + virtual := form.source.Kind == OpFP + if form.source.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(form.source, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed integer compare m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + return c.lowerLegacyPackedIntegerCompare(base, form.spec.laneBits, form.spec.mmx, local) + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.maskResult { + return true, false, c.emitPackedIntegerMaskCompare(base, form.spec.laneBits, form.byteWidth, + form.source, ins.Args[form.second], ins.Args[form.destination].Reg, mask, form.broadcast) + } + return true, false, c.emitPackedIntegerVectorCompare(base, form.spec.laneBits, form.byteWidth, + form.source, ins.Args[form.second], ins.Args[form.destination].Reg) +} + +func (c *amd64Ctx) lowerLegacyPackedIntegerCompare(baseOp string, laneBits int, allowMMX bool, ins Instr) (bool, bool, error) { + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects packed source, packed destination: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[1].Reg + if _, mmx := amd64ParseMReg(destination); mmx { + if !allowMMX { + return true, false, fmt.Errorf("%s %s has no MMX form in Go 1.27's yxm_q4 table: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if _, ok := amd64ParseMReg(ins.Args[0].Reg); !ok { + return true, false, fmt.Errorf("%s %s MMX form requires an MMX source: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s MMX form requires MMX or memory source: %q", c.goarch, baseOp, ins.Raw) + } + firstBits, err := c.loadLegacyMMXPackedSource(ins.Args[0]) + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(destination) + if err != nil { + return true, false, err + } + lanes := 64 / laneBits + first := c.bitcastI64ToIntegerLanes(lanes, laneBits, firstBits) + second := c.bitcastI64ToIntegerLanes(lanes, laneBits, secondBits) + result := c.emitLegacyPackedIntegerCompare(baseOp, lanes, laneBits, first, second) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", bits, lanes, laneBits, result) + return true, false, c.storeReg(destination, "%"+bits) + } + + if !c.isGoLegacyXReg(destination) { + return true, false, fmt.Errorf("%s %s destination must be an in-range MMX or X register: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg && !c.isGoLegacyXReg(ins.Args[0].Reg) { + return true, false, fmt.Errorf("%s %s XMM form requires an in-range X source: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpReg && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s XMM form requires X or memory source: %q", c.goarch, baseOp, ins.Raw) + } + secondBytes, err := c.loadX(destination) + if err != nil { + return true, false, err + } + lanes := 128 / laneBits + second := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, secondBytes) + var result string + if isAMD64MemoryOperand(ins.Args[0]) { + mnemonic := strings.ToLower(baseOp) + if laneBits == 32 { + mnemonic = strings.TrimSuffix(mnemonic, "l") + "d" + } + result, err = c.emitPackedBinaryNativeMemory(ins.Args[0], laneBits, second, mnemonic) + } else { + firstBytes, loadErr := c.loadXVecOperand(ins.Args[0]) + if loadErr != nil { + return true, false, loadErr + } + first := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, firstBytes) + result = c.emitLegacyPackedIntegerCompare(baseOp, lanes, laneBits, first, second) + } + if err != nil { + return true, false, err + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <16 x i8>\n", out, lanes, laneBits, result) + return true, false, c.storeLegacyPackedLogical(ins.Args[1], "%"+out) +} + +func (c *amd64Ctx) emitLegacyPackedIntegerCompare(baseOp string, lanes, laneBits int, first, second string) string { + compared := c.newTmp() + result := c.newTmp() + predicate := "eq" + if strings.HasPrefix(baseOp, "PCMPGT") { + predicate = "sgt" + } + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicate, lanes, laneBits, second, first) + fmt.Fprintf(c.b, " %%%s = sext <%d x i1> %%%s to <%d x i%d>\n", result, lanes, compared, lanes, laneBits) + return "%" + result +} + +func (c *amd64Ctx) isGoVEXVectorRegister(arg Operand, byteWidth int, allowMemory bool) bool { + if allowMemory && isAMD64MemoryOperand(arg) { + return true + } + if !amd64VEXVectorRegister(arg, byteWidth) { + return false + } + if c.goarch == "386" { + index, _ := amd64VectorRegisterIndex(arg.Reg, byteWidth) + return index < 8 + } + return true +} + +func amd64PackedCompareWidth(op Operand) (int, bool) { + if op.Kind != OpReg { + return 0, false + } + switch { + case isAMD64XReg(op.Reg): + return 16, true + case isAMD64YReg(op.Reg): + return 32, true + case isAMD64ZReg(op.Reg): + return 64, true + default: + return 0, false + } +} + +func (c *amd64Ctx) loadPackedCompareBytes(op Operand, byteWidth int) (string, error) { + if op.Kind == OpReg { + width, ok := amd64PackedCompareWidth(op) + if !ok || width != byteWidth { + return "", fmt.Errorf("expected %d-byte vector register, got %s", byteWidth, op.String()) + } + switch byteWidth { + case 16: + return c.loadX(op.Reg) + case 32: + return c.loadY(op.Reg) + case 64: + return c.loadZ(op.Reg) + } + } + if op.Kind == OpFP { + return c.loadFPVectorBytes(op.FPOffset, byteWidth) + } + if op.Kind == OpSym { + // As in Go's x86 assembler, an unprefixed integer denotes an + // absolute memory address rather than an immediate value. + if address, parseErr := parseInt(strings.TrimSpace(op.Sym)); parseErr == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load <%d x i8>, ptr %s, align 1\n", value, byteWidth, c.ptrFromAddrI64(fmt.Sprintf("%d", address))) + return "%" + value, nil + } + } + if !isAMD64MemoryOperand(op) { + return "", fmt.Errorf("expected vector register or memory, got %s", op.String()) + } + switch byteWidth { + case 16: + return c.loadXVecOperand(op) + case 32: + return c.loadYVecOperand(op) + case 64: + return c.loadZVecOperand(op) + default: + return "", fmt.Errorf("unsupported packed compare width %d", byteWidth) + } +} + +func (c *amd64Ctx) loadPackedCompareLanes(op Operand, byteWidth, laneBits int, broadcast bool) (string, error) { + lanes := byteWidth * 8 / laneBits + if broadcast { + value, err := c.evalIntSized(op, amd64IntegerTypeForBits(laneBits)) + if err != nil { + return "", err + } + return amd64SplatInteger(c, lanes, laneBits, value), nil + } + bytesValue, err := c.loadPackedCompareBytes(op, byteWidth) + if err != nil { + return "", err + } + lanesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", lanesValue, byteWidth, bytesValue, lanes, laneBits) + return "%" + lanesValue, nil +} + +func (c *amd64Ctx) comparePackedIntegerLanes(baseOp string, laneBits, byteWidth int, first, second Operand, broadcast bool, mask string) (string, error) { + // E4/E4.nb permits only inactive native lanes to suppress their accesses. + // Keep active reads observable even when the compare result is dead. + // Independent virtual FP fields still require the full checked byte span. + if first.Kind == OpFP { + width := byteWidth + if broadcast { + width = laneBits / 8 + } + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, width); err != nil { + return "", err + } + } + firstValue, err := c.loadX86PackedShiftLanes(first, byteWidth, laneBits, broadcast, mask) + if err != nil { + return "", err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, laneBits, false) + if err != nil { + return "", err + } + predicate := "eq" + if amd64PackedIntegerCompareSpecs[Op(baseOp)].greater { + predicate = "sgt" + } + lanes := byteWidth * 8 / laneBits + compared := c.newTmp() + // Go/Plan 9 lists the Intel r/m operand first: dst = second cmp first. + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicate, lanes, laneBits, secondValue, firstValue) + return "%" + compared, nil +} + +func (c *amd64Ctx) emitPackedIntegerVectorCompare(baseOp string, laneBits, byteWidth int, first, second Operand, dst Reg) error { + compared, err := c.comparePackedIntegerLanes(baseOp, laneBits, byteWidth, first, second, false, "") + if err != nil { + return err + } + lanes := byteWidth * 8 / laneBits + allBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext <%d x i1> %s to <%d x i%d>\n", allBits, lanes, compared, lanes, laneBits) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <%d x i8>\n", bytesValue, lanes, laneBits, allBits, byteWidth) + return c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: dst}, byteWidth, "%"+bytesValue) +} + +func (c *amd64Ctx) emitPackedIntegerMaskCompare(baseOp string, laneBits, byteWidth int, first, second Operand, dst Reg, writeMask string, broadcast bool) error { + compared, err := c.comparePackedIntegerLanes(baseOp, laneBits, byteWidth, first, second, broadcast, writeMask) + if err != nil { + return err + } + lanes := byteWidth * 8 / laneBits + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %s to %s\n", packed, lanes, compared, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if writeMask != "" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, writeMask) + result = "%" + masked + } + return c.storeK(dst, result) +} + +func (c *amd64Ctx) loadMaskedPackedCompareLanes(source Operand, byteWidth, laneBits int, broadcast bool, mask string) (string, error) { + // FP describes independent checked virtual slots, not one contiguous + // native allocation. Keep per-slot loads; never form a vector pointer + // from only the first slot, even when its address happens to be available. + if source.Kind == OpFP && !broadcast { + for offset := source.FPOffset; offset < source.FPOffset+int64(byteWidth); offset += 8 { + _, parameter := c.fpParam(offset) + _, _, result := c.fpResultAlloca(offset) + if !parameter && !result { + return "", fmt.Errorf("%s packed compare uses an undeclared FP vector slot at +%d(FP)", c.goarch, offset) + } + } + } + if mask == "" || source.Kind == OpReg || source.Kind == OpFP { + return c.loadPackedCompareLanes(source, byteWidth, laneBits, broadcast) + } + lanes := byteWidth * 8 / laneBits + if broadcast { + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)<", lanes, laneBits) + fmt.Fprintf(c.b, " %%%s = call %s @llvm.masked.load.v%di%d%s(%s align 1 %s, <%d x i1> %s, %s zeroinitializer)\n", value, vectorType, lanes, laneBits, suffix, pointerType, pointer, lanes, predicate, vectorType) + return "%" + value, nil +} diff --git a/amd64_lower_compress.go b/amd64_lower_compress.go new file mode 100644 index 000000000..beaf6c1b8 --- /dev/null +++ b/amd64_lower_compress.go @@ -0,0 +1,162 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64PackedCompressLaneBits = map[Op]int{ + "VCOMPRESSPS": 32, + "VCOMPRESSPD": 64, + "VPCOMPRESSB": 8, + "VPCOMPRESSW": 16, + "VPCOMPRESSD": 32, + "VPCOMPRESSQ": 64, +} + +// lowerPackedCompress implements the complete Go 1.27 _yvcompresspd table. +// Masked register forms compact selected source lanes and merge/zero only the +// tail. Masked memory forms write selected lanes contiguously and leave every +// byte after the compacted tail untouched, including for Go's accepted .Z +// spelling. +func (c *amd64Ctx) lowerPackedCompress(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + laneBits, ok := amd64PackedCompressLaneBits[Op(baseOp)] + if !ok { + return false, false, nil + } + zeroing := false + switch suffix { + case "": + case "Z": + zeroing = true + default: + return true, false, fmt.Errorf("%s %s accepts only Go 1.27's optional .Z suffix: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects source, [K1-K7,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if source.Kind != OpReg { + return true, false, fmt.Errorf("%s %s source must be an X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(source.Reg) + if byteWidth == 0 || !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source is outside Go's EVEX register range: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind == OpReg { + if !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s register destination must match the source width and range: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s %s destination must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + if !masked { + return true, false, c.storeVectorBytesOperand(destination, byteWidth, sourceBytes) + } + maskArg := ins.Args[1] + maskIndex, valid := amd64ParseKReg(maskArg.Reg) + if maskArg.Kind != OpReg || !valid || maskIndex == 0 { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + mask, err := c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + sourceValue := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, sourceBytes) + maskVector := amd64IntegerMaskVector(c, lanes, mask) + if destination.Kind != OpReg { + pointer, pointerType, intrinsicSuffix, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void @llvm.masked.compressstore.v%di%d%s(%s %s, %s %s, <%d x i1> %s)\n", + lanes, laneBits, intrinsicSuffix, vectorType, sourceValue, pointerType, pointer, lanes, maskVector) + if destination.Kind == OpFP { + c.markFPResultWritten(destination.FPOffset) + } + return true, false, nil + } + passthrough := "zeroinitializer" + if !zeroing { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + passthrough = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + } + compressed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.experimental.vector.compress.v%di%d(%s %s, <%d x i1> %s, %s %s)\n", + compressed, vectorType, lanes, laneBits, vectorType, sourceValue, lanes, maskVector, vectorType, passthrough) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", out, vectorType, compressed, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func amd64IntegerMaskVector(c *amd64Ctx, lanes int, mask string) string { + packed := mask + if lanes < 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, mask, lanes) + packed = "%" + narrow + } + vector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to <%d x i1>\n", vector, lanes, packed, lanes) + return "%" + vector +} + +func (c *amd64Ctx) packedVectorMemoryPointer(operand Operand) (ptr, ptrType, intrinsicSuffix string, err error) { + switch operand.Kind { + case OpMem: + ptr, ptrType, err = c.ptrFromMem(operand.Mem) + if err != nil { + return "", "", "", err + } + switch ptrType { + case "ptr": + return ptr, ptrType, "", nil + case "ptr addrspace(256)": + return ptr, ptrType, ".p256", nil + case "ptr addrspace(257)": + return ptr, ptrType, ".p257", nil + default: + return "", "", "", fmt.Errorf("unsupported packed-vector pointer type %s", ptrType) + } + case OpSym: + if address, parseErr := parseInt(strings.TrimSpace(operand.Sym)); parseErr == nil { + return c.ptrFromAddrI64(fmt.Sprintf("%d", address)), "ptr", "", nil + } + ptr, err = c.ptrFromSB(operand.Sym) + return ptr, "ptr", "", err + case OpFP: + if c.classicFrame != "" { + return c.classicFramePtr(operand.FPOffset), "ptr", "", nil + } + if ptr = c.fpParamAlloca[operand.FPOffset]; ptr != "" { + return ptr, "ptr", "", nil + } + if ptr, _, ok := c.fpResultAlloca(operand.FPOffset); ok { + return ptr, "ptr", "", nil + } + return "", "", "", fmt.Errorf("%s %s FP operand has no addressable slot", c.goarch, operand.String()) + default: + return "", "", "", fmt.Errorf("%s expected a packed-vector memory operand, got %s", c.goarch, operand.String()) + } +} diff --git a/amd64_lower_compress_test.go b/amd64_lower_compress_test.go new file mode 100644 index 000000000..5fe56490f --- /dev/null +++ b/amd64_lower_compress_test.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedCompressCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedcompressforms(SB),$0-0\n") + for _, op := range []string{"VCOMPRESSPS", "VCOMPRESSPD", "VPCOMPRESSB", "VPCOMPRESSW", "VPCOMPRESSD", "VPCOMPRESSQ"} { + for _, width := range []string{"X", "Y", "Z"} { + sourceReg, destinationReg := 20, 22 + if target.goarch == "386" && width == "Z" { + sourceReg, destinationReg = 6, 7 + } + fmt.Fprintf(&source, "\t%s %s%d, %s%d\n", op, width, sourceReg, width, destinationReg) + fmt.Fprintf(&source, "\t%s %s%d, 8(AX)\n", op, width, sourceReg) + fmt.Fprintf(&source, "\t%s %s%d, K1, %s%d\n", op, width, sourceReg, width, destinationReg) + fmt.Fprintf(&source, "\t%s.Z %s%d, K7, %s%d\n", op, width, sourceReg, width, destinationReg) + fmt.Fprintf(&source, "\t%s %s%d, K2, 8(AX)\n", op, width, sourceReg) + fmt.Fprintf(&source, "\t%s.Z %s%d, K3, 8(AX)\n", op, width, sourceReg) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedcompressforms": {Name: "packedcompressforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-compress-"+target.name+".ll", "packed-compress-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedCompressRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VCOMPRESSPS 8(AX), X1", + "VCOMPRESSPD X0, Y1", + "VCOMPRESSPS X0, K0, X1", + "VCOMPRESSPD.Z Z0, Z1", + "VPCOMPRESSB 8(AX), X1", + "VPCOMPRESSW X0, Y1", + "VPCOMPRESSD X0, K0, X1", + "VPCOMPRESSQ.Z Z0, Z1", + "VPCOMPRESSB.BCST X0, X1", + "VPCOMPRESSW X0, AX", + "VPCOMPRESSD AX, X1", + "VPCOMPRESSQ Z0, K1, K2, Z1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedCompressRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPCOMPRESSB Z8, Z0", + "VPCOMPRESSW Z0, Z8", + "VPCOMPRESSD Z8, K1, 8(AX)", + "VPCOMPRESSQ Z0, K1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedCompressRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedCompressRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvcompresspd forms for %s", instruction, goarch) + } +} + +func TestAMD64PackedCompressRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedcompress(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 (BX), Z0 + VMOVDQU64 (CX), Z1 + VPCOMPRESSB Z0, K1, Z1 + VMOVDQU64 Z1, 0(AX) + VMOVDQU64 (CX), Z1 + VPCOMPRESSW.Z Z0, K1, Z1 + VMOVDQU64 Z1, 64(AX) + VMOVDQU64 (CX), Z1 + VPCOMPRESSD Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VMOVDQU64 (CX), Z1 + VPCOMPRESSQ.Z Z0, K1, Z1 + VMOVDQU64 Z1, 192(AX) + VPCOMPRESSB Z0, K1, 256(AX) + VPCOMPRESSW.Z Z0, K1, 320(AX) + VPCOMPRESSD Z0, K1, 384(AX) + VPCOMPRESSQ.Z Z0, K1, 448(AX) + VMOVDQU64 (CX), Z1 + VCOMPRESSPS Z0, K1, Z1 + VMOVDQU64 Z1, 512(AX) + VMOVDQU64 (CX), Z1 + VCOMPRESSPD.Z Z0, K1, Z1 + VMOVDQU64 Z1, 576(AX) + VCOMPRESSPS Z0, K1, 640(AX) + VCOMPRESSPD.Z Z0, K1, 704(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedcompress": {Name: "packedcompress", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +extern void packedcompress(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static int check_group(const uint8_t *got, const uint8_t *source, const uint8_t *old, int lane_bytes, uint64_t mask, int zero_tail, int memory) { + int lanes = 64 / lane_bytes, packed = 0; + for (int lane = 0; lane < lanes; lane++) if ((mask >> lane) & 1) { + if (memcmp(got + packed*lane_bytes, source + lane*lane_bytes, lane_bytes)) return 10 + lane; + packed++; + } + for (int lane = packed; lane < lanes; lane++) { + const uint8_t *want = old + lane*lane_bytes; + uint8_t zeros[8] = {0}, untouched[8]; memset(untouched, 0xcc, sizeof(untouched)); + if (memory) want = untouched; else if (zero_tail) want = zeros; + if (memcmp(got + lane*lane_bytes, want, lane_bytes)) return 80 + lane; + } + return 0; +} +int main(void) { + uint8_t source[64], old[64], out[768]; + const uint64_t mask = 0xa55aa55aa55aa55aULL; + for (int i = 0; i < 64; i++) { source[i] = (uint8_t)(i * 29 + 7); old[i] = (uint8_t)(0xf0 - i); } + memset(out, 0xcc, sizeof(out)); + packedcompress(out, source, old, mask); + int r; + if ((r = check_group(out+0, source, old, 1, mask, 0, 0))) return 100+r; + if ((r = check_group(out+64, source, old, 2, mask, 1, 0))) return 200+r; + if ((r = check_group(out+128, source, old, 4, mask, 0, 0))) return 300+r; + if ((r = check_group(out+192, source, old, 8, mask, 1, 0))) return 400+r; + if ((r = check_group(out+256, source, old, 1, mask, 0, 1))) return 500+r; + if ((r = check_group(out+320, source, old, 2, mask, 0, 1))) return 600+r; + if ((r = check_group(out+384, source, old, 4, mask, 0, 1))) return 700+r; + if ((r = check_group(out+448, source, old, 8, mask, 0, 1))) return 800+r; + if ((r = check_group(out+512, source, old, 4, mask, 0, 0))) return 900+r; + if ((r = check_group(out+576, source, old, 8, mask, 1, 0))) return 1000+r; + if ((r = check_group(out+640, source, old, 4, mask, 0, 1))) return 1100+r; + if ((r = check_group(out+704, source, old, 8, mask, 0, 1))) return 1200+r; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_compress", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_conflict.go b/amd64_lower_conflict.go new file mode 100644 index 000000000..8d78220a9 --- /dev/null +++ b/amd64_lower_conflict.go @@ -0,0 +1,125 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedConflictSpec struct { + laneBits int +} + +// amd64PackedConflictSpecs is the complete Go 1.27 VPCONFLICTD/Q grammar. +// Both instructions use _yvexpandpd: EVEX X/Y/Z register-or-memory sources, +// optional K1-K7 merge/.Z masking, and scalar-memory broadcast. +var amd64PackedConflictSpecs = map[Op]amd64PackedConflictSpec{ + "VPCONFLICTD": {laneBits: 32}, + "VPCONFLICTQ": {laneBits: 64}, +} + +func (c *amd64Ctx) lowerPackedConflict(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64PackedConflictSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("%s %s has a suffix outside Go 1.27's _yvexpandpd form: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects source, [K1-K7,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s requires an X, Y, or Z destination: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth == 0 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go's EVEX register range: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) && !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must be a matching X/Y/Z register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + mask := "" + if masked { + maskArg := ins.Args[1] + maskIndex, valid := amd64ParseKReg(maskArg.Reg) + if maskArg.Kind != OpReg || !valid || maskIndex == 0 { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + lanes := byteWidth * 8 / spec.laneBits + values, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, broadcast) + if err != nil { + return true, false, err + } + computed := c.emitPackedConflict(values, lanes, spec.laneBits) + result := computed + if masked { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedConflict(values string, lanes, laneBits int) string { + elements := make([]string, lanes) + for lane := 0; lane < lanes; lane++ { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", extracted, lanes, laneBits, values, lane) + elements[lane] = "%" + extracted + } + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + conflicts := "0" + for prior := 0; prior < lane; prior++ { + equal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %s, %s\n", equal, laneBits, elements[lane], elements[prior]) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d 0\n", bit, equal, laneBits, uint64(1)< %s, i%d %s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, conflicts, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_conflict_test.go b/amd64_lower_conflict_test.go new file mode 100644 index 000000000..27d4976d3 --- /dev/null +++ b/amd64_lower_conflict_test.go @@ -0,0 +1,221 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedConflictGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64PackedConflictSpec{ + "VPCONFLICTD": {laneBits: 32}, + "VPCONFLICTQ": {laneBits: 64}, + } + if len(amd64PackedConflictSpecs) != len(expected) { + t.Fatalf("packed-conflict grammar has %d entries, want %d", len(amd64PackedConflictSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedConflictSpecs[op]; !ok { + t.Errorf("packed-conflict grammar omitted %s", op) + } else if got != want { + t.Errorf("packed-conflict grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PackedConflictCompleteGoFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedconflictforms(SB),$0-0\n") + for _, op := range []string{"VPCONFLICTD", "VPCONFLICTQ"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s0, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(R9), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 16(R9), K7, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST 24(R9), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(R9), K2, %s%d\n", op, width, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "packedconflictforms": {Name: "packedconflictforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"icmp eq", "select i1", "<16 x i32>", "<8 x i64>"} { + if !strings.Contains(ir, want) { + t.Fatalf("packed-conflict lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "packed-conflict-"+target.name+".ll", "packed-conflict-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedConflictRejectsFormsOutsideGoTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPCONFLICTD X0, Y1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ Y0, Z1"}, + {goarch: "amd64", instruction: "VPCONFLICTD X0, K0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ.Z X0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTD.BCST X0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ.Z.BCST 0(AX), K1, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTD.RN_SAE X0, X1"}, + {goarch: "amd64", instruction: "VPCONFLICTQ X0, 0(AX)"}, + {goarch: "amd64", instruction: "VPCONFLICTD X0, K1, K2, X1"}, + {goarch: "386", instruction: "VPCONFLICTD Z8, Z0"}, + {goarch: "386", instruction: "VPCONFLICTQ Z0, K1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvexpandpd conflict table", test.instruction) + } + }) + } +} + +func TestAMD64PackedConflictRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT packedconflictsemantics(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + STC + VPCONFLICTD Z0, Z2 + VMOVDQU64 Z2, 0(AX) + VPCONFLICTQ Z0, Z2 + VMOVDQU64 Z2, 64(AX) + VPCONFLICTD Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VMOVDQU64 0(CX), Z1 + VPCONFLICTQ.Z Z0, K1, Z1 + VMOVDQU64 Z1, 192(AX) + VPCONFLICTD.BCST 0(BX), Z2 + VMOVDQU64 Z2, 256(AX) + VPCONFLICTQ.BCST 0(BX), Z2 + VMOVDQU64 Z2, 320(AX) + SETCS 384(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"packedconflictsemantics": { + Name: "packedconflictsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void packedconflictsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +int main(void) { + uint32_t source32[16], old32[16]; uint8_t out[385] = {0}; + for (int i = 0; i < 16; i++) { source32[i] = (uint32_t)(i % 4); old32[i] = 0x90000000u + (uint32_t)i; } + const uint64_t mask = UINT64_C(0xa55a); + packedconflictsemantics(out, (const uint8_t *)source32, (const uint8_t *)old32, mask); + uint32_t *plain32 = (uint32_t *)(out + 0), *merge32 = (uint32_t *)(out + 128), *broadcast32 = (uint32_t *)(out + 256); + for (int i = 0; i < 16; i++) { + uint32_t conflict = 0; + for (int j = 0; j < i; j++) if (source32[j] == source32[i]) conflict |= UINT32_C(1) << j; + if (plain32[i] != conflict) return 10 + i; + uint32_t merged = (mask >> i) & 1 ? conflict : old32[i]; + if (merge32[i] != merged) return 30 + i; + uint32_t broadcast = i == 0 ? 0 : ((UINT32_C(1) << i) - 1); + if (broadcast32[i] != broadcast) return 50 + i; + } + uint64_t source64[8], old64[8], plain64[8], zero64[8], broadcast64[8]; + memcpy(source64, source32, 64); memcpy(old64, old32, 64); + memcpy(plain64, out + 64, 64); memcpy(zero64, out + 192, 64); memcpy(broadcast64, out + 320, 64); + for (int i = 0; i < 8; i++) { + uint64_t conflict = 0; + for (int j = 0; j < i; j++) if (source64[j] == source64[i]) conflict |= UINT64_C(1) << j; + if (plain64[i] != conflict) return 70 + i; + uint64_t masked = (mask >> i) & 1 ? conflict : 0; + if (zero64[i] != masked) return 80 + i; + uint64_t broadcast = i == 0 ? 0 : ((UINT64_C(1) << i) - 1); + if (broadcast64[i] != broadcast) return 90 + i; + } + if (out[384] != 1) return 100; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_conflict_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_convert.go b/amd64_lower_convert.go new file mode 100644 index 000000000..80c216d12 --- /dev/null +++ b/amd64_lower_convert.go @@ -0,0 +1,210 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" +) + +func amd64ParseMReg(r Reg) (idx int, ok bool) { + s := strings.ToUpper(strings.TrimSpace(string(r))) + if !strings.HasPrefix(s, "M") { + return 0, false + } + n, err := strconv.Atoi(strings.TrimPrefix(s, "M")) + if err != nil || n < 0 || n > 7 { + return 0, false + } + return n, true +} + +// Legacy named entries use the complete shared typed/native family. +func (c *amd64Ctx) lowerPackedFloatToDword(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedNumericConversion(op, ins) +} + +// lowerPackedSingleToDouble implements every operand shape in Go 1.27's +// CVTPS2PD yxm and VCVTPS2PD _yvcvtph2ps tables. The widening itself is exact; +// SAE therefore only changes the accepted encoding, not the generated value. +func (c *amd64Ctx) lowerPackedSingleToDouble(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + switch baseOp { + case "CVTPS2PD", "VCVTPS2PD": + // handled below + default: + return false, false, nil + } + + broadcast := false + suppressAllExceptions := false + zeroMasking := false + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + suffixes := strings.Split(rawOp[dot+1:], ".") + for i, suffix := range suffixes { + switch suffix { + case "BCST": + if broadcast || suppressAllExceptions || zeroMasking || i != 0 { + return true, false, fmt.Errorf("amd64 %s has invalid or duplicate .BCST suffix: %q", baseOp, ins.Raw) + } + broadcast = true + case "SAE": + if suppressAllExceptions || broadcast || zeroMasking || i != 0 { + return true, false, fmt.Errorf("amd64 %s has invalid or duplicate .SAE suffix: %q", baseOp, ins.Raw) + } + suppressAllExceptions = true + case "Z": + if zeroMasking || i != len(suffixes)-1 { + return true, false, fmt.Errorf("amd64 %s has invalid or duplicate .Z suffix: %q", baseOp, ins.Raw) + } + zeroMasking = true + default: + return true, false, fmt.Errorf("amd64 %s has unsupported suffix .%s: %q", baseOp, suffix, ins.Raw) + } + } + } + + vectorForm := baseOp == "VCVTPS2PD" + if !vectorForm { + if rawOp != baseOp || len(ins.Args) != 2 { + return true, false, fmt.Errorf("amd64 CVTPS2PD expects X/m64 source and X destination: %q", ins.Raw) + } + } else if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("amd64 VCVTPS2PD expects source, [K mask,] vector destination: %q", ins.Raw) + } + masked := vectorForm && len(ins.Args) == 3 + if zeroMasking && !masked { + return true, false, fmt.Errorf("amd64 VCVTPS2PD .Z form requires a K mask: %q", ins.Raw) + } + if broadcast && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("amd64 VCVTPS2PD .BCST requires a memory source: %q", ins.Raw) + } + + dstArg := ins.Args[len(ins.Args)-1] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects a vector destination: %q", baseOp, ins.Raw) + } + dst := dstArg.Reg + lanes := 0 + byteWidth := 0 + var loadDst func() (string, error) + var storeDst func(string) error + switch { + case isAMD64XReg(dst): + lanes, byteWidth = 2, 16 + loadDst = func() (string, error) { return c.loadX(dst) } + storeDst = func(value string) error { return c.storeX(dst, value) } + case vectorForm && isAMD64YReg(dst): + lanes, byteWidth = 4, 32 + loadDst = func() (string, error) { return c.loadY(dst) } + storeDst = func(value string) error { return c.storeY(dst, value) } + case vectorForm && isAMD64ZReg(dst): + lanes, byteWidth = 8, 64 + loadDst = func() (string, error) { return c.loadZ(dst) } + storeDst = func(value string) error { return c.storeZ(dst, value) } + default: + return true, false, fmt.Errorf("amd64 %s has mismatched or non-vector destination: %q", baseOp, ins.Raw) + } + + srcArg := ins.Args[0] + if suppressAllExceptions && (lanes != 8 || srcArg.Kind != OpReg) { + return true, false, fmt.Errorf("amd64 VCVTPS2PD .SAE requires a Y register source and Z destination: %q", ins.Raw) + } + if srcArg.Kind == OpReg { + valid := (lanes <= 4 && isAMD64XReg(srcArg.Reg)) || (lanes == 8 && isAMD64YReg(srcArg.Reg)) + if !valid { + return true, false, fmt.Errorf("amd64 %s source width does not match destination: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(srcArg) { + return true, false, fmt.Errorf("amd64 %s expects a vector register or memory source: %q", baseOp, ins.Raw) + } + + var source string + if broadcast { + scalar, err := c.evalF32(srcArg) + if err != nil { + return true, false, err + } + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x float> poison, float %s, i32 0\n", seed, lanes, scalar) + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x float> %%%s, <%d x float> poison, <%d x i32> zeroinitializer\n", splat, lanes, seed, lanes, lanes) + source = "%" + splat + } else { + switch lanes { + case 2: + if srcArg.Kind == OpReg { + bytesValue, err := c.loadX(srcArg.Reg) + if err != nil { + return true, false, err + } + all := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x float>\n", all, bytesValue) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <4 x float> %%%s, <4 x float> poison, <2 x i32> \n", low, all) + source = "%" + low + } else { + bits, err := c.evalIntSized(srcArg, I64) + if err != nil { + return true, false, err + } + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <2 x float>\n", cast, bits) + source = "%" + cast + } + case 4: + bytesValue, err := c.loadXVecOperand(srcArg) + if err != nil { + return true, false, err + } + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x float>\n", cast, bytesValue) + source = "%" + cast + case 8: + bytesValue, err := c.loadYVecOperand(srcArg) + if err != nil { + return true, false, err + } + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x float>\n", cast, bytesValue) + source = "%" + cast + } + } + + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext <%d x float> %s to <%d x double>\n", converted, lanes, source, lanes) + result := "%" + converted + if masked { + maskArg := ins.Args[1] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 VCVTPS2PD masked form expects K1-K7: %q", ins.Raw) + } + maskIndex, ok := amd64ParseKReg(maskArg.Reg) + if !ok || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 VCVTPS2PD masked form expects K1-K7: %q", ins.Raw) + } + mask, err := c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + computedBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x double> %s to <%d x i64>\n", computedBits, lanes, result, lanes) + oldBytes, err := loadDst() + if err != nil { + return true, false, err + } + oldBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i64>\n", oldBits, byteWidth, oldBytes, lanes) + maskedBits := amd64ApplyIntegerLaneMask(c, lanes, 64, "%"+computedBits, "%"+oldBits, mask, zeroMasking) + maskedValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x double>\n", maskedValue, lanes, maskedBits, lanes) + result = "%" + maskedValue + } + + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x double> %s to <%d x i8>\n", out, lanes, result, byteWidth) + return true, false, storeDst("%" + out) +} diff --git a/amd64_lower_convert_double_dword.go b/amd64_lower_convert_double_dword.go new file mode 100644 index 000000000..8c859a82c --- /dev/null +++ b/amd64_lower_convert_double_dword.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedDoubleDwordMode uint8 + +const ( + amd64PackedDwordToDouble amd64PackedDoubleDwordMode = iota + amd64PackedDoubleToDword +) + +type amd64PackedDoubleDwordSpec struct { + mode amd64PackedDoubleDwordMode + legacy bool + truncate bool + sourceBytes int + destBytes int + resultLanes int + explicitMode string +} + +// This table is both the translation dispatch and the source of truth used by +// cmd/plan9asmll's supported-opcode extractor. +var amd64PackedDoubleDwordOps = map[string]amd64PackedDoubleDwordSpec{ + "CVTPD2PL": {mode: amd64PackedDoubleToDword, legacy: true, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "CVTTPD2PL": {mode: amd64PackedDoubleToDword, legacy: true, truncate: true, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "VCVTDQ2PD": {mode: amd64PackedDwordToDouble}, + "VCVTPD2DQ": {mode: amd64PackedDoubleToDword, sourceBytes: 64, destBytes: 32, resultLanes: 8, explicitMode: "rounding"}, + "VCVTPD2DQX": {mode: amd64PackedDoubleToDword, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "VCVTPD2DQY": {mode: amd64PackedDoubleToDword, sourceBytes: 32, destBytes: 16, resultLanes: 4}, + "VCVTTPD2DQ": {mode: amd64PackedDoubleToDword, truncate: true, sourceBytes: 64, destBytes: 32, resultLanes: 8, explicitMode: "sae"}, + "VCVTTPD2DQX": {mode: amd64PackedDoubleToDword, truncate: true, sourceBytes: 16, destBytes: 16, resultLanes: 2}, + "VCVTTPD2DQY": {mode: amd64PackedDoubleToDword, truncate: true, sourceBytes: 32, destBytes: 16, resultLanes: 4}, +} + +type amd64PackedDoubleDwordSuffix struct { + broadcast bool + rounding string + sae bool + zeroing bool +} + +func parseAMD64PackedDoubleDwordSuffix(rawOp, baseOp string) (amd64PackedDoubleDwordSuffix, error) { + var result amd64PackedDoubleDwordSuffix + if rawOp == baseOp { + return result, nil + } + suffixes := strings.Split(strings.TrimPrefix(rawOp, baseOp+"."), ".") + for index, suffix := range suffixes { + switch suffix { + case "BCST": + if result.broadcast || result.rounding != "" || result.sae || result.zeroing || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .BCST suffix", baseOp) + } + result.broadcast = true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + if result.broadcast || result.rounding != "" || result.sae || result.zeroing || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .%s suffix", baseOp, suffix) + } + result.rounding = suffix + case "SAE": + if result.broadcast || result.rounding != "" || result.sae || result.zeroing || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .SAE suffix", baseOp) + } + result.sae = true + case "Z": + if result.zeroing || index != len(suffixes)-1 { + return result, fmt.Errorf("amd64 %s has invalid or misplaced .Z suffix", baseOp) + } + result.zeroing = true + default: + return result, fmt.Errorf("amd64 %s has unsupported suffix .%s", baseOp, suffix) + } + } + return result, nil +} + +// Dword-to-double retains its reviewed source/caller grammar; the remaining +// eight names use the complete packed numeric conversion family. +func (c *amd64Ctx) lowerPackedDoubleDword(op Op, ins Instr) (bool, bool, error) { + if _, known, _ := x86SignedPackedFloatProperties(ins, c.goarch); known { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + return c.lowerPackedNumericConversion(op, ins) +} + +func (c *amd64Ctx) lowIntegerLanes(value string, physicalLanes, resultLanes, laneBits int) string { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> poison, <%d x i32> <", low, physicalLanes, laneBits, value, physicalLanes, laneBits, resultLanes) + for lane := 0; lane < resultLanes; lane++ { + if lane != 0 { + c.b.WriteString(", ") + } + fmt.Fprintf(c.b, "i32 %d", lane) + } + c.b.WriteString(">\n") + return "%" + low +} diff --git a/amd64_lower_convert_double_dword_test.go b/amd64_lower_convert_double_dword_test.go new file mode 100644 index 000000000..ca5effdcd --- /dev/null +++ b/amd64_lower_convert_double_dword_test.go @@ -0,0 +1,310 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedDoubleDwordCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + high := 20 + conversionHigh, conversionBroadcast := 20, 9 + zBroadcast, zRound, zRoundMask := 8, 9, 11 + if target.goarch == "386" { + high = 4 + zBroadcast, zRound, zRoundMask = 0, 1, 3 + conversionHigh, conversionBroadcast = 4, 6 + } + source := fmt.Sprintf(` +TEXT packeddoubledwordforms(SB),$0-0 + CVTPD2PL X0, X1 + CVTTPD2PL 16(AX), X2 + VCVTDQ2PD X0, X1 + VCVTDQ2PD 32(AX), Y2 + VCVTDQ2PD Y3, Z4 + VCVTDQ2PD X%d, K1, X%d + VCVTDQ2PD.Z 64(AX), K2, Y7 + VCVTDQ2PD.BCST 80(AX), Z%d + VCVTDQ2PD.BCST.Z 84(AX), K3, X%d + VCVTPD2DQX X0, X1 + VCVTPD2DQX 88(AX), X2 + VCVTPD2DQX X%d, K1, X%d + VCVTPD2DQX 96(AX), K1, X2 + VCVTPD2DQX.Z 104(AX), K2, X3 + VCVTPD2DQX.BCST 112(AX), X4 + VCVTPD2DQX.BCST.Z 120(AX), K3, X5 + VCVTPD2DQY Y%d, X%d + VCVTPD2DQY 128(AX), X4 + VCVTPD2DQY Y3, K1, X4 + VCVTPD2DQY.Z 128(AX), K2, X5 + VCVTPD2DQY.BCST 136(AX), X6 + VCVTPD2DQY.BCST.Z 144(AX), K3, X7 + VCVTPD2DQ Z6, Y7 + VCVTPD2DQ 160(AX), K1, Y8 + VCVTPD2DQ.Z 168(AX), K2, Y9 + VCVTPD2DQ.BCST 176(AX), Y10 + VCVTPD2DQ.BCST.Z 184(AX), K3, Y11 + VCVTPD2DQ.RN_SAE Z%d, Y10 + VCVTPD2DQ.RD_SAE.Z Z%d, K3, Y12 + VCVTPD2DQ.RU_SAE Z%d, Y13 + VCVTPD2DQ.RZ_SAE.Z Z%d, K4, Y14 + VCVTTPD2DQX X0, X1 + VCVTTPD2DQX 216(AX), X2 + VCVTTPD2DQX X%d, K1, X%d + VCVTTPD2DQX 224(AX), K1, X2 + VCVTTPD2DQX.Z 232(AX), K2, X3 + VCVTTPD2DQX.BCST 240(AX), X4 + VCVTTPD2DQX.BCST.Z 248(AX), K3, X5 + VCVTTPD2DQY Y%d, X%d + VCVTTPD2DQY 256(AX), X4 + VCVTTPD2DQY Y3, K1, X4 + VCVTTPD2DQY.Z 256(AX), K2, X5 + VCVTTPD2DQY.BCST 264(AX), X6 + VCVTTPD2DQY.BCST.Z 272(AX), K3, X7 + VCVTTPD2DQ Z6, Y7 + VCVTTPD2DQ 288(AX), K1, Y8 + VCVTTPD2DQ.Z 296(AX), K2, Y9 + VCVTTPD2DQ.BCST 304(AX), Y10 + VCVTTPD2DQ.BCST.Z 312(AX), K3, Y11 + VCVTTPD2DQ.SAE Z%d, Y10 + VCVTTPD2DQ.SAE.Z Z%d, K3, Y12 + RET +`, conversionHigh, conversionHigh+1, zBroadcast, conversionBroadcast, + high, high+1, high, high+1, zRound, zRoundMask, zRound, zRoundMask, + high, high+1, high, high+1, zRound, zRoundMask) + if target.goarch == "386" { + for n := 8; n <= 14; n++ { + source = strings.ReplaceAll(source, fmt.Sprintf("Y%d", n), fmt.Sprintf("Y%d", n%8)) + } + } + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packeddoubledwordforms": {Name: "packeddoubledwordforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"vcvtdq2pd", "cvtpd2dq", "vcvttpd2dq"} { + if !strings.Contains(ll, want) { + t.Fatalf("packed double/dword IR is missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-double-dword-"+target.name+".ll", "packed-double-dword-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedDoubleDwordRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CVTPD2PL Y0, X1", + "CVTTPD2PL X0, Y1", + "CVTTPD2PL.Z X0, X1", + "VCVTDQ2PD Y0, X1", + "VCVTDQ2PD X0, Z1", + "VCVTDQ2PD.BCST X0, X1", + "VCVTDQ2PD.RN_SAE Y0, Z1", + "VCVTPD2DQ X0, X1", + "VCVTPD2DQY X0, X1", + "VCVTPD2DQX Y0, X1", + "VCVTPD2DQY.BCST X0, X1", + "VCVTPD2DQ.RN_SAE 0(AX), Y1", + "VCVTTPD2DQX.SAE X0, X1", + "VCVTTPD2DQ.SAE 0(AX), Y1", + "VCVTTPD2DQ.Z Z0, Y1", + "VCVTTPD2DQ Z0, K0, Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDoubleDwordRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86PackedDoubleDwordRejected(t, "386", "i386-unknown-linux-gnu", "VCVTPD2DQ Z8, Y1") +} + +func assertX86PackedDoubleDwordRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed double/dword tables", instruction) + } +} + +func TestAMD64PackedDoubleDwordRuntimeSemantics(t *testing.T) { + amd64PackedDoubleDwordRuntime(t, x86FloatCompareRuntimeTools(t, x86FloatCompareSSE), false) +} + +func x86PackedDoubleDwordOriginalAssembly() string { + return ` +TEXT packeddoubledword(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ doubles+8(FP), BX + MOVQ dwords+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), SI + KMOVQ SI, K1 + + VMOVDQU64 (DX), Z1 + VCVTDQ2PD (CX), K1, Z1 + VMOVDQU64 Z1, 0(AX) + + VCVTTPD2DQY (BX), X2 + VMOVDQU X2, 64(AX) + + VMOVDQU64 (BX), Z0 + VCVTPD2DQ.RD_SAE.Z Z0, K1, Y3 + VMOVDQU Y3, 80(AX) + + MOVOU (BX), X4 + CVTPD2PL X4, X0 + MOVQ X0, 112(AX) + VZEROUPPER + RET +` +} + +func x86PackedDoubleDwordRawAssembly(source string) string { + return strings.NewReplacer( + "TEXT packeddoubledword(SB),NOSPLIT,$0-40", "TEXT packeddoubledword(SB),4,$0-40", + "VCVTDQ2PD (CX), K1, Z1", "BYTE $0x62; BYTE $0xf1; BYTE $0x7e; BYTE $0x49; BYTE $0xe6; BYTE $0x09", + "VCVTTPD2DQY (BX), X2", "BYTE $0xc5; BYTE $0xfd; BYTE $0xe6; BYTE $0x13", + "VCVTPD2DQ.RD_SAE.Z Z0, K1, Y3", "BYTE $0x62; BYTE $0xf1; BYTE $0xff; BYTE $0xb9; BYTE $0xe6; BYTE $0xd8", + "CVTPD2PL X4, X0", "BYTE $0xf2; BYTE $0x0f; BYTE $0xe6; BYTE $0xc4", + ).Replace(source) +} + +func x86PackedDoubleDwordSSEAssembly(t *testing.T) string { + t.Helper() + source := x86PackedDoubleDwordOriginalAssembly() + preamble := strings.Index(source, "\tKMOVQ SI, K1") + legacy := strings.Index(source, "\tMOVOU (BX), X4") + wideExit := strings.Index(source, "\tVZEROUPPER") + if preamble < 0 || legacy <= preamble || wideExit <= legacy { + t.Fatal("original complete double/dword fixture lost its explicit SSE block") + } + return source[:preamble] + source[legacy:wideExit] + "\tRET\n" +} + +func amd64PackedDoubleDwordRuntime(t *testing.T, execution x86FloatCompareExecution, nativeWide bool) { + t.Helper() + source := x86PackedDoubleDwordOriginalAssembly() + if !nativeWide { + source = x86PackedDoubleDwordSSEAssembly(t) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }} + triple := execution.triple + translationOptions := Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packeddoubledword": { + Name: "packeddoubledword", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame, + }, + }, + } + ll, err := Translate(file, translationOptions) + if err != nil { + t.Fatal(err) + } + // The entire original named and raw functions still reach Go and LLVM + // objects in the portable lane. Only their execution requires native ISA. + for _, complete := range []string{x86PackedDoubleDwordOriginalAssembly(), x86PackedDoubleDwordRawAssembly(x86PackedDoubleDwordOriginalAssembly())} { + goAssemblerOracleOutput(t, "amd64", x86PackedDoubleDwordGoAssemblerSource(complete), "", true) + parsed, err := Parse(ArchAMD64, complete) + if err != nil { + t.Fatal(err) + } + objectIR, err := Translate(parsed, translationOptions) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, execution.llc, triple, "complete-double-dword.ll", "complete-double-dword.o", objectIR) + } + mainC := fmt.Sprintf("#define NATIVE_WIDE %d\n", x86FloatAddSubBoolInt(nativeWide)) + ` +#include +#include +#include +#include +extern void packeddoubledword(uint8_t *, const double *, const int32_t *, const double *, uint64_t); +int main(void) { + const double doubles[8] = {1.75, -2.75, 2147483648.0, NAN, -2147483648.0, -2147483649.0, 3.0, -0.0}; + const int32_t dwords[8] = {0, 1, -1, INT32_MAX, INT32_MIN, 123456789, -987654321, 42}; + const double old[8] = {100.5, 101.5, 102.5, 103.5, 104.5, 105.5, 106.5, 107.5}; + const uint64_t mask = 0xa5; + uint8_t out[120]; memset(out, 0xcc, sizeof(out)); + packeddoubledword(out, doubles, dwords, old, mask); + +#if NATIVE_WIDE + const double *widened = (const double *)(out+0); + for (int i = 0; i < 8; i++) { + double want = (mask >> i) & 1 ? (double)dwords[i] : old[i]; + if (widened[i] != want) return 10+i; + } + const int32_t *truncated = (const int32_t *)(out+64); + const int32_t wantTruncated[4] = {1, -2, INT32_MIN, INT32_MIN}; + for (int i = 0; i < 4; i++) if (truncated[i] != wantTruncated[i]) return 30+i; + + const int32_t *roundedDown = (const int32_t *)(out+80); + const int32_t wantRoundedDown[8] = {1, 0, INT32_MIN, 0, 0, INT32_MIN, 0, 0}; + for (int i = 0; i < 8; i++) if (roundedDown[i] != wantRoundedDown[i]) return 50+i; +#endif + + const int32_t *legacy = (const int32_t *)(out+112); + if (legacy[0] != 2 || legacy[1] != -3) return 70; + return 0; +} +` + execution.compileAndRunLLVM(t, "packed_double_dword", ll, mainC, "-O2") + + rawSource := x86PackedDoubleDwordRawAssembly(source) + goAssemblerOracleOutput(t, "amd64", x86PackedDoubleDwordGoAssemblerSource(rawSource), "", true) + rawFile, err := Parse(ArchAMD64, rawSource) + if err != nil { + t.Fatal(err) + } + rawIR, err := Translate(rawFile, translationOptions) + if err != nil { + t.Fatal(err) + } + execution.compileAndRunLLVM(t, "raw_packed_double_dword", rawIR, mainC, "-O2") + t.Logf("complete named/raw objects; SSE legacy output executed; full 120-byte native-wide assertions=%v", nativeWide) +} diff --git a/amd64_lower_convert_fp16.go b/amd64_lower_convert_fp16.go new file mode 100644 index 000000000..5df1151c4 --- /dev/null +++ b/amd64_lower_convert_fp16.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// FP16 map5/map6 conversions are both load forms. The lane types determine +// access and result widths; encoded VL distinguishes the two X destinations. +func (c *amd64Ctx) lowerRawFP16Conversion(ins Instr, spec amd64HalfConversionSpec) (bool, bool, error) { + if !ins.x86Encoded { + return true, false, fmt.Errorf("%s has no named Go assembler form: %q", ins.Op, ins.Raw) + } + vl := ins.x86VectorBytes + if !spec.scalar && vl != 16 && vl != 32 && vl != 64 { + return true, false, fmt.Errorf("%s has invalid encoded vector length %d", ins.Op, vl) + } + operandCount := 2 + if spec.scalar { + operandCount = 3 + } + if len(ins.Args) != operandCount && len(ins.Args) != operandCount+1 { + return true, false, fmt.Errorf("%s requires source, [K mask,] destination", ins.Op) + } + base := strings.SplitN(string(ins.Op), ".", 2)[0] + suffix := strings.TrimPrefix(string(ins.Op), base) + sae := suffix == ".SAE" || suffix == ".SAE.Z" + if sae { + suffix = strings.TrimPrefix(suffix, ".SAE") + } + properties, valid := parseAMD64FMA3Suffix(strings.TrimPrefix(suffix, ".")) + if !valid || spec.inputBits == 16 && properties.rounding != "" || spec.outputBits == 16 && sae { + return true, false, fmt.Errorf("%s has invalid conversion controls", ins.Op) + } + masked := len(ins.Args) == operandCount+1 + if properties.zeroing && !masked || masked && !amd64NonzeroKOperand(ins.Args[len(ins.Args)-2]) { + return true, false, fmt.Errorf("%s requires K1-K7 for masking", ins.Op) + } + if spec.scalar { + return c.lowerRawScalarHalfConversion(ins, spec, properties, sae) + } + source, destination := ins.Args[0], ins.Args[len(ins.Args)-1] + lanes := vl / 4 + inputBytes, outputBytes := lanes*spec.inputBits/8, lanes*spec.outputBits/8 + sourceRegisterBytes, destinationRegisterBytes := inputBytes, outputBytes + if sourceRegisterBytes < 16 { + sourceRegisterBytes = 16 + } + if destinationRegisterBytes < 16 { + destinationRegisterBytes = 16 + } + if !c.isGoEVEXVectorRegister(destination, destinationRegisterBytes) { + return true, false, fmt.Errorf("%s has an invalid destination register", ins.Op) + } + if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, sourceRegisterBytes) || properties.broadcast { + return true, false, fmt.Errorf("%s has invalid source width or register broadcast", ins.Op) + } + } else if !isAMD64MemoryOperand(source) || sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s has invalid memory or rounding operands", ins.Op) + } + if (sae || properties.rounding != "") && vl != 64 { + return true, false, fmt.Errorf("%s explicit rounding/SAE requires 512-bit VL", ins.Op) + } + + mask := "" + var err error + if masked { + mask, err = c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + } + var input string + if source.Kind == OpReg || !masked && !properties.broadcast { + input, err = c.loadPackedExtendInputs(source, sourceRegisterBytes, lanes, spec.inputBits) + } else { + input, err = c.loadMaskedPackedCompareLanes(source, inputBytes, spec.inputBits, properties.broadcast, mask) + } + if err != nil { + return true, false, err + } + result := c.emitHalfConversion(lanes, spec, input, properties.rounding) + if masked { + old, err := c.loadPackedExtendInputs(destination, destinationRegisterBytes, lanes, spec.outputBits) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, spec.outputBits, result, old, mask, properties.zeroing) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, spec.outputBits, result, outputBytes) + return true, false, c.storePackedHalfResult(destination, destinationRegisterBytes, outputBytes, "%"+bytes) +} + +func (c *amd64Ctx) emitHalfConversion(lanes int, spec amd64HalfConversionSpec, input, rounding string) string { + inputType := amd64FMA3LLVMType(1, spec.inputBits) + outputType := amd64FMA3LLVMType(1, spec.outputBits) + conversion := "fpext" + if spec.outputBits == 16 { + conversion = "fptrunc" + } + floats, converted, bits := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x %s>\n", floats, lanes, spec.inputBits, input, lanes, inputType) + fmt.Fprintf(c.b, " %%%s = %s <%d x %s> %%%s to <%d x %s>\n", converted, conversion, lanes, inputType, floats, lanes, outputType) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x %s> %%%s to <%d x i%d>\n", bits, lanes, outputType, converted, lanes, spec.outputBits) + result := "%" + bits + if spec.outputBits == 16 { + result = c.adjustFP16NarrowRounding(lanes, inputType, "%"+floats, "%"+converted, result, rounding) + } + return result +} + +// Directed rounding differs from nearest-even by at most one half ULP. +// Comparing the exactly widened result lets integer correction preserve +// signed zero, subnormals, NaNs and directed overflow without changing the +// host FP environment. LLVM 22's AArch64 constrained fptrunc does not honor +// these static rounding modes for half, so it cannot serve as this oracle. +func (c *amd64Ctx) adjustFP16NarrowRounding(lanes int, inputType, original, halves, bits, rounding string) string { + if rounding == "" || rounding == "RN_SAE" { + return bits + } + widened, below, above, negative := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext <%d x half> %s to <%d x %s>\n", widened, lanes, halves, lanes, inputType) + fmt.Fprintf(c.b, " %%%s = fcmp olt <%d x %s> %%%s, %s\n", below, lanes, inputType, widened, original) + fmt.Fprintf(c.b, " %%%s = fcmp ogt <%d x %s> %%%s, %s\n", above, lanes, inputType, widened, original) + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i16> %s, zeroinitializer\n", negative, lanes, bits) + one := amd64SplatInteger(c, lanes, 16, "1") + plus, minus := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add <%d x i16> %s, %s\n", plus, lanes, bits, one) + fmt.Fprintf(c.b, " %%%s = sub <%d x i16> %s, %s\n", minus, lanes, bits, one) + condition, adjusted := "%"+above, "%"+minus + choice := c.newTmp() + switch rounding { + case "RD_SAE": + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i16> %%%s, <%d x i16> %%%s\n", choice, lanes, negative, lanes, plus, lanes, minus) + adjusted = "%" + choice + case "RU_SAE": + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i16> %%%s, <%d x i16> %%%s\n", choice, lanes, negative, lanes, minus, lanes, plus) + condition, adjusted = "%"+below, "%"+choice + case "RZ_SAE": + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i1> %%%s, <%d x i1> %%%s\n", choice, lanes, negative, lanes, below, lanes, above) + condition = "%" + choice + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %s, <%d x i16> %s, <%d x i16> %s\n", result, lanes, condition, lanes, adjusted, lanes, bits) + return "%" + result +} diff --git a/amd64_lower_convert_fp16_scalar.go b/amd64_lower_convert_fp16_scalar.go new file mode 100644 index 000000000..b68fad272 --- /dev/null +++ b/amd64_lower_convert_fp16_scalar.go @@ -0,0 +1,42 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerRawScalarHalfConversion(ins Instr, spec amd64HalfConversionSpec, properties amd64FMA3Suffix, sae bool) (bool, bool, error) { + source, upper, destination := ins.Args[0], ins.Args[1], ins.Args[len(ins.Args)-1] + if !c.isGoEVEXVectorRegister(upper, 16) || !c.isGoEVEXVectorRegister(destination, 16) || properties.broadcast { + return true, false, fmt.Errorf("%s requires X registers without broadcast", ins.Op) + } + if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, 16) { + return true, false, fmt.Errorf("%s source must be X or scalar memory", ins.Op) + } + } else if !isAMD64MemoryOperand(source) || sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s has invalid source or memory rounding", ins.Op) + } + mask := "" + var err error + if len(ins.Args) == 4 { + mask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + value, err := c.loadFMA3Scalar(source, spec.inputBits, mask) + if err != nil { + return true, false, err + } + inputType := amd64FMA3LLVMType(1, spec.inputBits) + vector, bits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x %s> poison, %s %s, i32 0\n", vector, inputType, inputType, value) + fmt.Fprintf(c.b, " %%%s = bitcast <1 x %s> %%%s to <1 x i%d>\n", bits, inputType, vector, spec.inputBits) + result := c.emitHalfConversion(1, spec, "%"+bits, properties.rounding) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.outputBits, result) + upperBytes, err := c.loadX(upper.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.outputBits, spec.outputBits, upperBytes) + return true, false, c.storeScalarMoveRegister(destination.Reg, spec.outputBits, "%"+low, base, mask, properties.zeroing) +} diff --git a/amd64_lower_convert_half.go b/amd64_lower_convert_half.go new file mode 100644 index 000000000..3c3a38afc --- /dev/null +++ b/amd64_lower_convert_half.go @@ -0,0 +1,256 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64HalfConversionSuffix struct { + sae bool + zeroing bool +} + +type amd64HalfConversionSpec struct { + inputBits, outputBits int + rawOnly bool + scalar bool +} + +var amd64HalfConversionSpecs = map[string]amd64HalfConversionSpec{ + "VCVTPH2PS": {inputBits: 16, outputBits: 32}, + "VCVTPS2PH": {inputBits: 32, outputBits: 16}, + "VCVTPH2PSX": {inputBits: 16, outputBits: 32, rawOnly: true}, + "VCVTPS2PHX": {inputBits: 32, outputBits: 16, rawOnly: true}, + "VCVTSH2SS": {inputBits: 16, outputBits: 32, rawOnly: true, scalar: true}, + "VCVTSS2SH": {inputBits: 32, outputBits: 16, rawOnly: true, scalar: true}, + "VCVTSH2SD": {inputBits: 16, outputBits: 64, rawOnly: true, scalar: true}, + "VCVTSD2SH": {inputBits: 64, outputBits: 16, rawOnly: true, scalar: true}, +} + +func parseAMD64HalfConversionSuffix(rawOp, baseOp string) (amd64HalfConversionSuffix, error) { + var result amd64HalfConversionSuffix + if rawOp == baseOp { + return result, nil + } + suffixes := strings.Split(strings.TrimPrefix(rawOp, baseOp+"."), ".") + for index, suffix := range suffixes { + switch suffix { + case "SAE": + if result.sae || index != 0 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .SAE suffix", baseOp) + } + result.sae = true + case "Z": + if result.zeroing || index != len(suffixes)-1 { + return result, fmt.Errorf("amd64 %s has invalid or duplicate .Z suffix", baseOp) + } + result.zeroing = true + default: + return result, fmt.Errorf("amd64 %s has unsupported suffix .%s", baseOp, suffix) + } + } + return result, nil +} + +// lowerPackedHalfConversion implements every Go 1.27 _yvcvtph2ps and +// _yvcvtps2ph row, including VEX signed immediates, EVEX masks, zeroing, SAE, +// exact-width memory accesses, and the distinct X/Y/Z narrowing widths. +func (c *amd64Ctx) lowerPackedHalfConversion(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, supported := amd64HalfConversionSpecs[baseOp] + if !supported { + return false, false, nil + } + if spec.rawOnly { + return c.lowerRawFP16Conversion(ins, spec) + } + properties, err := parseAMD64HalfConversionSuffix(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + if baseOp == "VCVTPH2PS" { + return c.lowerPackedHalfToSingle(ins, properties) + } + return c.lowerPackedSingleToHalf(ins, properties) +} + +func (c *amd64Ctx) lowerPackedHalfToSingle(ins Instr, properties amd64HalfConversionSuffix) (bool, bool, error) { + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("amd64 VCVTPH2PS expects source, [K mask,] destination: %q", ins.Raw) + } + masked := len(ins.Args) == 3 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("amd64 VCVTPH2PS .Z requires K1-K7: %q", ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 VCVTPH2PS masked form expects K1-K7: %q", ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 VCVTPH2PS destination must be X, Y, or Z: %q", ins.Raw) + } + destinationBytes := amd64VectorByteWidth(destination.Reg) + if destinationBytes != 16 && destinationBytes != 32 && destinationBytes != 64 || !c.isGoEVEXVectorRegister(destination, destinationBytes) { + return true, false, fmt.Errorf("amd64 VCVTPH2PS destination is outside Go 1.27's vector-register class: %q", ins.Raw) + } + sourceRegisterBytes := 16 + if destinationBytes == 64 { + sourceRegisterBytes = 32 + } + source := ins.Args[0] + if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, sourceRegisterBytes) { + return true, false, fmt.Errorf("amd64 VCVTPH2PS source width does not match its destination: %q", ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("amd64 VCVTPH2PS source must be X/Y or memory: %q", ins.Raw) + } + if properties.sae && (destinationBytes != 64 || source.Kind != OpReg) { + return true, false, fmt.Errorf("amd64 VCVTPH2PS .SAE requires a Y register source and Z destination: %q", ins.Raw) + } + + lanes := destinationBytes / 4 + inputBits, err := c.loadPackedExtendInputs(source, sourceRegisterBytes, lanes, 16) + if err != nil { + return true, false, err + } + halves := c.newTmp() + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x half>\n", halves, lanes, inputBits, lanes) + fmt.Fprintf(c.b, " %%%s = fpext <%d x half> %%%s to <%d x float>\n", converted, lanes, halves, lanes) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x float> %%%s to <%d x i32>\n", bits, lanes, converted, lanes) + result := "%" + bits + if masked { + mask, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, destinationBytes) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(destinationBytes, lanes, 32, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, 32, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i8>\n", out, lanes, result, destinationBytes) + return true, false, c.storeVectorBytes(destination.Reg, destinationBytes, "%"+out) +} + +func (c *amd64Ctx) lowerPackedSingleToHalf(ins Instr, properties amd64HalfConversionSuffix) (bool, bool, error) { + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("amd64 VCVTPS2PH expects imm8, source, [K mask,] destination: %q", ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < -128 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("amd64 VCVTPS2PH expects a signed-or-unsigned imm8: %q", ins.Raw) + } + masked := len(ins.Args) == 4 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("amd64 VCVTPS2PH .Z requires K1-K7: %q", ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 VCVTPS2PH masked form expects K1-K7: %q", ins.Raw) + } + if masked && c.goarch == "386" { + return true, false, fmt.Errorf("386 VCVTPS2PH does not expose Go 1.27's four-operand EVEX forms: %q", ins.Raw) + } + source := ins.Args[1] + if source.Kind != OpReg { + return true, false, fmt.Errorf("amd64 VCVTPS2PH source must be X, Y, or Z: %q", ins.Raw) + } + sourceBytes := amd64VectorByteWidth(source.Reg) + if sourceBytes != 16 && sourceBytes != 32 && sourceBytes != 64 || !c.isGoEVEXVectorRegister(source, sourceBytes) { + return true, false, fmt.Errorf("amd64 VCVTPS2PH source is outside Go 1.27's vector-register class: %q", ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + destinationRegisterBytes := 16 + if sourceBytes == 64 { + destinationRegisterBytes = 32 + } + if destination.Kind == OpReg { + if !c.isGoEVEXVectorRegister(destination, destinationRegisterBytes) { + return true, false, fmt.Errorf("amd64 VCVTPS2PH destination width does not match its source: %q", ins.Raw) + } + } else if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("amd64 VCVTPS2PH destination must be X/Y or memory: %q", ins.Raw) + } + if properties.zeroing && destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 VCVTPS2PH .Z requires a register destination: %q", ins.Raw) + } + requiresEVEX := masked || properties.sae || sourceBytes == 64 || amd64HalfConversionHighRegister(source) + if destination.Kind == OpReg && amd64HalfConversionHighRegister(destination) { + requiresEVEX = true + } + if ins.Args[0].Imm < 0 && requiresEVEX { + return true, false, fmt.Errorf("amd64 VCVTPS2PH EVEX forms require an unsigned imm8: %q", ins.Raw) + } + if properties.sae && (sourceBytes != 64 || source.Kind != OpReg) { + return true, false, fmt.Errorf("amd64 VCVTPS2PH .SAE requires a Z register source: %q", ins.Raw) + } + + lanes := sourceBytes / 4 + sourceBits, err := c.loadPackedExtendInputs(source, sourceBytes, lanes, 32) + if err != nil { + return true, false, err + } + floats := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x float>\n", floats, lanes, sourceBits, lanes) + converted := c.newTmp() + rounding := int(ins.Args[0].Imm) & 7 + fmt.Fprintf(c.b, " %%%s = fptrunc <%d x float> %%%s to <%d x half>\n", converted, lanes, floats, lanes) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x half> %%%s to <%d x i16>\n", bits, lanes, converted, lanes) + result := "%" + bits + if rounding > 0 && rounding < 4 { + mode := [...]string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"}[rounding] + result = c.adjustFP16NarrowRounding(lanes, "float", "%"+floats, "%"+converted, result, mode) + } + outputBytes := sourceBytes / 2 + if masked { + mask, err := c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + old, err := c.loadPackedExtendInputs(destination, destinationRegisterBytes, lanes, 16) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, 16, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x i8>\n", out, lanes, result, outputBytes) + return true, false, c.storePackedHalfResult(destination, destinationRegisterBytes, outputBytes, "%"+out) +} + +func amd64HalfConversionHighRegister(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + width := amd64VectorByteWidth(operand.Reg) + index, ok := amd64VectorRegisterIndex(operand.Reg, width) + return ok && index >= 16 +} + +func (c *amd64Ctx) storePackedHalfResult(destination Operand, registerBytes, outputBytes int, value string) error { + if destination.Kind != OpReg { + return c.storeVectorBytesOperand(destination, outputBytes, value) + } + if outputBytes == registerBytes { + return c.storePackedMoveOperand(destination, registerBytes, value) + } + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> <", widened, outputBytes, value, outputBytes, registerBytes) + for lane := 0; lane < registerBytes; lane++ { + if lane > 0 { + c.b.WriteString(", ") + } + fmt.Fprintf(c.b, "i32 %d", lane) + } + c.b.WriteString(">\n") + return c.storePackedMoveOperand(destination, registerBytes, "%"+widened) +} diff --git a/amd64_lower_convert_half_test.go b/amd64_lower_convert_half_test.go new file mode 100644 index 000000000..86fa326db --- /dev/null +++ b/amd64_lower_convert_half_test.go @@ -0,0 +1,144 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedHalfConversionCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 defines the whole F16C/AVX-512 FP16 conversion pair in + // _yvcvtph2ps and _yvcvtps2ph: + // + // VCVTPH2PS X/m -> X or Y; Y/m -> Z + // VCVTPS2PH imm8, X -> X/m; Y -> X/m; Z -> Y/m + // + // EVEX forms additionally accept K1-K7 and register-destination zeroing. + // The VEX VCVTPS2PH rows accept both signed and unsigned imm8 spellings. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString(`TEXT halfconversionforms(SB),NOSPLIT,$0-0 + VCVTPH2PS X0, X1 + VCVTPH2PS 8(AX), X2 + VCVTPH2PS X20, X21 + VCVTPH2PS X3, K1, X4 + VCVTPH2PS.Z 16(AX), K2, X5 + VCVTPH2PS X6, Y7 + VCVTPH2PS 32(AX), Y8 + VCVTPH2PS X20, K3, Y21 + VCVTPH2PS.Z 48(AX), K4, Y9 + VCVTPH2PS Y10, Z1 + VCVTPH2PS 64(AX), Z2 + VCVTPH2PS Y20, K5, Z3 + VCVTPH2PS.Z 96(AX), K6, Z4 + VCVTPH2PS.SAE Y10, Z5 + VCVTPH2PS.SAE.Z Y11, K7, Z6 + VCVTPS2PH $0, X0, X1 + VCVTPS2PH $-1, X2, 128(AX) + VCVTPS2PH $255, Y3, X4 + VCVTPS2PH $-128, Y5, 160(AX) + VCVTPS2PH $7, Z1, Y2 + VCVTPS2PH.SAE $8, Z3, Y4 +`) + // Go 1.27's 386 assembler accepts the EVEX VCVTPH2PS mask + // rows, but its operand parser does not expose the four-operand + // masked VCVTPS2PH rows. Keep this difference visible here. + if target.goarch == "amd64" { + source.WriteString(` VCVTPS2PH $126, X7, K5, X17 + VCVTPS2PH.Z $126, X7, K5, X17 + VCVTPS2PH $126, X7, K5, 192(AX) + VCVTPS2PH $94, Y1, K7, X15 + VCVTPS2PH.Z $94, Y1, K7, X15 + VCVTPS2PH $94, Y1, K7, 224(AX) + VCVTPS2PH $121, Z5, K7, Y28 + VCVTPS2PH.SAE.Z $13, Z2, K6, Y13 + VCVTPS2PH $121, Z5, K7, 256(AX) +`) + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "halfconversionforms": {Name: "halfconversionforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"fpext <", "half>", "fptrunc <"} { + if !strings.Contains(ll, want) { + t.Fatalf("half conversion IR is missing %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-half-conversion-"+target.name+".ll", "packed-half-conversion-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedHalfConversionRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VCVTPH2PS X0", + "VCVTPH2PS X0, X1, X2", + "VCVTPH2PS Y0, X1", + "VCVTPH2PS X0, Z1", + "VCVTPH2PS X0, K0, X1", + "VCVTPH2PS.Z X0, X1", + "VCVTPH2PS.BCST (AX), X1", + "VCVTPS2PH X0, X1", + "VCVTPS2PH $256, X0, X1", + "VCVTPS2PH $0, Z0, X1", + "VCVTPS2PH $0, X0, Y1", + "VCVTPS2PH $0, X0, K0, X1", + "VCVTPS2PH.Z $0, X0, X1", + "VCVTPS2PH.Z $0, X0, K1, (AX)", + "VCVTPS2PH.BCST $0, X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedHalfConversionRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VCVTPS2PH $-1, X20, X21", + "VCVTPS2PH $-1, Y20, X21", + "VCVTPS2PH $0, X0, K1, X1", + "VCVTPS2PH $0, Y0, K1, X1", + "VCVTPS2PH $0, Z0, K1, Y1", + } { + assertX86PackedHalfConversionRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedHalfConversionRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, fmt.Sprintf("TEXT bad(SB),NOSPLIT,$0-0\n\t%s\n\tRET\n", instruction)) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VCVTPH2PS/VCVTPS2PH tables", instruction) + } +} diff --git a/amd64_lower_convert_integer_float.go b/amd64_lower_convert_integer_float.go new file mode 100644 index 000000000..a75e4167f --- /dev/null +++ b/amd64_lower_convert_integer_float.go @@ -0,0 +1,39 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerLegacyIntegerToFloat implements Go 1.27's complete legacy integer to +// floating-point conversion group: yxcvlf/yxcvqf scalar conversions and the +// two yxcvm2 packed conversions. +func (c *amd64Ctx) lowerLegacyIntegerToFloat(op Op, ins Instr) (ok bool, terminated bool, err error) { + if _, known, _ := x86SignedPackedFloatProperties(ins, c.goarch); known { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + rawOp := strings.ToUpper(string(op)) + if strings.Contains(rawOp, ".") { + base := strings.SplitN(rawOp, ".", 2)[0] + switch base { + case "CVTSL2SS", "CVTSL2SD", "CVTSQ2SS", "CVTSQ2SD", "CVTPL2PS", "CVTPL2PD": + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", base, ins.Raw) + } + } + switch rawOp { + case "CVTSL2SS", "CVTSL2SD", "CVTSQ2SS", "CVTSQ2SD": + return c.lowerLegacyScalarIntegerToFloat(rawOp, ins) + case "CVTPL2PS", "CVTPL2PD": + return c.lowerLegacyPackedIntegerToFloat(rawOp, ins) + default: + return false, false, nil + } +} + +func (c *amd64Ctx) lowerLegacyScalarIntegerToFloat(op string, ins Instr) (bool, bool, error) { + return c.lowerSignedIntegerScalarFloat(Op(op), ins) +} + +func (c *amd64Ctx) lowerLegacyPackedIntegerToFloat(op string, ins Instr) (bool, bool, error) { + return c.lowerSignedPackedIntegerFloat(Op(op), ins) +} diff --git a/amd64_lower_convert_narrow.go b/amd64_lower_convert_narrow.go new file mode 100644 index 000000000..c49168aa3 --- /dev/null +++ b/amd64_lower_convert_narrow.go @@ -0,0 +1,209 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedDoubleToSingleProperties struct { + broadcast bool + rounding string + zeroing bool +} + +func amd64PackedDoubleToSingleSuffixes(rawOp, baseOp string) (amd64PackedDoubleToSingleProperties, error) { + var properties amd64PackedDoubleToSingleProperties + dot := strings.IndexByte(rawOp, '.') + if dot < 0 { + return properties, nil + } + suffixes := strings.Split(rawOp[dot+1:], ".") + for i, suffix := range suffixes { + switch suffix { + case "BCST": + if properties.broadcast || properties.rounding != "" || properties.zeroing || i != 0 { + return properties, fmt.Errorf("amd64 %s has invalid or duplicate .BCST suffix", baseOp) + } + properties.broadcast = true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + if properties.rounding != "" || properties.broadcast || properties.zeroing || i != 0 { + return properties, fmt.Errorf("amd64 %s has invalid or duplicate .%s suffix", baseOp, suffix) + } + properties.rounding = suffix + case "Z": + if properties.zeroing || i != len(suffixes)-1 { + return properties, fmt.Errorf("amd64 %s has invalid or duplicate .Z suffix", baseOp) + } + properties.zeroing = true + default: + return properties, fmt.Errorf("amd64 %s has unsupported suffix .%s", baseOp, suffix) + } + } + return properties, nil +} + +// lowerPackedDoubleToSingle implements the complete Go 1.27 CVTPD2PS family: +// CVTPD2PS X/m128->X, VCVTPD2PSX X/m128->X, VCVTPD2PSY Y/m256->X, +// and VCVTPD2PS Z/m512->Y, including EVEX masks, broadcast, and rounding. +func (c *amd64Ctx) lowerPackedDoubleToSingle(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if baseOp != "CVTPD2PS" && baseOp != "VCVTPD2PS" && baseOp != "VCVTPD2PSX" && baseOp != "VCVTPD2PSY" { + return false, false, nil + } + + properties, err := amd64PackedDoubleToSingleSuffixes(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + vectorForm := baseOp != "CVTPD2PS" + if !vectorForm { + if rawOp != baseOp || len(ins.Args) != 2 { + return true, false, fmt.Errorf("amd64 CVTPD2PS expects X/m128 source and X destination: %q", ins.Raw) + } + } else if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("amd64 %s expects source, [K mask,] vector destination: %q", baseOp, ins.Raw) + } + + masked := vectorForm && len(ins.Args) == 3 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s .Z form requires a K mask: %q", baseOp, ins.Raw) + } + if properties.broadcast && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("amd64 %s .BCST requires a memory source: %q", baseOp, ins.Raw) + } + if properties.rounding != "" && (baseOp != "VCVTPD2PS" || ins.Args[0].Kind != OpReg) { + return true, false, fmt.Errorf("amd64 %s .%s requires a Z register source and Y destination: %q", baseOp, properties.rounding, ins.Raw) + } + + dstArg := ins.Args[len(ins.Args)-1] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects a vector destination: %q", baseOp, ins.Raw) + } + inputBytes, convertedLanes, outputBytes := 0, 0, 0 + switch baseOp { + case "CVTPD2PS", "VCVTPD2PSX": + inputBytes, convertedLanes, outputBytes = 16, 2, 16 + if !isAMD64XReg(dstArg.Reg) { + return true, false, fmt.Errorf("amd64 %s expects an X destination: %q", baseOp, ins.Raw) + } + case "VCVTPD2PSY": + inputBytes, convertedLanes, outputBytes = 32, 4, 16 + if !isAMD64XReg(dstArg.Reg) { + return true, false, fmt.Errorf("amd64 VCVTPD2PSY expects an X destination: %q", ins.Raw) + } + case "VCVTPD2PS": + inputBytes, convertedLanes, outputBytes = 64, 8, 32 + if !isAMD64YReg(dstArg.Reg) { + return true, false, fmt.Errorf("amd64 VCVTPD2PS expects a Y destination: %q", ins.Raw) + } + } + if c.goarch == "386" && baseOp == "CVTPD2PS" { + dstIndex, _ := amd64ParseXReg(dstArg.Reg) + if dstIndex > 7 { + return true, false, fmt.Errorf("386 CVTPD2PS destination is outside Go 1.27's X register class: %q", ins.Raw) + } + } + + sourceArg := ins.Args[0] + if sourceArg.Kind == OpReg { + if !amd64VectorRegisterHasWidth(sourceArg, inputBytes) { + return true, false, fmt.Errorf("amd64 %s source width does not match its form: %q", baseOp, ins.Raw) + } + if c.goarch == "386" { + switch baseOp { + case "CVTPD2PS": + index, _ := amd64ParseXReg(sourceArg.Reg) + if index > 7 { + return true, false, fmt.Errorf("386 CVTPD2PS source is outside Go 1.27's X register class: %q", ins.Raw) + } + case "VCVTPD2PS": + index, _ := amd64ParseZReg(sourceArg.Reg) + if index > 7 { + return true, false, fmt.Errorf("386 VCVTPD2PS source is outside Go 1.27's Yzm register class: %q", ins.Raw) + } + } + } + } else if !isAMD64MemoryOperand(sourceArg) { + return true, false, fmt.Errorf("amd64 %s expects a vector register or memory source: %q", baseOp, ins.Raw) + } + + mask := "" + if masked { + maskArg := ins.Args[1] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + maskIndex, ok := amd64ParseKReg(maskArg.Reg) + if !ok || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + var source string + if properties.broadcast { + scalar, err := c.evalF64(sourceArg) + if err != nil { + return true, false, err + } + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x double> poison, double %s, i32 0\n", seed, convertedLanes, scalar) + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x double> %%%s, <%d x double> poison, <%d x i32> zeroinitializer\n", splat, convertedLanes, seed, convertedLanes, convertedLanes) + source = "%" + splat + } else { + bytesValue, err := c.loadPackedCompareBytes(sourceArg, inputBytes) + if err != nil { + return true, false, err + } + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x double>\n", cast, inputBytes, bytesValue, convertedLanes) + source = "%" + cast + } + + converted := c.newTmp() + if properties.rounding == "" { + fmt.Fprintf(c.b, " %%%s = fptrunc <%d x double> %s to <%d x float>\n", converted, convertedLanes, source, convertedLanes) + } else { + fmt.Fprintf(c.b, " %%%s = call <%d x float> @llvm.experimental.constrained.fptrunc.v%df32.v%df64(<%d x double> %s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", converted, convertedLanes, convertedLanes, convertedLanes, convertedLanes, source, amd64FMA3RoundingMetadata(properties.rounding)) + } + result := "%" + converted + + if masked { + computedBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x float> %s to <%d x i32>\n", computedBits, convertedLanes, result, convertedLanes) + oldBytes, err := c.loadPackedCompareBytes(dstArg, outputBytes) + if err != nil { + return true, false, err + } + physicalLanes := outputBytes / 4 + oldAll := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i32>\n", oldAll, outputBytes, oldBytes, physicalLanes) + oldBits := "%" + oldAll + if physicalLanes != convertedLanes { + oldLow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i32> %s, <%d x i32> poison, <%d x i32> \n", oldLow, physicalLanes, oldBits, physicalLanes, convertedLanes) + oldBits = "%" + oldLow + } + maskedBits := amd64ApplyIntegerLaneMask(c, convertedLanes, 32, "%"+computedBits, oldBits, mask, properties.zeroing) + maskedValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x float>\n", maskedValue, convertedLanes, maskedBits, convertedLanes) + result = "%" + maskedValue + } + + if convertedLanes == 2 { + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x float> %s, <2 x float> zeroinitializer, <4 x i32> \n", widened, result) + result = "%" + widened + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x float> %s to <%d x i8>\n", out, outputBytes/4, result, outputBytes) + return true, false, c.storeVectorBytes(dstArg.Reg, outputBytes, "%"+out) +} diff --git a/amd64_lower_convert_narrow_test.go b/amd64_lower_convert_narrow_test.go new file mode 100644 index 000000000..2fddce7aa --- /dev/null +++ b/amd64_lower_convert_narrow_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedDoubleToSingleCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses four related tables for this family: + // CVTPD2PS: X/m128 -> X + // VCVTPD2PSX: X/m128 -> X + // VCVTPD2PSY: Y/m256 -> X + // VCVTPD2PS: Z/m512 -> Y + // The EVEX forms accept scalar-double broadcast and K1-K7 masking. Only + // the 512-bit source form accepts explicit rounding. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + // Go permits EVEX X/Y registers through 31 in 386 mode, while a + // Z source (the Yzm class) remains limited to Z0-Z7 there. + lastZ := 21 + if target.goarch == "386" { + lastZ = 7 + } + var src strings.Builder + src.WriteString("TEXT packeddoubletosingleforms(SB),NOSPLIT,$0-0\n") + src.WriteString("\tCVTPD2PS X0, X1\n") + src.WriteString("\tCVTPD2PS 8(AX), X2\n") + src.WriteString("\tVCVTPD2PSX X20, X0\n") + src.WriteString("\tVCVTPD2PSX (AX), X1\n") + src.WriteString("\tVCVTPD2PSX X20, K1, X21\n") + src.WriteString("\tVCVTPD2PSX.Z 16(AX), K2, X3\n") + src.WriteString("\tVCVTPD2PSX.BCST 24(AX), X4\n") + src.WriteString("\tVCVTPD2PSX.BCST.Z 32(AX), K3, X5\n") + src.WriteString("\tVCVTPD2PSY Y20, X0\n") + src.WriteString("\tVCVTPD2PSY (AX), X1\n") + src.WriteString("\tVCVTPD2PSY Y20, K4, X21\n") + src.WriteString("\tVCVTPD2PSY.Z 40(AX), K5, X3\n") + src.WriteString("\tVCVTPD2PSY.BCST 48(AX), X4\n") + src.WriteString("\tVCVTPD2PSY.BCST.Z 56(AX), K6, X5\n") + fmt.Fprintf(&src, "\tVCVTPD2PS Z%d, Y0\n", lastZ) + src.WriteString("\tVCVTPD2PS (AX), Y1\n") + fmt.Fprintf(&src, "\tVCVTPD2PS Z%d, K1, Y21\n", lastZ) + src.WriteString("\tVCVTPD2PS.Z 64(AX), K2, Y3\n") + src.WriteString("\tVCVTPD2PS.BCST 72(AX), Y4\n") + src.WriteString("\tVCVTPD2PS.BCST.Z 80(AX), K3, Y5\n") + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&src, "\tVCVTPD2PS.%s Z%d, Y6\n", rounding, lastZ) + } + fmt.Fprintf(&src, "\tVCVTPD2PS.RN_SAE.Z Z%d, K7, Y6\n", lastZ) + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packeddoubletosingleforms": {Name: "packeddoubletosingleforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-double-to-single-"+target.name+".ll", "packed-double-to-single-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedDoubleToSingleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CVTPD2PS X0", + "CVTPD2PS Y0, X1", + "CVTPD2PS X0, (AX)", + "CVTPD2PS.Z X0, X1", + "VCVTPD2PSX Y0, X1", + "VCVTPD2PSX X0, Y1", + "VCVTPD2PSY X0, X1", + "VCVTPD2PSY Y0, Y1", + "VCVTPD2PS Z0, X1", + "VCVTPD2PS Y0, Y1", + "VCVTPD2PS Z0, Z1", + "VCVTPD2PSX X0, K0, X1", + "VCVTPD2PSY.Z Y0, X1", + "VCVTPD2PS.Z Z0, Y1", + "VCVTPD2PSX.BCST X0, X1", + "VCVTPD2PSY.BCST Y0, X1", + "VCVTPD2PS.BCST Z0, Y1", + "VCVTPD2PSX.RN_SAE X0, X1", + "VCVTPD2PSY.RD_SAE Y0, X1", + "VCVTPD2PS.RU_SAE (AX), Y1", + "VCVTPD2PS.SAE Z0, Y1", + "VCVTPD2PS.BCST.RN_SAE (AX), Y1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDoubleToSingleRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "CVTPD2PS X8, X0", + "CVTPD2PS X0, X8", + "VCVTPD2PS Z8, Y0", + "VCVTPD2PS Z20, K1, Y0", + } { + assertX86PackedDoubleToSingleRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86PackedDoubleToSingleRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed double-to-single forms", instruction) + } +} diff --git a/amd64_lower_convert_samewidth.go b/amd64_lower_convert_samewidth.go new file mode 100644 index 000000000..ed918a931 --- /dev/null +++ b/amd64_lower_convert_samewidth.go @@ -0,0 +1,153 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SameWidthConversionMode uint8 + +const ( + amd64SameWidthIntToFloat amd64SameWidthConversionMode = iota + amd64SameWidthFloatToInt + amd64SameWidthFloatToIntTruncate + amd64SameWidthSqrt +) + +type amd64SameWidthConversionSpec struct { + laneBits int + mode amd64SameWidthConversionMode + sae bool + rawOnly bool +} + +var amd64SameWidthConversionSpecs = map[Op]amd64SameWidthConversionSpec{ + "VCVTDQ2PS": {laneBits: 32, mode: amd64SameWidthIntToFloat}, + "VCVTPS2DQ": {laneBits: 32, mode: amd64SameWidthFloatToInt}, + "VCVTTPS2DQ": {laneBits: 32, mode: amd64SameWidthFloatToIntTruncate, sae: true}, + "VSQRTPD": {laneBits: 64, mode: amd64SameWidthSqrt}, + "VSQRTPS": {laneBits: 32, mode: amd64SameWidthSqrt}, + "VSQRTPH": {laneBits: 16, mode: amd64SameWidthSqrt, rawOnly: true}, +} + +// lowerSameWidthPackedConversion implements all five opcodes sharing Go +// 1.27's _yvcvtdq2ps table. Although the table is shared, VCVTTPS2DQ enables +// SAE while the other four opcodes enable explicit rounding on Z forms. +func (c *amd64Ctx) lowerSameWidthPackedConversion(op Op, ins Instr) (ok bool, terminated bool, err error) { + if handled, terminated, err := c.lowerPackedNumericConversion(op, ins); handled { + return handled, terminated, err + } + if _, known, _ := x86SignedPackedFloatProperties(ins, c.goarch); known { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64SameWidthConversionSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if spec.mode == amd64SameWidthIntToFloat { + return c.lowerSignedPackedIntegerFloat(op, ins) + } + if spec.rawOnly && !ins.x86Encoded { + return true, false, fmt.Errorf("%s is absent from Go's named assembler table; raw encoding required", baseOp) + } + + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix { + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvcvtdq2ps encodings: %q", c.goarch, baseOp, ins.Raw) + } + if spec.sae { + if properties.rounding != "" { + return true, false, fmt.Errorf("%s %s supports SAE, not explicit rounding: %q", c.goarch, baseOp, ins.Raw) + } + } else if properties.sae { + return true, false, fmt.Errorf("%s %s supports explicit rounding, not SAE: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects source, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's vector-register class: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + explicitControl := properties.rounding != "" || properties.sae + if explicitControl && (byteWidth != 64 || source.Kind != OpReg) { + return true, false, fmt.Errorf("%s %s explicit rounding/SAE requires a Z register source: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth * 8 / spec.laneBits + mask := "" + if masked { + mask, err = c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + } + sourceBits, err := c.loadMaskedPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast, mask) + if err != nil { + return true, false, err + } + computedBits := c.emitSameWidthPackedConversion(spec, sourceBits, byteWidth, properties) + if masked { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + computedBits = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computedBits, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, computedBits, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitSameWidthPackedConversion(spec amd64SameWidthConversionSpec, sourceBits string, byteWidth int, properties amd64BinaryFloatingSuffix) string { + lanes := byteWidth * 8 / spec.laneBits + integerType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + floatType := amd64FMA3LLVMType(lanes, spec.laneBits) + switch spec.mode { + case amd64SameWidthSqrt: + floating := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", floating, integerType, sourceBits, floatType) + computed := c.newTmp() + if properties.rounding != "" { + name := fmt.Sprintf("llvm.experimental.constrained.sqrt.v%df%d", lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %%%s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", computed, floatType, name, floatType, floating, amd64FMA3RoundingMetadata(properties.rounding)) + } else { + name := fmt.Sprintf("llvm.sqrt.v%df%d", lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %%%s)\n", computed, floatType, name, floatType, floating) + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to %s\n", bits, floatType, computed, integerType) + return "%" + bits + } + panic("unreachable same-width packed conversion mode") +} diff --git a/amd64_lower_convert_samewidth_test.go b/amd64_lower_convert_samewidth_test.go new file mode 100644 index 000000000..4a26813b7 --- /dev/null +++ b/amd64_lower_convert_samewidth_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86SameWidthPackedConversionCompleteGoAssemblerForms(t *testing.T) { + // These five opcodes share Go 1.27's _yvcvtdq2ps table. All accept X/Y/Z + // register or memory sources, scalar-memory broadcast, and K1-K7 masks. + // The Z forms of all but VCVTTPS2DQ accept explicit rounding; the + // truncating conversion accepts SAE instead. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 21 + if target.goarch == "386" { + zLast = 7 + } + var src strings.Builder + src.WriteString("TEXT samewidthpackedconversionforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VCVTDQ2PS", "VCVTPS2DQ", "VCVTTPS2DQ", "VSQRTPD", "VSQRTPS"} { + high := 20 + if strings.HasPrefix(op, "VCVT") && target.goarch == "386" { + // The exact native conversion cannot borrow the wider + // software bank used by the other conversion families. + high = 4 + } + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s %s%d, %s%d\n", op, width, high, width, high+1) + fmt.Fprintf(&src, "\t%s %s%d, K1, %s%d\n", op, width, high, width, high+1) + fmt.Fprintf(&src, "\t%s.Z (AX), K2, %s%d\n", op, width, high+2) + fmt.Fprintf(&src, "\t%s.BCST 8(AX), %s%d\n", op, width, high+2) + fmt.Fprintf(&src, "\t%s.BCST.Z 8(AX), K3, %s%d\n", op, width, high+3) + } + fmt.Fprintf(&src, "\t%s Z0, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s (AX), Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s Z6, K4, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.Z (AX), K5, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST 8(AX), Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST.Z 8(AX), K6, Z%d\n", op, zLast) + if op == "VCVTTPS2DQ" { + fmt.Fprintf(&src, "\t%s.SAE Z6, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.SAE.Z Z6, K7, Z%d\n", op, zLast) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&src, "\t%s.%s Z6, Z%d\n", op, rounding, zLast) + } + fmt.Fprintf(&src, "\t%s.RN_SAE.Z Z6, K7, Z%d\n", op, zLast) + } + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "samewidthpackedconversionforms": {Name: "samewidthpackedconversionforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "same-width-packed-conversion-"+target.name+".ll", "same-width-packed-conversion-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86SameWidthPackedConversionRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VCVTDQ2PS X0", + "VCVTPS2DQ X0, Y1", + "VCVTTPS2DQ X0, (AX)", + "VSQRTPD X0, K0, X1", + "VSQRTPS.Z X0, X1", + "VCVTDQ2PS.BCST X0, X1", + "VCVTPS2DQ.SAE Z0, Z1", + "VCVTTPS2DQ.RN_SAE Z0, Z1", + "VSQRTPD.RN_SAE X0, X1", + "VSQRTPS.RN_SAE (AX), Z1", + "VCVTDQ2PS.BCST.RN_SAE (AX), Z1", + "VCVTTPS2DQ.BCST.SAE (AX), Z1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86SameWidthPackedConversionRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "VCVTDQ2PS Z8, Z0", + "VCVTPS2DQ Z0, Z8", + "VCVTTPS2DQ.SAE Z8, Z0", + "VSQRTPD.RN_SAE Z0, Z8", + "VSQRTPS Z0, K1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86SameWidthPackedConversionRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86SameWidthPackedConversionRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvcvtdq2ps forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_counter_zero_branch_test.go b/amd64_lower_counter_zero_branch_test.go new file mode 100644 index 000000000..cce53dac6 --- /dev/null +++ b/amd64_lower_counter_zero_branch_test.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86CounterZeroBranchCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := "TEXT counterzerobranchforms(SB),NOSPLIT,$0-0\n" + + "\tJCXZW width16\n" + + "width16:\n" + + "\tJCXZL width32\n" + + "width32:\n" + + "\tJCXZQ width64\n" + + "width64:\n" + + "\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "counterzerobranchforms": {Name: "counterzerobranchforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "counter-zero-branch-"+target.name+".ll", "counter-zero-branch-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86CounterZeroBranchMatchesGoAssemblerAddressWidths(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + widths map[string]string + }{ + { + name: "amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", + widths: map[string]string{"JCXZW": "i64", "JCXZL": "i32", "JCXZQ": "i64"}, + }, + { + name: "386", goarch: "386", triple: "i386-unknown-linux-gnu", + widths: map[string]string{"JCXZW": "i32", "JCXZL": "i16", "JCXZQ": "i32"}, + }, + } { + for op, width := range target.widths { + t.Run(target.name+"_"+op, func(t *testing.T) { + source := fmt.Sprintf("TEXT counterzero(SB),NOSPLIT,$0-0\n\t%s zero\n\tRET\nzero:\n\tRET\n", op) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"counterzero": {Name: "counterzero", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "icmp eq "+width) { + t.Fatalf("%s/%s did not test the Go assembler's %s counter width:\n%s", target.goarch, op, width, ll) + } + }) + } + } +} + +func TestTranslateX86CounterZeroBranchRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "JCXZW", + "JCXZL $1, done", + "JCXZQ done, other", + "JCXZQ AX", + "JCXZQ.Z done", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),NOSPLIT,$0-0\n\t" + instruction + "\ndone:\nother:\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yloop table", instruction) + } + }) + } +} diff --git a/amd64_lower_crc32.go b/amd64_lower_crc32.go index 054f910af..3a9944a03 100644 --- a/amd64_lower_crc32.go +++ b/amd64_lower_crc32.go @@ -3,69 +3,57 @@ package plan9asm import "fmt" func (c *amd64Ctx) lowerCrc32(op Op, ins Instr) (ok bool, terminated bool, err error) { + sourceType := LLVMType("") + intrinsic := "" + resultType := I32 switch op { - case "CRC32B", "CRC32W", "CRC32L", "CRC32Q": - // ok + case "CRC32B": + sourceType = I8 + intrinsic = "llvm.x86.sse42.crc32.32.8" + case "CRC32W": + sourceType = I16 + intrinsic = "llvm.x86.sse42.crc32.32.16" + case "CRC32L": + sourceType = I32 + intrinsic = "llvm.x86.sse42.crc32.32.32" + case "CRC32Q": + if c.goarch == "386" { + return true, false, fmt.Errorf("386 CRC32Q is illegal in 32-bit mode: %q", ins.Raw) + } + sourceType = I64 + resultType = I64 + intrinsic = "llvm.x86.sse42.crc32.64.64" default: return false, false, nil } - if len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects mem, reg: %q", op, ins.Raw) + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects register/memory source and register destination: %q", c.goarch, op, ins.Raw) + } + source, destination := ins.Args[0], ins.Args[1] + if source.Kind == OpReg { + valid := isX86YrlRegisterForArch(source.Reg, c.goarch) + if op == "CRC32B" { + valid = isGoYmbRegisterForArch(source.Reg, c.goarch) + } + if !valid { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's %s class: %q", c.goarch, op, map[bool]string{true: "Ymb", false: "Yml"}[op == "CRC32B"], ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's register/memory class: %q", c.goarch, op, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, op, ins.Raw) } - src := ins.Args[0].Mem - dst := ins.Args[1].Reg - addr, err := c.addrFromMem(src) + sourceValue, err := c.evalIntSized(source, sourceType) if err != nil { return true, false, err } - p := c.ptrFromAddrI64(addr) - - dv, err := c.loadReg(dst) + crcValue, err := c.evalIntSized(destination, resultType) if err != nil { return true, false, err } - - switch op { - case "CRC32Q": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i64 @llvm.x86.sse42.crc32.64.64(i64 %s, i64 %%%s)\n", call, dv, ld) - return true, false, c.storeReg(dst, "%"+call) - - case "CRC32L": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", ld, p) - crc32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", crc32, dv) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse42.crc32.32.32(i32 %%%s, i32 %%%s)\n", call, crc32, ld) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(dst, "%"+z) - - case "CRC32W": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i16, ptr %s, align 1\n", ld, p) - crc32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", crc32, dv) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse42.crc32.32.16(i32 %%%s, i16 %%%s)\n", call, crc32, ld) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(dst, "%"+z) - - case "CRC32B": - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i8, ptr %s, align 1\n", ld, p) - crc32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", crc32, dv) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse42.crc32.32.8(i32 %%%s, i8 %%%s)\n", call, crc32, ld) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) - return true, false, c.storeReg(dst, "%"+z) - } - return true, false, fmt.Errorf("amd64: unknown CRC32 op %s", op) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s)\n", call, resultType, intrinsic, resultType, crcValue, sourceType, sourceValue) + return true, false, c.storeRegSized(destination.Reg, resultType, "%"+call) } diff --git a/amd64_lower_crc32_complete_test.go b/amd64_lower_crc32_complete_test.go new file mode 100644 index 000000000..815c66991 --- /dev/null +++ b/amd64_lower_crc32_complete_test.go @@ -0,0 +1,232 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86CRC32CompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA crc32data+0(SB)/8, $1\n") + source.WriteString("GLOBL crc32data(SB), $8\n") + source.WriteString("TEXT crc32forms(SB),$0-0\n") + for _, width := range []string{"B", "W", "L"} { + op := "CRC32" + width + sourceRegister := "AX" + if width == "B" { + sourceRegister = "AL" + } + fmt.Fprintf(&source, "\t%s %s, BX\n", op, sourceRegister) + fmt.Fprintf(&source, "\t%s 8(BX), CX\n", op) + fmt.Fprintf(&source, "\t%s crc32data(SB), DX\n", op) + if width == "B" { + fmt.Fprintf(&source, "\t%s AX, DI\n", op) + } + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11, R12\n", op) + } else { + fmt.Fprintf(&source, "\t%s %s, SP\n", op, sourceRegister) + } + } + if goarch == "amd64" { + for _, form := range []string{ + "CRC32Q AX, BX", + "CRC32Q 8(BX), CX", + "CRC32Q crc32data(SB), DX", + "CRC32Q R11, R12", + } { + fmt.Fprintf(&source, "\t%s\n", form) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86CRC32CompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86CRC32CompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"crc32forms": {Name: "crc32forms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "@llvm.x86.sse42.crc32.32.8", + "@llvm.x86.sse42.crc32.32.16", + "@llvm.x86.sse42.crc32.32.32", + `"target-features"="+crc32,+sse4.2"`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("CRC32 lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@llvm.x86.sse42.crc32.64.64") { + t.Fatalf("CRC32Q lowering omitted i64 intrinsic:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "crc32-"+target.name+".ll", "crc32-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86CRC32RejectsFormsOutsideGo127Optabs(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CRC32B $1, AX"}, + {goarch: "amd64", instruction: "CRC32W AL, AX"}, + {goarch: "amd64", instruction: "CRC32L X0, AX"}, + {goarch: "amd64", instruction: "CRC32Q AX, 0(BX)"}, + {goarch: "amd64", instruction: "CRC32L AX, X0"}, + {goarch: "amd64", instruction: "CRC32L AX, AL"}, + {goarch: "amd64", instruction: "CRC32L AX"}, + {goarch: "amd64", instruction: "CRC32L.Z AX, BX"}, + {goarch: "386", instruction: "CRC32Q AX, BX"}, + {goarch: "386", instruction: "CRC32L R8, AX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ycrc32b/ycrc32l tables", test.instruction) + } + }) + } +} + +func TestAMD64CRC32RuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT crc32semantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + + MOVL $0x12345678, BX + MOVL $0xab00, AX + CRC32B AH, BX + MOVQ BX, 0(DI) + + MOVL $0x23456789, BX + MOVL $0xcdef, AX + CRC32W AX, BX + MOVQ BX, 8(DI) + + MOVL $0x3456789a, BX + MOVL $0x89abcdef, 40(DI) + CRC32L 40(DI), BX + MOVQ BX, 16(DI) + + MOVL $0x456789ab, BX + MOVQ $0x0123456789abcdef, 48(DI) + CRC32Q 48(DI), BX + MOVQ BX, 24(DI) + + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVL $0, BX + CRC32B AL, BX + SETCS 32(DI) + SETOS 33(DI) + SETEQ 34(DI) + SETMI 35(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "crc32semantics": { + Name: "crc32semantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void crc32semantics(uint8_t *); +static uint32_t crc32c(uint32_t crc, uint64_t value, unsigned bytes) { + for (unsigned i = 0; i < bytes; i++) { + crc ^= (uint8_t)value; + value >>= 8; + for (unsigned bit = 0; bit < 8; bit++) + crc = (crc >> 1) ^ (UINT32_C(0x82f63b78) & (uint32_t)-(int32_t)(crc & 1)); + } + return crc; +} +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[64] = {0}; + crc32semantics(out); + if (load64(out + 0) != crc32c(UINT32_C(0x12345678), UINT64_C(0xab), 1)) return 10; + if (load64(out + 8) != crc32c(UINT32_C(0x23456789), UINT64_C(0xcdef), 2)) return 11; + if (load64(out + 16) != crc32c(UINT32_C(0x3456789a), UINT64_C(0x89abcdef), 4)) return 12; + if (load64(out + 24) != crc32c(UINT32_C(0x456789ab), UINT64_C(0x0123456789abcdef), 8)) return 13; + if (out[32] != 1 || out[33] != 1 || out[34] != 0 || out[35] != 1) return 14; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "crc32_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_descriptor.go b/amd64_lower_descriptor.go new file mode 100644 index 000000000..5d8cf6f98 --- /dev/null +++ b/amd64_lower_descriptor.go @@ -0,0 +1,166 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86DescriptorOps = map[string]struct{}{ + "LGDT": {}, "LIDT": {}, "SGDT": {}, "SIDT": {}, + "LLDT": {}, "LTR": {}, "LMSW": {}, + "SLDTW": {}, "SLDTL": {}, "SLDTQ": {}, + "SMSWW": {}, "SMSWL": {}, "SMSWQ": {}, + "STRW": {}, "STRL": {}, "STRQ": {}, +} + +// lowerDescriptorTable covers Go 1.27's complete x86 descriptor-table and +// system-selector family. These instructions are privileged or may be blocked +// by UMIP, so the lowering preserves them as LLVM inline assembly rather than +// approximating their machine-state effects. +func (c *amd64Ctx) lowerDescriptorTable(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ok, terminated, err := c.lowerSegmentQuery(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerFarPointer(op, ins); ok { + return ok, terminated, err + } + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if _, ok := x86DescriptorOps[baseOp]; !ok { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one operand: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && strings.HasSuffix(baseOp, "Q") { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + + operand := ins.Args[0] + switch baseOp { + case "LGDT", "LIDT": + if !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects Go 1.27 Ym memory: %q", c.goarch, baseOp, ins.Raw) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(operand) + if err != nil { + return true, false, err + } + bytes := 10 + if c.goarch == "386" { + bytes = 6 + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + strings.ToLower(baseOp)+" $0", "*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, bytes, ptr) + return true, false, nil + + case "SGDT", "SIDT": + if !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects Go 1.27 Ym memory: %q", c.goarch, baseOp, ins.Raw) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(operand) + if err != nil { + return true, false, err + } + bytes := 10 + if c.goarch == "386" { + bytes = 6 + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + strings.ToLower(baseOp)+" $0", "=*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, bytes, ptr) + return true, false, nil + + case "LLDT", "LTR", "LMSW": + if operand.Kind != OpReg && !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects Go 1.27 Yml register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if operand.Kind == OpReg { + if !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s register is outside Go 1.27's Yml class: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.evalIntSized(operand, I16) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i16 %s)\n", + strings.ToLower(baseOp)+" $0", "r,~{memory},~{dirflag},~{fpsr},~{flags}", value) + return true, false, nil + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(operand) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype(i16) %s)\n", + strings.ToLower(baseOp)+" $0", "*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, ptr) + return true, false, nil + + case "SLDTW", "SLDTL", "SLDTQ", "SMSWW", "SMSWL", "SMSWQ", "STRW", "STRL", "STRQ": + return true, false, c.lowerDescriptorResult(baseOp, operand, ins) + default: + panic("descriptor opcode precheck drift") + } +} + +func (c *amd64Ctx) lowerDescriptorResult(op string, destination Operand, ins Instr) error { + if destination.Kind != OpReg && !isAMD64MemoryOperand(destination) { + return fmt.Errorf("%s %s expects Go 1.27 Yml register or memory: %q", c.goarch, op, ins.Raw) + } + if destination.Kind == OpReg && !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return fmt.Errorf("%s %s register is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + typ := I16 + switch op[len(op)-1] { + case 'L': + typ = I32 + case 'Q': + typ = I64 + } + asm := strings.ToLower(op) + " $0" + if destination.Kind == OpReg { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", + result, typ, asm, "=r,~{memory},~{dirflag},~{fpsr},~{flags}") + return c.storeRegSized(destination.Reg, typ, "%"+result) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(destination) + if err != nil { + return err + } + // Intel defines every memory destination as m16 even when Go selects the + // L/Q spelling (the operand-size prefix only affects a register target). + // LLVM's integrated assembler consequently rejects sldtl/sldtq, smswl/q, + // and strl/q with memory, so emit the architectural unsuffixed m16 form. + memoryAsm := strings.ToLower(op[:len(op)-1]) + " $0" + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype(%s) %s)\n", + memoryAsm, "=*m,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, I16, ptr) + return nil +} + +func (c *amd64Ctx) x86DescriptorMemoryPointer(operand Operand) (ptr, ptrType string, err error) { + switch operand.Kind { + case OpMem: + return c.ptrFromMem(operand.Mem) + case OpSym: + ptr, err := c.ptrFromSB(operand.Sym) + return ptr, "ptr", err + case OpFP: + if c.classicFrame != "" { + return c.classicFramePtr(operand.FPOffset), "ptr", nil + } + if ptr := c.fpParamAlloca[operand.FPOffset]; ptr != "" { + return ptr, "ptr", nil + } + if ptr, _, ok := c.fpResultAlloca(operand.FPOffset); ok { + return ptr, "ptr", nil + } + return "", "", fmt.Errorf("%s descriptor memory operand has no mutable FP slot at +%d(FP)", c.goarch, operand.FPOffset) + default: + return "", "", fmt.Errorf("%s descriptor instruction expected memory, got %s", c.goarch, operand.String()) + } +} diff --git a/amd64_lower_dot_float.go b/amd64_lower_dot_float.go new file mode 100644 index 000000000..98c97e7b8 --- /dev/null +++ b/amd64_lower_dot_float.go @@ -0,0 +1,141 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedFloatingDotForm uint8 + +const ( + // amd64PackedFloatingDotLegacy is Go's yxshuf table: imm8, X/m128, X. + amd64PackedFloatingDotLegacy amd64PackedFloatingDotForm = iota + // amd64PackedFloatingDotVEX128 is Go's _yvdppd table: imm8, X/m128, + // X, X. The Go 386 frontend cannot represent the four operands. + amd64PackedFloatingDotVEX128 + // amd64PackedFloatingDotVEX128Or256 is Go's _yvblendpd table, adding + // the corresponding Y/m256, Y, Y row. + amd64PackedFloatingDotVEX128Or256 +) + +type amd64PackedFloatingDotSpec struct { + form amd64PackedFloatingDotForm + laneBits int +} + +// amd64PackedFloatingDotSpecs is the complete Go 1.27 grammar for the SSE4.1 +// and AVX floating dot-product family. The form class captures operand count, +// legal vector widths, and the 386 frontend restriction; laneBits selects the +// hardware intrinsic and immediate-bit interpretation. +var amd64PackedFloatingDotSpecs = map[Op]amd64PackedFloatingDotSpec{ + "DPPD": {form: amd64PackedFloatingDotLegacy, laneBits: 64}, + "DPPS": {form: amd64PackedFloatingDotLegacy, laneBits: 32}, + "VDPPD": {form: amd64PackedFloatingDotVEX128, laneBits: 64}, + "VDPPS": {form: amd64PackedFloatingDotVEX128Or256, laneBits: 32}, +} + +func (c *amd64Ctx) lowerPackedFloatingDot(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64PackedFloatingDotSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) == 0 || ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("%s %s expects a Yu8 immediate: %q", c.goarch, baseOp, ins.Raw) + } + + if spec.form == amd64PackedFloatingDotLegacy { + return c.lowerLegacyPackedFloatingDot(baseOp, spec, ins) + } + return c.lowerVEXPackedFloatingDot(baseOp, spec, ins) +} + +func (c *amd64Ctx) lowerLegacyPackedFloatingDot(baseOp string, spec amd64PackedFloatingDotSpec, ins Instr) (bool, bool, error) { + if len(ins.Args) != 3 || !c.isGoVEXVectorRegister(ins.Args[2], 16, false) { + return true, false, fmt.Errorf("%s %s expects imm8, X/m128, X: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoVEXVectorRegister(ins.Args[1], 16, true) { + return true, false, fmt.Errorf("%s %s source must be X/m128: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[1].Kind == OpMem && !x86MemoryRegistersValidForArch(ins.Args[1].Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s source uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + first, err := c.loadPackedFloatingDotOperand(ins.Args[1], 16, spec.laneBits) + if err != nil { + return true, false, err + } + second, err := c.loadPackedFloatingDotOperand(ins.Args[2], 16, spec.laneBits) + if err != nil { + return true, false, err + } + result := c.emitPackedFloatingDot(spec, 16, first, second, uint8(ins.Args[0].Imm)) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <16 x i8>\n", out, amd64FMA3LLVMType(128/spec.laneBits, spec.laneBits), result) + return true, false, c.storeX(ins.Args[2].Reg, "%"+out) +} + +func (c *amd64Ctx) lowerVEXPackedFloatingDot(baseOp string, spec amd64PackedFloatingDotSpec, ins Instr) (bool, bool, error) { + if c.goarch == "386" { + return true, false, fmt.Errorf("386 %s exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 4 || ins.Args[3].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects imm8, source1, source2, destination: %q", baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(ins.Args[3].Reg) + if byteWidth != 16 && (spec.form != amd64PackedFloatingDotVEX128Or256 || byteWidth != 32) { + return true, false, fmt.Errorf("amd64 %s destination width is absent from its Go 1.27 table: %q", baseOp, ins.Raw) + } + if !c.isGoVEXVectorRegister(ins.Args[3], byteWidth, false) || !c.isGoVEXVectorRegister(ins.Args[2], byteWidth, false) { + return true, false, fmt.Errorf("amd64 %s second source and destination must be matching VEX registers: %q", baseOp, ins.Raw) + } + if !c.isGoVEXVectorRegister(ins.Args[1], byteWidth, true) { + return true, false, fmt.Errorf("amd64 %s first source must be a matching VEX register or memory: %q", baseOp, ins.Raw) + } + if ins.Args[1].Kind == OpMem && !x86MemoryRegistersValidForArch(ins.Args[1].Mem, c.goarch) { + return true, false, fmt.Errorf("amd64 %s source uses an out-of-range address register: %q", baseOp, ins.Raw) + } + first, err := c.loadPackedFloatingDotOperand(ins.Args[1], byteWidth, spec.laneBits) + if err != nil { + return true, false, err + } + second, err := c.loadPackedFloatingDotOperand(ins.Args[2], byteWidth, spec.laneBits) + if err != nil { + return true, false, err + } + result := c.emitPackedFloatingDot(spec, byteWidth, first, second, uint8(ins.Args[0].Imm)) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, amd64FMA3LLVMType(byteWidth*8/spec.laneBits, spec.laneBits), result, byteWidth) + return true, false, c.storeVectorBytes(ins.Args[3].Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadPackedFloatingDotOperand(operand Operand, byteWidth, laneBits int) (string, error) { + bytesValue, err := c.loadPackedCompareBytes(operand, byteWidth) + if err != nil { + return "", err + } + lanes := byteWidth * 8 / laneBits + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", result, byteWidth, bytesValue, amd64FMA3LLVMType(lanes, laneBits)) + return "%" + result, nil +} + +func (c *amd64Ctx) emitPackedFloatingDot(spec amd64PackedFloatingDotSpec, byteWidth int, first, second string, immediate uint8) string { + lanes := byteWidth * 8 / spec.laneBits + typeName := amd64FMA3LLVMType(lanes, spec.laneBits) + intrinsic := "llvm.x86.sse41.dpps" + if spec.laneBits == 64 { + intrinsic = "llvm.x86.sse41.dppd" + } else if byteWidth == 32 { + intrinsic = "llvm.x86.avx.dp.ps.256" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, i8 %d)\n", result, typeName, intrinsic, typeName, second, typeName, first, immediate) + return "%" + result +} diff --git a/amd64_lower_dot_float_test.go b/amd64_lower_dot_float_test.go new file mode 100644 index 000000000..368077684 --- /dev/null +++ b/amd64_lower_dot_float_test.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedFloatingDotGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64PackedFloatingDotSpec{ + "DPPD": {form: amd64PackedFloatingDotLegacy, laneBits: 64}, + "DPPS": {form: amd64PackedFloatingDotLegacy, laneBits: 32}, + "VDPPD": {form: amd64PackedFloatingDotVEX128, laneBits: 64}, + "VDPPS": {form: amd64PackedFloatingDotVEX128Or256, laneBits: 32}, + } + if len(amd64PackedFloatingDotSpecs) != len(expected) { + t.Fatalf("floating dot grammar has %d entries, want %d", len(amd64PackedFloatingDotSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedFloatingDotSpecs[op]; !ok { + t.Errorf("floating dot grammar omitted %s", op) + } else if got != want { + t.Errorf("floating dot grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PackedFloatingDotCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var sourceBuilder strings.Builder + sourceBuilder.WriteString("TEXT floatdotforms(SB),$0-0\n") + fmt.Fprintf(&sourceBuilder, "\tDPPD $0x31, X0, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tDPPD $0x31, 0(AX), X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tDPPS $0xf1, X0, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tDPPS $0xf1, 16(AX), X%d\n", last) + if target.goarch == "amd64" { + fmt.Fprintf(&sourceBuilder, "\tVDPPD $0x31, X0, X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPD $0x31, 32(AX), X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, X0, X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, 48(AX), X1, X%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, Y0, Y1, Y%d\n", last) + fmt.Fprintf(&sourceBuilder, "\tVDPPS $0xf1, 64(AX), Y1, Y%d\n", last) + } + sourceBuilder.WriteString("\tRET\n") + source := sourceBuilder.String() + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"floatdotforms": {Name: "floatdotforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "call <2 x double> @llvm.x86.sse41.dppd", + "call <4 x float> @llvm.x86.sse41.dpps", + } { + if !strings.Contains(ir, want) { + t.Fatalf("packed floating dot lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "call <8 x float> @llvm.x86.avx.dp.ps.256") { + t.Fatalf("packed floating dot lowering omitted Y-width intrinsic:\n%s", ir) + } + wantFeatures := `"target-features"="+sse4.1"` + if target.goarch == "amd64" { + wantFeatures = `"target-features"="+avx,+sse4.1"` + } + if !strings.Contains(ir, wantFeatures) { + t.Fatalf("packed floating dot lowering omitted %s:\n%s", wantFeatures, ir) + } + compileLLVMToObject(t, llc, target.triple, "float-dot-"+target.name+".ll", "float-dot-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedFloatingDotRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "DPPD $-1, X0, X1"}, + {goarch: "amd64", instruction: "DPPD $256, X0, X1"}, + {goarch: "amd64", instruction: "DPPS.Z $1, X0, X1"}, + {goarch: "amd64", instruction: "DPPD $1, Y0, Y1"}, + {goarch: "amd64", instruction: "VDPPD $1, Y0, Y1, Y2"}, + {goarch: "amd64", instruction: "VDPPS $1, Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VDPPS $1, X0, Y1, Y2"}, + {goarch: "386", instruction: "DPPD $1, X0, X8"}, + {goarch: "386", instruction: "VDPPS $1, Y0, Y1, Y2"}, + {goarch: "386", instruction: "VDPPS $1, Y0, Y1, Y8"}, + {goarch: "386", instruction: "VDPPS $1, 8(R9), Y1, Y2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64PackedFloatingDotRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT floatdotsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), CX + MOVQ b+16(FP), DX + MOVUPS 0(CX), X0 + MOVUPS 0(DX), X1 + STC + DPPS $0xf1, X0, X1 + MOVUPS X1, 0(AX) + SETCS 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "floatdotsemantics": { + Name: "floatdotsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatdotsemantics(uint8_t *, const float *, const float *); +int main(void) { + uint8_t out[17] = {0}; + const float a[4] = {1.0f, 2.0f, 3.0f, 4.0f}; + const float b[4] = {5.0f, 6.0f, 7.0f, 8.0f}; + float got[4]; + floatdotsemantics(out, a, b); + memcpy(got, out, 16); + if (got[0] != 70.0f || got[1] != 0.0f || got[2] != 0.0f || got[3] != 0.0f) return 10; + if (out[16] != 1) return 11; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_float_dot_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_dot_product.go b/amd64_lower_dot_product.go new file mode 100644 index 000000000..87e9e1d5b --- /dev/null +++ b/amd64_lower_dot_product.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedDotProductSpec struct { + inputBits int + saturate bool + bfloat bool +} + +var amd64PackedDotProductSpecs = map[Op]amd64PackedDotProductSpec{ + "VPDPBUSD": {inputBits: 8}, + "VPDPBUSDS": {inputBits: 8, saturate: true}, + "VPDPWSSD": {inputBits: 16}, + "VPDPWSSDS": {inputBits: 16, saturate: true}, + "VDPBF16PS": {inputBits: 16, bfloat: true}, +} + +// lowerPackedDotProduct implements the four VNNI dot-product opcodes sharing +// Go 1.27's _yvblendmpd table. Plan 9 lists the signed r/m source first and +// the encoded V source second; VPDPBUSD treats that second source as unsigned. +func (c *amd64Ctx) lowerPackedDotProduct(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64PackedDotProductSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if spec.bfloat && !ins.x86Encoded { + return true, false, fmt.Errorf("%s has no named Go encoder form; raw encoding required", baseOp) + } + + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvblendmpd encodings: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects source2, source1, [K mask,] accumulator: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7 as its third operand: %q", baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s accumulator must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || + !c.isGoPackedVectorMoveRegister(destination, byteWidth) || + !c.isGoPackedVectorMoveRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s source1 and accumulator must be matching Go vector registers: %q", c.goarch, baseOp, ins.Raw) + } + rmSource := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source2: %q", c.goarch, baseOp, ins.Raw) + } + } else if rmSource.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(rmSource, byteWidth) { + return true, false, fmt.Errorf("%s %s source2 register must match the accumulator width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s source2 must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + dwordLanes := byteWidth / 4 + inputLanes := byteWidth * 8 / spec.inputBits + mask := "" + if masked { + mask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + // One mask bit controls a complete output dword and its input pair/group. + rmDwords, err := c.loadMaskedPackedCompareLanes(rmSource, byteWidth, 32, properties.broadcast, mask) + if err != nil { + return true, false, err + } + rmValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i%d>\n", rmValue, dwordLanes, rmDwords, inputLanes, spec.inputBits) + vBytes, err := c.loadPackedCompareBytes(ins.Args[1], byteWidth) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + rm := "%" + rmValue + v := c.bitcastVectorBytesToIntegerLanes(byteWidth, inputLanes, spec.inputBits, vBytes) + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, dwordLanes, 32, oldBytes) + computed := c.emitPackedDotProduct(spec, dwordLanes, rm, v, old) + if masked { + computed = amd64ApplyI32LaneMask(c, dwordLanes, computed, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i8>\n", out, dwordLanes, computed, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +// LLVM's dedicated intrinsic preserves the instruction's RNE, DAZ/FTZ and +// NaN-priority semantics independently of MXCSR. Generic fmul/fadd cannot. +func (c *amd64Ctx) emitBF16DotProduct(lanes int, rm, v, accumulator string) string { + left, right, old := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x bfloat>\n", left, lanes*2, v, lanes*2) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x bfloat>\n", right, lanes*2, rm, lanes*2) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x float>\n", old, lanes, accumulator, lanes) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <%d x float> @llvm.x86.avx512bf16.dpbf16ps.%d(<%d x float> %%%s, <%d x bfloat> %%%s, <%d x bfloat> %%%s)\n", + result, lanes, lanes*32, lanes, old, lanes*2, left, lanes*2, right) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x float> %%%s to <%d x i32>\n", bits, lanes, result, lanes) + return "%" + bits +} + +func (c *amd64Ctx) emitPackedDotProduct(spec amd64PackedDotProductSpec, dwordLanes int, rm, v, accumulator string) string { + if spec.bfloat { + return c.emitBF16DotProduct(dwordLanes, rm, v, accumulator) + } + inputsPerDword := 32 / spec.inputBits + result := "poison" + for lane := 0; lane < dwordLanes; lane++ { + accumulatorLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i32> %s, i32 %d\n", accumulatorLane, dwordLanes, accumulator, lane) + wideAccumulator := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i32 %%%s to i64\n", wideAccumulator, accumulatorLane) + total := "%" + wideAccumulator + for item := 0; item < inputsPerDword; item++ { + inputLane := lane*inputsPerDword + item + rmLane := c.newTmp() + vLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", rmLane, dwordLanes*inputsPerDword, spec.inputBits, rm, inputLane) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", vLane, dwordLanes*inputsPerDword, spec.inputBits, v, inputLane) + wideRM := c.newTmp() + wideV := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i%d %%%s to i64\n", wideRM, spec.inputBits, rmLane) + if spec.inputBits == 8 { + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", wideV, vLane) + } else { + fmt.Fprintf(c.b, " %%%s = sext i16 %%%s to i64\n", wideV, vLane) + } + product := c.newTmp() + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, %%%s\n", product, wideRM, wideV) + fmt.Fprintf(c.b, " %%%s = add i64 %s, %%%s\n", next, total, product) + total = "%" + next + } + if spec.saturate { + above := c.newTmp() + below := c.newTmp() + capped := c.newTmp() + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt i64 %s, 2147483647\n", above, total) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 2147483647, i64 %s\n", capped, above, total) + fmt.Fprintf(c.b, " %%%s = icmp slt i64 %%%s, -2147483648\n", below, capped) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 -2147483648, i64 %%%s\n", bounded, below, capped) + total = "%" + bounded + } + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", narrow, total) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i32> %s, i32 %%%s, i32 %d\n", inserted, dwordLanes, result, narrow, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_dot_product_test.go b/amd64_lower_dot_product_test.go new file mode 100644 index 000000000..cab55ff6c --- /dev/null +++ b/amd64_lower_dot_product_test.go @@ -0,0 +1,119 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedDotProductCompleteGoAssemblerForms(t *testing.T) { + // These four opcodes share Go 1.27's six-row _yvblendmpd table. Each + // width accepts register/memory source 2, register source 1, and an + // accumulator destination; amd64 additionally accepts K1-K7 merge/.Z. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 22 + if target.goarch == "386" { + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packeddotproductforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPDPBUSD", "VPDPBUSDS", "VPDPWSSD", "VPDPWSSDS"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20, %s21\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST 12(AX), %s20, %s22\n", op, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 8(AX), %s21, K7, %s22\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z 12(AX), %s21, K2, %s22\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s.BCST 12(AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K3, Z22\n", op) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K4, Z22\n", op) + fmt.Fprintf(&source, "\t%s.BCST.Z 12(AX), Z21, K5, Z22\n", op) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packeddotproductforms": {Name: "packeddotproductforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-dot-product-"+target.name+".ll", "packed-dot-product-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedDotProductRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPDPBUSD X0, X1", + "VPDPBUSDS X0, X1, (AX)", + "VPDPWSSD X0, Y1, X2", + "VPDPWSSDS Z0, Z1, Y2", + "VPDPBUSD X0, X1, K0, X2", + "VPDPBUSD X0, K1, X1, X2", + "VPDPBUSD.Z X0, X1, X2", + "VPDPBUSDS.SAE X0, X1, X2", + "VPDPWSSD.BCST X0, X1, X2", + "VPDPWSSDS.BCST 8(AX), Y1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDotProductRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPDPBUSD Z8, Z0, Z1", + "VPDPBUSDS Z0, Z8, Z1", + "VPDPWSSD Z0, Z1, Z8", + "VPDPWSSDS X0, X1, K1, X2", + "VPDPBUSD.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedDotProductRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedDotProductRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvblendmpd forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_duplicate.go b/amd64_lower_duplicate.go new file mode 100644 index 000000000..d8fbf2fc6 --- /dev/null +++ b/amd64_lower_duplicate.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerDuplicateMove implements the complete Go x86 duplicate-move family. +// The legacy forms use yxm (X/m -> X), while all V-prefixed forms share +// _yvmovddup (X/Y/Z memory or matching register, with optional EVEX K mask). +func (c *amd64Ctx) lowerDuplicateMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := string(op) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + switch baseOp { + case "MOVDDUP", "VMOVDDUP", "MOVSHDUP", "VMOVSHDUP", "MOVSLDUP", "VMOVSLDUP": + // handled below + default: + return false, false, nil + } + zeroMasking := false + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + for _, suffix := range strings.Split(rawOp[dot+1:], ".") { + if suffix != "Z" || zeroMasking { + return true, false, fmt.Errorf("amd64 %s has unsupported suffix .%s: %q", rawOp[:dot], suffix, ins.Raw) + } + zeroMasking = true + } + rawOp = baseOp + } + + elemBits := 0 + duplicateOdd := false + switch rawOp { + case "MOVDDUP", "VMOVDDUP": + elemBits = 64 + case "MOVSHDUP", "VMOVSHDUP": + elemBits = 32 + duplicateOdd = true + case "MOVSLDUP", "VMOVSLDUP": + elemBits = 32 + default: + panic("duplicate-move opcode precheck drift") + } + + vectorForm := strings.HasPrefix(rawOp, "V") + wantArgs := 2 + if vectorForm { + wantArgs = 3 + } + if len(ins.Args) != 2 && (!vectorForm || len(ins.Args) != wantArgs) { + return true, false, fmt.Errorf("amd64 %s expects src, [K mask,] vector dst: %q", rawOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if !vectorForm && (masked || zeroMasking) { + return true, false, fmt.Errorf("amd64 %s legacy form does not accept EVEX masking: %q", rawOp, ins.Raw) + } + if zeroMasking && !masked { + return true, false, fmt.Errorf("amd64 %s .Z form requires a K mask: %q", rawOp, ins.Raw) + } + + dstArg := ins.Args[len(ins.Args)-1] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects vector destination: %q", rawOp, ins.Raw) + } + byteWidth := 0 + var loadOperand func(Operand) (string, error) + var loadDst func() (string, error) + var storeDst func(string) error + switch { + case isAMD64XReg(dstArg.Reg): + byteWidth = 16 + loadOperand = c.loadXVecOperand + loadDst = func() (string, error) { return c.loadX(dstArg.Reg) } + storeDst = func(value string) error { return c.storeX(dstArg.Reg, value) } + case vectorForm && isAMD64YReg(dstArg.Reg): + byteWidth = 32 + loadOperand = c.loadYVecOperand + loadDst = func() (string, error) { return c.loadY(dstArg.Reg) } + storeDst = func(value string) error { return c.storeY(dstArg.Reg, value) } + case vectorForm && isAMD64ZReg(dstArg.Reg): + byteWidth = 64 + loadOperand = c.loadZVecOperand + loadDst = func() (string, error) { return c.loadZ(dstArg.Reg) } + storeDst = func(value string) error { return c.storeZ(dstArg.Reg, value) } + default: + return true, false, fmt.Errorf("amd64 %s has mismatched or non-vector destination: %q", rawOp, ins.Raw) + } + + lanes := byteWidth * 8 / elemBits + var sourceLanes string + // MOVDDUP's 128-bit memory form reads one scalar double and duplicates it. + // Wider memory forms read a full Y/Z vector and duplicate the low double of + // each 128-bit lane. A register source always reads the full vector. + if elemBits == 64 && byteWidth == 16 && ins.Args[0].Kind != OpReg { + scalar, err := c.evalF64(ins.Args[0]) + if err != nil { + return true, false, err + } + bits := c.newTmp() + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", bits, scalar) + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> poison, i64 %%%s, i32 0\n", seed, bits) + sourceLanes = "%" + seed + } else { + sourceBytes, err := loadOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", cast, byteWidth, sourceBytes, lanes, elemBits) + sourceLanes = "%" + cast + } + + indices := make([]int, lanes) + for lane := range indices { + if duplicateOdd { + indices[lane] = lane | 1 + } else { + indices[lane] = lane &^ 1 + } + } + shuffle := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> poison, <%d x i32> %s\n", + shuffle, lanes, elemBits, sourceLanes, lanes, elemBits, lanes, llvmI32Mask(indices)) + result := "%" + shuffle + + if masked { + maskArg := ins.Args[1] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K register: %q", rawOp, ins.Raw) + } + if maskIndex, ok := amd64ParseKReg(maskArg.Reg); !ok || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s masked form expects K register: %q", rawOp, ins.Raw) + } + mask, err := c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + oldBytes, err := loadDst() + if err != nil { + return true, false, err + } + old := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", old, byteWidth, oldBytes, lanes, elemBits) + result = amd64ApplyIntegerLaneMask(c, lanes, elemBits, result, "%"+old, mask, zeroMasking) + } + + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, elemBits, result, byteWidth) + return true, false, storeDst("%" + out) +} + +func llvmI32Mask(indices []int) string { + var b strings.Builder + b.WriteByte('<') + for i, index := range indices { + if i != 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "i32 %d", index) + } + b.WriteByte('>') + return b.String() +} diff --git a/amd64_lower_enqueue.go b/amd64_lower_enqueue.go new file mode 100644 index 000000000..468d203c3 --- /dev/null +++ b/amd64_lower_enqueue.go @@ -0,0 +1,78 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86EnqueueOps = map[string]byte{ + "ENQCMD": 0xf2, + "ENQCMDS": 0xf3, +} + +// Keep the 64-byte device enqueue operation in hardware. An ordinary memory +// copy cannot implement PASID handling, atomic submission, or retry status. +func (c *amd64Ctx) lowerEnqueue(op Op, ins Instr) (bool, bool, error) { + name := strings.ToUpper(string(op)) + base := strings.SplitN(name, ".", 2)[0] + if _, ok := x86EnqueueOps[base]; !ok { + return false, false, nil + } + if name != base || len(ins.Args) != 2 || !isAMD64MemoryOperand(ins.Args[0]) || + ins.Args[1].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[1].Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s expects a memory source and GP address register without suffixes: %q", c.goarch, base, ins.Raw) + } + if ins.Args[0].Kind == OpMem && !x86MemoryRegistersValidForArch(ins.Args[0].Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s source uses an invalid address register: %q", c.goarch, base, ins.Raw) + } + source := ins.Args[0] + assemblySource := "$1" + if source.Kind == OpMem && source.Mem.Segment != "" { + switch source.Mem.Segment { + case FS, GS: + // LLVM inline-asm memory operands do not preserve the pointer's + // segment address space. Spell the prefix explicitly and pass + // its offset, not an ordinary unsegmented effective address. + assemblySource = "%" + strings.ToLower(string(source.Mem.Segment)) + ":$1" + source.Mem.Segment = "" + default: + return true, false, fmt.Errorf("unsupported enqueue source segment %s", source.Mem.Segment) + } + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(source) + if err != nil { + return true, false, err + } + wordType, wordBits := I64, 64 + if c.goarch == "386" { + wordType, wordBits = I32, 32 + } + destination, err := c.evalIntSized(ins.Args[1], wordType) + if err != nil { + return true, false, err + } + if bits := ins.x86AddressBits; bits != 0 && bits != wordBits { + if bits != wordBits/2 { + return true, false, fmt.Errorf("invalid %s address size %d", c.goarch, bits) + } + // Truncate the effective offset before hardware applies any explicit + // source-segment override. + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint %s %s to i%d\n", address, ptrType, ptr, bits) + wrapped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i%d %%%s to %s\n", wrapped, bits, address, ptrType) + ptr = "%" + wrapped + low, extended := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to i%d\n", low, wordType, destination, bits) + fmt.Fprintf(c.b, " %%%s = zext i%d %%%s to %s\n", extended, bits, low, wordType) + destination = "%" + extended + } + + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i8 asm sideeffect %q, %q(%s elementtype([64 x i8]) %s, %s %s)\n", + status, strings.ToLower(base)+" "+assemblySource+", $2; sete $0", + "=q,*m,r,~{memory},~{dirflag},~{fpsr},~{flags}", ptrType, ptr, wordType, destination) + c.storeSegmentQueryZF("%" + status) + c.clearX86StatusFlags(x86StatusAll &^ x86StatusZF) + return true, false, nil +} diff --git a/amd64_lower_exchange.go b/amd64_lower_exchange.go new file mode 100644 index 000000000..64d7508fd --- /dev/null +++ b/amd64_lower_exchange.go @@ -0,0 +1,174 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ExchangeKind uint8 + +const ( + amd64ExchangeSwap amd64ExchangeKind = iota + amd64ExchangeAdd +) + +type amd64ExchangeSpec struct { + kind amd64ExchangeKind + bits int +} + +// The complete Go yrb_mb/yrl_ml (XADD) and yml_mb/yxchg (XCHG) families. +// Width selects register classes; kind selects operand symmetry and flags. +var amd64ExchangeSpecs = map[Op]amd64ExchangeSpec{ + "XADDB": {amd64ExchangeAdd, 8}, + "XADDW": {amd64ExchangeAdd, 16}, + "XADDL": {amd64ExchangeAdd, 32}, + "XADDQ": {amd64ExchangeAdd, 64}, + "XCHGB": {amd64ExchangeSwap, 8}, + "XCHGW": {amd64ExchangeSwap, 16}, + "XCHGL": {amd64ExchangeSwap, 32}, + "XCHGQ": {amd64ExchangeSwap, 64}, +} + +type amd64ExchangeForm struct { + register Operand + other Operand + nop bool +} + +func parseAMD64ExchangeForm(goarch string, op Op, spec amd64ExchangeSpec, ins Instr) (amd64ExchangeForm, error) { + invalid := func(reason string) (amd64ExchangeForm, error) { + return amd64ExchangeForm{}, fmt.Errorf("%s %s %s: %q", goarch, op, reason, ins.Raw) + } + if strings.Contains(string(op), ".") { + return invalid("does not accept instruction suffixes") + } + if registered, ok := amd64ExchangeSpecs[op]; !ok || registered != spec || + ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return invalid("does not have its registered source identity and width") + } + if goarch != "amd64" && goarch != "386" { + return invalid("requires a registered x86 mode") + } + if goarch == "386" && spec.bits == 64 { + return invalid("is illegal in 32-bit mode") + } + if len(ins.Args) != 2 { + return invalid("expects two operands") + } + first, second := ins.Args[0], ins.Args[1] + if spec.kind == amd64ExchangeSwap && isAMD64MemoryOperand(first) { + first, second = second, first + } + registerAllowed := func(reg Reg) bool { + if spec.bits == 8 { + return isGoYmbRegisterForArch(reg, goarch) + } + return isX86YrlRegisterForArch(reg, goarch) + } + if first.Kind != OpReg || !registerAllowed(first.Reg) { + return invalid("requires a source in Go's byte/general-register class") + } + if second.Kind == OpReg { + if !registerAllowed(second.Reg) { + return invalid("destination is outside Go's byte/general-register class") + } + } else if !isAMD64MemoryOperand(second) { + return invalid("expects a register or memory destination") + } else if second.Kind == OpMem && !x86MemoryRegistersValidForArch(second.Mem, goarch) { + return invalid("uses an out-of-range address register") + } + form := amd64ExchangeForm{register: first, other: second} + if spec.kind == amd64ExchangeSwap { + if goarch == "amd64" && spec.bits == 8 && x86CompareExchangeHasREXByteView(x86CompareExchangeForm{ + source: first, destination: second, + }) { + for _, operand := range []Operand{first, second} { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == CH || operand.Reg == DH || operand.Reg == BH) { + return amd64ExchangeForm{}, fmt.Errorf("%w: %s %s Go's REX encoding changes a high-byte spelling to an unbound low-byte physical view: %q", ErrProbeNeedsContext, goarch, op, ins.Raw) + } + } + } + // Go's accumulator/accumulator W/L/Q rows emit opcode 90 (with + // their width prefix), a NOP. In particular, EAX/EAX does not clear + // the upper RAX bytes as a genuine ModRM r32 exchange would. + form.nop = spec.bits != 8 && first.Kind == OpReg && second.Kind == OpReg && first.Reg == AX && second.Reg == AX + } + return form, nil +} + +func (c *amd64Ctx) lowerExchange(op Op, spec amd64ExchangeSpec, ins Instr) (bool, bool, error) { + form, err := parseAMD64ExchangeForm(c.goarch, op, spec, ins) + if err != nil { + return true, false, err + } + if form.nop { + return true, false, nil + } + typ := amd64IntegerTypeForBits(spec.bits) + value, err := c.evalIntSized(form.register, typ) + if err != nil { + return true, false, err + } + var old string + if form.other.Kind == OpReg { + old, err = c.evalIntSized(form.other, typ) + } else { + old, err = c.emitExchangeMemory(spec, form.other, value) + } + if err != nil { + return true, false, err + } + result := value + if spec.kind == amd64ExchangeAdd { + sum := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", sum, typ, old, value) + result = "%" + sum + } + // Read both operands before writeback. Source-first order preserves identical + // register XADD and overlapping AH/AL exchanges, including partial writes. + if err := c.storeRegSized(form.register.Reg, typ, old); err != nil { + return true, false, err + } + if form.other.Kind == OpReg { + if err := c.storeRegSized(form.other.Reg, typ, result); err != nil { + return true, false, err + } + } + if spec.kind == amd64ExchangeAdd { + c.setScalarAddSubFlags(typ, true, old, value, result) + } + return true, false, nil +} + +func (c *amd64Ctx) emitExchangeMemory(spec amd64ExchangeSpec, memory Operand, value string) (string, error) { + ptr, ptrType, err := c.compareExchangePointer(memory) + if err != nil { + return "", err + } + typ := amd64IntegerTypeForBits(spec.bits) + width := map[int]string{8: "b", 16: "w", 32: "l", 64: "q"}[spec.bits] + mnemonic := "xchg" + width // A memory XCHG is implicitly locked. + if spec.kind == amd64ExchangeAdd { + mnemonic = "lock; xadd" + width + } + registerClass := "r" + if spec.bits == 8 { + registerClass = "q" + } + segment := "" + if memory.Kind == OpMem && memory.Mem.Segment != "" { + // LLVM's inline-asm memory operand does not print the address space's + // FS/GS override; preserve it explicitly in the instruction template. + segment = "%" + strings.ToLower(string(memory.Mem.Segment)) + ":" + } + // Native x86 exchanges support unaligned memory. LLVM atomicrmw align 1 + // can introduce libatomic calls, while claiming natural alignment is false. + // Explicit read/write memory constraints and the tied register model the + // native operation without imposing an alignment precondition. Keep memory + // clobbering for synchronization and preserve segment-relative pointer types. + old := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect \"%s $0, %s$1\", \"=%s,=*m,0,*m,~{memory},~{dirflag},~{fpsr},~{flags}\"(%s elementtype(%s) %s, %s %s, %s elementtype(%s) %s)\n", + old, typ, mnemonic, segment, registerClass, ptrType, typ, ptr, typ, value, ptrType, typ, ptr) + return "%" + old, nil +} diff --git a/amd64_lower_expand.go b/amd64_lower_expand.go new file mode 100644 index 000000000..a48b274a3 --- /dev/null +++ b/amd64_lower_expand.go @@ -0,0 +1,149 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedExpandSpec struct { + laneBits int +} + +// amd64PackedExpandSpecs is the complete Go 1.27 _yvexpandpd grammar for the +// packed floating-point and integer expand families. None of these optabs has +// an EVEX broadcast attribute. +var amd64PackedExpandSpecs = map[Op]amd64PackedExpandSpec{ + "VEXPANDPD": {laneBits: 64}, + "VEXPANDPS": {laneBits: 32}, + "VPEXPANDB": {laneBits: 8}, + "VPEXPANDW": {laneBits: 16}, + "VPEXPANDD": {laneBits: 32}, + "VPEXPANDQ": {laneBits: 64}, +} + +// lowerPackedExpand implements all six _yvexpandpd rows for every instruction +// above: X/Y/Z register or memory sources and the corresponding K1-K7 merge or +// .Z forms. Masked memory uses llvm.masked.expandload so disabled lanes do not +// spuriously read beyond the compact source array. +func (c *amd64Ctx) lowerPackedExpand(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, ok := amd64PackedExpandSpecs[Op(baseOp)] + if !ok { + return false, false, nil + } + zeroing := false + switch suffix { + case "": + case "Z": + zeroing = true + default: + return true, false, fmt.Errorf("%s %s accepts only Go 1.27's optional .Z suffix: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects source, [K1-K7,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be an X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth == 0 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go's EVEX register range: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if !isAMD64MemoryOperand(source) && !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must be matching X/Y/Z register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if !masked { + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, sourceBytes) + } + + maskArg := ins.Args[1] + maskIndex, valid := amd64ParseKReg(maskArg.Reg) + if maskArg.Kind != OpReg || !valid || maskIndex == 0 { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + mask, err := c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + maskVector := amd64IntegerMaskVector(c, lanes, mask) + passthrough := "zeroinitializer" + if !zeroing { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + passthrough = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + } + + result := "" + if isAMD64MemoryOperand(source) { + pointer, pointerType, intrinsicSuffix, err := c.packedVectorMemoryPointer(source) + if err != nil { + return true, false, err + } + expanded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.masked.expandload.v%di%d%s(%s %s, <%d x i1> %s, %s %s)\n", + expanded, vectorType, lanes, spec.laneBits, intrinsicSuffix, pointerType, pointer, lanes, maskVector, vectorType, passthrough) + result = "%" + expanded + } else { + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + sourceLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, sourceBytes) + result = c.expandPackedRegisterLanes(lanes, spec.laneBits, sourceLanes, passthrough, mask, maskVector) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, vectorType, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +// expandPackedRegisterLanes is the register-source inverse of vector +// compression. For destination lane i, the compact source index is the number +// of set mask bits below i. Building each lane from the original source before +// the final store keeps source/destination aliasing safe. +func (c *amd64Ctx) expandPackedRegisterLanes(lanes, laneBits int, source, passthrough, mask, maskVector string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + result := passthrough + for lane := 0; lane < lanes; lane++ { + active := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i1> %s, i32 %d\n", active, lanes, maskVector, lane) + sourceIndex := "0" + if lane != 0 { + below := c.newTmp() + count := c.newTmp() + index := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", below, mask, (uint64(1)< +#include +extern void packedexpandsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static int check(const uint8_t *got, const uint8_t *source, const uint8_t *old, + int lane_bytes, uint64_t mask, int zeroing) { + int lanes = 64 / lane_bytes, packed = 0; + for (int lane = 0; lane < lanes; lane++) { + const uint8_t *want = zeroing ? (const uint8_t[8]){0} : old + lane*lane_bytes; + if ((mask >> lane) & 1) want = source + packed++*lane_bytes; + if (memcmp(got + lane*lane_bytes, want, lane_bytes)) return 10 + lane; + } + return 0; +} +int main(void) { + uint8_t source[64], old[64], out[513]; + const uint64_t mask = UINT64_C(0xa55aa55aa55aa55a); + for (int i = 0; i < 64; i++) { source[i] = (uint8_t)(i*37 + 9); old[i] = (uint8_t)(0xe0 - i); } + memset(out, 0xcc, sizeof(out)); + packedexpandsemantics(out, source, old, mask); + int r; + if ((r = check(out+0, source, old, 1, mask, 0))) return 100+r; + if ((r = check(out+64, source, old, 2, mask, 1))) return 200+r; + if ((r = check(out+128, source, old, 4, mask, 0))) return 300+r; + if ((r = check(out+192, source, old, 8, mask, 1))) return 400+r; + if ((r = check(out+256, source, old, 4, mask, 0))) return 500+r; + if ((r = check(out+320, source, old, 8, mask, 1))) return 600+r; + if ((r = check(out+384, source, source, 4, mask, 0))) return 700+r; + if (memcmp(out+448, old, 64)) return 800; + if (out[512] != 1) return 801; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_expand_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_explicit_mask_move.go b/amd64_lower_explicit_mask_move.go new file mode 100644 index 000000000..e8628ef54 --- /dev/null +++ b/amd64_lower_explicit_mask_move.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ExplicitMaskMoveSpec struct { + laneBits int +} + +// amd64ExplicitMaskMoveSpecs is Go 1.27's complete _yvmaskmovpd grammar. +// Floating and integer spellings have identical bitwise lane-mask semantics; +// only the lane width differs. +var amd64ExplicitMaskMoveSpecs = map[Op]amd64ExplicitMaskMoveSpec{ + "VMASKMOVPS": {laneBits: 32}, + "VMASKMOVPD": {laneBits: 64}, + "VPMASKMOVD": {laneBits: 32}, + "VPMASKMOVQ": {laneBits: 64}, +} + +func (c *amd64Ctx) lowerExplicitMaskMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64ExplicitMaskMoveSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed forms in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects memory/data, mask, destination/memory: %q", c.goarch, baseOp, ins.Raw) + } + load := isAMD64MemoryOperand(ins.Args[0]) + store := isAMD64MemoryOperand(ins.Args[2]) + if load == store { + return true, false, fmt.Errorf("%s %s requires memory only as its first (load) or last (store) operand: %q", c.goarch, baseOp, ins.Raw) + } + maskArg := ins.Args[1] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("%s %s mask must be an X or Y register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(maskArg.Reg) + if byteWidth != 16 && byteWidth != 32 || !amd64VEXVectorRegister(maskArg, byteWidth) { + return true, false, fmt.Errorf("%s %s mask must be an in-range X or Y register: %q", c.goarch, baseOp, ins.Raw) + } + registerArg := ins.Args[2] + memoryArg := ins.Args[0] + if store { + registerArg = ins.Args[0] + memoryArg = ins.Args[2] + } + if !amd64VEXVectorRegister(registerArg, byteWidth) { + return true, false, fmt.Errorf("%s %s data register must match the mask width: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth * 8 / spec.laneBits + mask, err := c.loadPackedCompareLanes(maskArg, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + predicate := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i%d> %s, zeroinitializer\n", predicate, lanes, spec.laneBits, mask) + pointer, pointerType, intrinsicSuffix, err := c.packedVectorMemoryPointer(memoryArg) + if err != nil { + return true, false, err + } + intrinsicType := fmt.Sprintf("v%di%d", lanes, spec.laneBits) + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + if load { + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.masked.load.%s%s(%s align 1 %s, <%d x i1> %%%s, %s zeroinitializer)\n", + loaded, vectorType, intrinsicType, intrinsicSuffix, pointerType, pointer, lanes, predicate, vectorType) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytesValue, vectorType, loaded, byteWidth) + return true, false, c.storeVectorBytes(registerArg.Reg, byteWidth, "%"+bytesValue) + } + data, err := c.loadPackedCompareLanes(registerArg, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void @llvm.masked.store.%s%s(%s %s, %s align 1 %s, <%d x i1> %%%s)\n", + intrinsicType, intrinsicSuffix, vectorType, data, pointerType, pointer, lanes, predicate) + return true, false, nil +} diff --git a/amd64_lower_explicit_mask_move_test.go b/amd64_lower_explicit_mask_move_test.go new file mode 100644 index 000000000..6258fc2e4 --- /dev/null +++ b/amd64_lower_explicit_mask_move_test.go @@ -0,0 +1,209 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64ExplicitMaskMoveGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64ExplicitMaskMoveSpec{ + "VMASKMOVPS": {laneBits: 32}, + "VMASKMOVPD": {laneBits: 64}, + "VPMASKMOVD": {laneBits: 32}, + "VPMASKMOVQ": {laneBits: 64}, + } + if len(amd64ExplicitMaskMoveSpecs) != len(expected) { + t.Fatalf("explicit-mask-move grammar has %d entries, want %d", len(amd64ExplicitMaskMoveSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64ExplicitMaskMoveSpecs[op]; !ok { + t.Errorf("explicit-mask-move grammar omitted %s", op) + } else if got != want { + t.Errorf("explicit-mask-move grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86ExplicitMaskMoveCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT explicitmaskmoveforms(SB),$0-0\n") + for _, op := range []string{"VMASKMOVPS", "VMASKMOVPD", "VPMASKMOVD", "VPMASKMOVQ"} { + for _, width := range []string{"X", "Y"} { + last := 15 + fmt.Fprintf(&source, "\t%s 8(AX), %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s %s%d, %s1, 16(AX)\n", op, width, last, width) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"explicitmaskmoveforms": {Name: "explicitmaskmoveforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.masked.load.v4i32", "@llvm.masked.store.v8i32", "@llvm.masked.load.v2i64", "@llvm.masked.store.v4i64"} { + if !strings.Contains(ir, want) { + t.Errorf("IR omitted fault-suppressing intrinsic %s", want) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "explicit-mask-move-"+target.name+".ll", "explicit-mask-move-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ExplicitMaskMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPMASKMOVD (AX), X0"}, + {goarch: "amd64", instruction: "VPMASKMOVD X0, X1, X2"}, + {goarch: "amd64", instruction: "VPMASKMOVD (AX), X0, (BX)"}, + {goarch: "amd64", instruction: "VPMASKMOVD X0, (AX), X1"}, + {goarch: "amd64", instruction: "VPMASKMOVD (AX), X0, Y1"}, + {goarch: "amd64", instruction: "VPMASKMOVD (AX), Z0, Z1"}, + {goarch: "amd64", instruction: "VPMASKMOVD.Z (AX), X0, X1"}, + {goarch: "amd64", instruction: "VMASKMOVPD (AX), X0, AX"}, + {goarch: "386", instruction: "VPMASKMOVQ (AX), X16, X0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's explicit-mask-move table", test.instruction) + } + }) + } +} + +func TestAMD64ExplicitMaskMoveRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT explicitmaskmovesemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ loadmem+8(FP), BX + MOVQ data+16(FP), CX + MOVQ mask+24(FP), DX + MOVQ storemem+32(FP), R8 + VMOVDQU 0(CX), Y0 + VMOVDQU 0(DX), Y1 + STC + VPMASKMOVD 0(BX), Y1, Y2 + VMOVDQU Y2, 0(AX) + VPMASKMOVD Y0, Y1, 0(R8) + VPMASKMOVQ 0(BX), Y1, Y3 + VMOVDQU Y3, 32(AX) + VPMASKMOVQ Y0, Y1, 32(R8) + VMASKMOVPS 0(BX), Y1, Y4 + VMOVDQU Y4, 64(AX) + VMASKMOVPS Y0, Y1, 64(R8) + VMASKMOVPD 0(BX), Y1, Y5 + VMOVDQU Y5, 96(AX) + VMASKMOVPD Y0, Y1, 96(R8) + SETCS 128(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"explicitmaskmovesemantics": { + Name: "explicitmaskmovesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void explicitmaskmovesemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint8_t *); +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint32_t load32[8], data32[8], mask32[8]; + uint8_t out[129]={0}, store[128]; + for (int i=0;i<8;i++) { + load32[i]=UINT32_C(0x10203040)+(uint32_t)i; + data32[i]=UINT32_C(0xa0b0c000)+(uint32_t)i; + mask32[i]=(i==1||i==2||i==5)?UINT32_C(0x80000000):UINT32_C(0x7fffffff); + } + memset(store,0x5a,sizeof(store)); + explicitmaskmovesemantics(out,(uint8_t*)load32,(uint8_t*)data32,(uint8_t*)mask32,store); + for (int i=0;i<8;i++) { + int enabled=(int32_t)mask32[i]<0; + if (u32(out+4*i)!=(enabled?load32[i]:0)) return 10; + if (u32(store+4*i)!=(enabled?data32[i]:UINT32_C(0x5a5a5a5a))) return 11; + if (u32(out+64+4*i)!=(enabled?load32[i]:0)) return 12; + if (u32(store+64+4*i)!=(enabled?data32[i]:UINT32_C(0x5a5a5a5a))) return 13; + } + for (int i=0;i<4;i++) { + int enabled=(int64_t)u64((uint8_t*)mask32+8*i)<0; + if (u64(out+32+8*i)!=(enabled?u64((uint8_t*)load32+8*i):0)) return 20; + if (u64(store+32+8*i)!=(enabled?u64((uint8_t*)data32+8*i):UINT64_C(0x5a5a5a5a5a5a5a5a))) return 21; + if (u64(out+96+8*i)!=(enabled?u64((uint8_t*)load32+8*i):0)) return 22; + if (u64(store+96+8*i)!=(enabled?u64((uint8_t*)data32+8*i):UINT64_C(0x5a5a5a5a5a5a5a5a))) return 23; + } + if (out[128]!=1) return 30; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "explicit_mask_move_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_extract.go b/amd64_lower_extract.go new file mode 100644 index 000000000..e48dae899 --- /dev/null +++ b/amd64_lower_extract.go @@ -0,0 +1,45 @@ +package plan9asm + +import "fmt" + +// lowerVectorLaneExtract implements all Go 1.27 forms in _yvextractf128, +// _yvextractf32x4, and _yvextractf32x8 for their F/I lane-extract mnemonics. +func (c *amd64Ctx) lowerVectorLaneExtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerInsertExtractFamily(op, ins) +} + +func (c *amd64Ctx) storeVectorBytesOperand(dst Operand, byteWidth int, value string) error { + return c.storeVectorBytesOperandWithMetadata(dst, byteWidth, value, "") +} + +func (c *amd64Ctx) storeVectorBytesOperandWithMetadata(dst Operand, byteWidth int, value, metadata string) error { + if dst.Kind == OpReg { + if metadata != "" { + return fmt.Errorf("vector register destination cannot carry store metadata: %s", dst.String()) + } + if !amd64VectorRegisterHasWidth(dst, byteWidth) { + return fmt.Errorf("expected %d-byte vector register, got %s", byteWidth, dst.String()) + } + return c.storeVectorBytes(dst.Reg, byteWidth, value) + } + switch dst.Kind { + case OpMem: + addr, err := c.addrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <%d x i8> %s, ptr %s, align 1%s\n", byteWidth, value, c.ptrFromAddrI64(addr), metadata) + return nil + case OpSym: + p, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <%d x i8> %s, ptr %s, align 1%s\n", byteWidth, value, p, metadata) + return nil + case OpFP: + return c.storeFPVectorBytes(dst.FPOffset, byteWidth, value, metadata) + default: + return fmt.Errorf("expected vector register or memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_extract_scalar.go b/amd64_lower_extract_scalar.go new file mode 100644 index 000000000..57091337d --- /dev/null +++ b/amd64_lower_extract_scalar.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedScalarExtractSpec struct { + bits int + vector bool + maxImmediate int64 + allowMMX bool +} + +var amd64PackedScalarExtractSpecs = map[Op]amd64PackedScalarExtractSpec{ + "EXTRACTPS": {bits: 32, maxImmediate: 3}, + "VEXTRACTPS": {bits: 32, vector: true, maxImmediate: 255}, + "PEXTRB": {bits: 8, maxImmediate: 255, allowMMX: true}, + "PEXTRW": {bits: 16, maxImmediate: 255}, + "PEXTRD": {bits: 32, maxImmediate: 255, allowMMX: true}, + "PEXTRQ": {bits: 64, maxImmediate: 255, allowMMX: true}, + "VPEXTRB": {bits: 8, vector: true, maxImmediate: 255}, + "VPEXTRW": {bits: 16, vector: true, maxImmediate: 255}, + "VPEXTRD": {bits: 32, vector: true, maxImmediate: 255}, + "VPEXTRQ": {bits: 64, vector: true, maxImmediate: 255}, +} + +func (c *amd64Ctx) lowerPackedScalarExtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64PackedScalarExtractSpecs[Op(base)]; !known { + return false, false, nil + } + var f x86ScalarLaneForm + if base == "EXTRACTPS" || base == "VEXTRACTPS" { + var ps x86InsertExtractForm + ins, ps, _, err = c.bindInsertExtractForm(op, ins) + f = x86ScalarLaneForm{bits: 32, gpBits: 32, source: ps.source, destination: ps.destination, offset: ps.offset, memory: ps.memory} + if len(ins.Args) == 3 && ins.Args[2].Kind == OpReg { + f.transferReg = ins.Args[2].Reg + } + } else { + ins, f, _, err = c.bindScalarLaneForm(op, ins) + } + if err != nil { + return true, false, err + } + source, err := c.loadX(ins.Args[f.source].Reg) + if err != nil { + return true, false, err + } + lanes := 128 / f.bits + laneVector, extracted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x i%d>\n", laneVector, source, lanes, f.bits) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %%%s, i32 %d\n", extracted, lanes, f.bits, laneVector, f.offset/(f.bits/8)) + value := "%" + extracted + destination := ins.Args[f.destination] + if destination.Kind == OpReg { + destination.Reg = f.transferReg + if f.gpBits > f.bits { + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i%d\n", widened, f.bits, value, f.gpBits) + value = "%" + widened + } else if f.gpBits < f.bits { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %s to i%d\n", truncated, f.bits, value, f.gpBits) + value = "%" + truncated + } + return true, false, c.storeRegSized(destination.Reg, amd64IntegerTypeForBits(f.gpBits), value) + } + if (base == "EXTRACTPS" || base == "VEXTRACTPS") && destination.Kind == OpFP { + // Retain the old complete PS field-writer contract. The new integer + // lane observer does not retroactively grant or remove that contract. + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %s to <4 x i8>\n", bytes, value) + return true, false, c.storeFPVectorBytes(destination.FPOffset, 4, "%"+bytes, "") + } + return true, false, c.storePackedScalarExtractMemory(destination, amd64IntegerTypeForBits(f.bits), value) +} + +func amd64MMXEncodingGP(index int) Reg { + return [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI}[index] +} + +func (c *amd64Ctx) storePackedScalarExtractMemory(destination Operand, typ LLVMType, value string) error { + bits := map[LLVMType]int{I8: 8, I16: 16, I32: 32, I64: 64}[typ] + if destination.Kind == OpFP { + return c.storeScalarLaneFPBits(destination.FPOffset, bits, value) + } + return c.storeVMOVIntegerScalar(destination, bits, value) +} diff --git a/amd64_lower_far_pointer.go b/amd64_lower_far_pointer.go new file mode 100644 index 000000000..95b71a520 --- /dev/null +++ b/amd64_lower_far_pointer.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FarPointerSpec struct { + segment Reg + bits int +} + +// amd64FarPointerSpecs is the complete Go 1.27 LFS/LGS/LSS grammar. Segment +// selection and offset width are data; memory shape, destination class, and +// architecture availability are parsed once for all nine opcodes. +var amd64FarPointerSpecs = map[Op]amd64FarPointerSpec{ + "LFSW": {segment: FS, bits: 16}, + "LFSL": {segment: FS, bits: 32}, + "LFSQ": {segment: FS, bits: 64}, + "LGSW": {segment: GS, bits: 16}, + "LGSL": {segment: GS, bits: 32}, + "LGSQ": {segment: GS, bits: 64}, + "LSSW": {segment: SS, bits: 16}, + "LSSL": {segment: SS, bits: 32}, + "LSSQ": {segment: SS, bits: 64}, +} + +type amd64FarPointerForm struct { + memory Operand + destination Reg +} + +func (c *amd64Ctx) lowerFarPointer(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64FarPointerSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseFarPointerForm(baseOp, ins) + if err != nil { + return true, false, err + } + return c.emitFarPointer(spec, form) +} + +func (c *amd64Ctx) parseFarPointerForm(op string, ins Instr) (amd64FarPointerForm, error) { + var form amd64FarPointerForm + if len(ins.Args) != 2 { + return form, fmt.Errorf("%s %s expects far-pointer memory and GP destination: %q", c.goarch, op, ins.Raw) + } + memory, destination := ins.Args[0], ins.Args[1] + if !isAMD64MemoryOperand(memory) { + return form, fmt.Errorf("%s %s source must be Go 1.27 Ym memory: %q", c.goarch, op, ins.Raw) + } + if memory.Kind == OpMem && !x86MemoryRegistersValidForArch(memory.Mem, c.goarch) { + return form, fmt.Errorf("%s %s source address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return form, fmt.Errorf("%s %s destination must be a Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + return amd64FarPointerForm{memory: memory, destination: destination.Reg}, nil +} + +func (c *amd64Ctx) emitFarPointer(spec amd64FarPointerSpec, form amd64FarPointerForm) (bool, bool, error) { + pointer, pointerType, err := c.x86DescriptorMemoryPointer(form.memory) + if err != nil { + return true, false, err + } + typ := amd64IntegerTypeForBits(spec.bits) + memoryBytes := spec.bits/8 + 2 + width := map[int]string{16: "w", 32: "l", 64: "q"}[spec.bits] + result := c.newTmp() + assembly := "l" + strings.ToLower(string(spec.segment)) + width + " $1, $0" + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + result, typ, assembly, "=r,*m,~{memory},~{dirflag},~{fpsr},~{flags}", pointerType, memoryBytes, pointer) + if err := c.storeRegSized(form.destination, typ, "%"+result); err != nil { + return true, false, err + } + return true, false, nil +} diff --git a/amd64_lower_far_pointer_test.go b/amd64_lower_far_pointer_test.go new file mode 100644 index 000000000..45e8b3f64 --- /dev/null +++ b/amd64_lower_far_pointer_test.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "reflect" + "sort" + "strings" + "testing" +) + +func TestAMD64FarPointerGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64FarPointerSpecs)) + for op := range amd64FarPointerSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{"LFSL", "LFSQ", "LFSW", "LGSL", "LGSQ", "LGSW", "LSSL", "LSSQ", "LSSW"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("far-pointer grammar opcodes = %v, want %v", got, want) + } +} + +func TestTranslateX86FarPointerCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + destination := "DI" + base := "BX" + if target.goarch == "amd64" { + destination = "R12" + base = "R13" + } + source := "TEXT farpointerforms(SB),$0-0\n" + for _, family := range []string{"LFS", "LGS", "LSS"} { + source += "\t" + family + "W 0(BX), AX\n" + source += "\t" + family + "W 2(" + base + "), " + destination + "\n" + source += "\t" + family + "L 4(BX), DX\n" + if target.goarch == "amd64" { + source += "\t" + family + "Q 8(" + base + "), " + destination + "\n" + } else { + source += "\t" + family + "L 8(BX), SP\n" + } + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"farpointerforms": {Name: "farpointerforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "far-pointer-"+target.name+".ll", "far-pointer-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FarPointerRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LFSW AX, DX"}, + {goarch: "amd64", instruction: "LFSL (BX), (CX)"}, + {goarch: "amd64", instruction: "LGSQ (BX), X0"}, + {goarch: "amd64", instruction: "LSSW.Z (BX), DX"}, + {goarch: "386", instruction: "LFSQ (BX), AX"}, + {goarch: "386", instruction: "LGSL (R11), AX"}, + {goarch: "386", instruction: "LSSL (BX), R11"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's far-pointer tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_far_return.go b/amd64_lower_far_return.go new file mode 100644 index 000000000..205ebc36c --- /dev/null +++ b/amd64_lower_far_return.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FarReturnKind uint8 + +const ( + amd64InterruptReturn amd64FarReturnKind = iota + amd64FarReturn +) + +type amd64FarReturnSpec struct { + kind amd64FarReturnKind + bits int + allowImmediate bool +} + +// amd64FarReturnSpecs is the complete Go 1.27 interrupt/far-return grammar. +// Every RETF width accepts both the operand-free CB encoding and Go's Yi32 +// immediate row, whose value is truncated to the architectural imm16. Q-width +// rows are absent from the 386 encoder even though W/L rows are shared. +var amd64FarReturnSpecs = map[Op]amd64FarReturnSpec{ + "IRETW": {kind: amd64InterruptReturn, bits: 16}, + "IRETL": {kind: amd64InterruptReturn, bits: 32}, + "IRETQ": {kind: amd64InterruptReturn, bits: 64}, + "RETFW": {kind: amd64FarReturn, bits: 16, allowImmediate: true}, + "RETFL": {kind: amd64FarReturn, bits: 32, allowImmediate: true}, + "RETFQ": {kind: amd64FarReturn, bits: 64, allowImmediate: true}, +} + +type amd64FarReturnForm struct { + hasImmediate bool + immediate uint16 +} + +func (c *amd64Ctx) lowerFarReturn(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64FarReturnSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseFarReturnForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", + spec.encoding(form), "~{memory},~{dirflag},~{fpsr},~{flags}") + c.b.WriteString(" unreachable\n") + return true, true, nil +} + +func (c *amd64Ctx) parseFarReturnForm(op string, spec amd64FarReturnSpec, ins Instr) (amd64FarReturnForm, error) { + var form amd64FarReturnForm + if len(ins.Args) == 0 { + return form, nil + } + if !spec.allowImmediate || len(ins.Args) != 1 || ins.Args[0].Kind != OpImm || + ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" { + return form, fmt.Errorf("%s %s expects no operands or one Go 1.27 Yi32 immediate: %q", c.goarch, op, ins.Raw) + } + value := ins.Args[0].Imm + if c.goarch != "386" && (value < -2147483648 || value > 4294967295) { + return form, fmt.Errorf("amd64 %s immediate is outside Go 1.27's Yi32 class: %q", op, ins.Raw) + } + form.hasImmediate = true + form.immediate = uint16(value) + return form, nil +} + +func (spec amd64FarReturnSpec) encoding(form amd64FarReturnForm) string { + bytes := make([]byte, 0, 4) + switch spec.bits { + case 16: + bytes = append(bytes, 0x66) + case 32: + case 64: + bytes = append(bytes, 0x48) + default: + panic("unknown amd64 far-return width") + } + if spec.kind == amd64InterruptReturn { + bytes = append(bytes, 0xcf) + } else if form.hasImmediate { + bytes = append(bytes, 0xca, byte(form.immediate), byte(form.immediate>>8)) + } else { + bytes = append(bytes, 0xcb) + } + parts := make([]string, len(bytes)) + for index, value := range bytes { + parts[index] = fmt.Sprintf("0x%02x", value) + } + return ".byte " + strings.Join(parts, ", ") +} diff --git a/amd64_lower_far_return_test.go b/amd64_lower_far_return_test.go new file mode 100644 index 000000000..6a8033f67 --- /dev/null +++ b/amd64_lower_far_return_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestAMD64FarReturnGrammarIsComplete(t *testing.T) { + want := map[Op]amd64FarReturnSpec{ + "IRETW": {kind: amd64InterruptReturn, bits: 16}, + "IRETL": {kind: amd64InterruptReturn, bits: 32}, + "IRETQ": {kind: amd64InterruptReturn, bits: 64}, + "RETFW": {kind: amd64FarReturn, bits: 16, allowImmediate: true}, + "RETFL": {kind: amd64FarReturn, bits: 32, allowImmediate: true}, + "RETFQ": {kind: amd64FarReturn, bits: 64, allowImmediate: true}, + } + if !reflect.DeepEqual(amd64FarReturnSpecs, want) { + t.Fatalf("far-return grammar = %+v, want %+v", amd64FarReturnSpecs, want) + } +} + +func TestAMD64FarReturnSplitsFollowingInstructions(t *testing.T) { + source := "TEXT splitfarreturn(SB),$0-0\n\tIRETL\n\tMOVL $1, AX\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"splitfarreturn": {Name: "splitfarreturn", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, amd64LLVMBlockName("anon_1")+":") || !strings.Contains(ir, "zext i32 1 to i64") { + t.Fatalf("instruction after IRETL was not retained in its own block:\n%s", ir) + } +} + +func TestTranslateX86FarReturnCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + forms := []struct { + op string + arg string + }{ + {op: "IRETW"}, + {op: "IRETL"}, + {op: "RETFW"}, + {op: "RETFW", arg: "$4"}, + {op: "RETFL"}, + {op: "RETFL", arg: "$4294967295"}, + } + if target.goarch == "386" { + forms = append(forms, struct { + op string + arg string + }{op: "RETFL", arg: "$4294967296"}) + } else { + forms = append(forms, + struct { + op string + arg string + }{op: "IRETQ"}, + struct { + op string + arg string + }{op: "RETFQ"}, + struct { + op string + arg string + }{op: "RETFQ", arg: "$4294967295"}, + ) + } + var source strings.Builder + sigs := make(map[string]FuncSig, len(forms)) + for index, form := range forms { + name := fmt.Sprintf("farreturnform%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n\t%s", name, form.op) + if form.arg != "" { + fmt.Fprintf(&source, " %s", form.arg) + } + source.WriteByte('\n') + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `.byte 0x66, 0xcf`, + `.byte 0xcf`, + `.byte 0x66, 0xcb`, + `.byte 0x66, 0xca, 0x04, 0x00`, + `.byte 0xcb`, + `.byte 0xca, 0xff, 0xff`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if target.goarch == "386" { + if !strings.Contains(ir, `.byte 0xca, 0x00, 0x00`) { + t.Errorf("386 IR is missing truncated Yi32 compatibility encoding:\n%s", ir) + } + } else { + for _, want := range []string{`.byte 0x48, 0xcf`, `.byte 0x48, 0xcb`, `.byte 0x48, 0xca, 0xff, 0xff`} { + if !strings.Contains(ir, want) { + t.Errorf("amd64 IR is missing %s:\n%s", want, ir) + } + } + } + if got := strings.Count(ir, "unreachable"); got != len(forms) { + t.Errorf("unreachable count = %d, want %d", got, len(forms)) + } + compileLLVMToObject(t, llc, target.triple, "far-return-"+target.name+".ll", "far-return-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FarReturnRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "IRETW AX"}, + {goarch: "amd64", instruction: "IRETL.P"}, + {goarch: "amd64", instruction: "IRETQ $4"}, + {goarch: "amd64", instruction: "RETFW AX"}, + {goarch: "amd64", instruction: "RETFL $-2147483649"}, + {goarch: "amd64", instruction: "RETFQ $4294967296"}, + {goarch: "amd64", instruction: "RETFL $1, $2"}, + {goarch: "386", instruction: "IRETQ"}, + {goarch: "386", instruction: "RETFQ"}, + {goarch: "386", instruction: "RETFQ $4"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's far-return tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_fixed_system.go b/amd64_lower_fixed_system.go new file mode 100644 index 000000000..afa579ef7 --- /dev/null +++ b/amd64_lower_fixed_system.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FixedSystemEffect uint8 + +const ( + amd64FixedSystemContinue amd64FixedSystemEffect = iota + amd64FixedSystemTrap + amd64FixedSystemControlTransfer +) + +type amd64FixedSystemSpec struct { + encoding string + effect amd64FixedSystemEffect + rawOnly bool +} + +// amd64FixedSystemSpecs is the complete Go 1.27 operand-free system/state +// family that has no modeled GP-register data flow, plus raw-only ENDBR32. +// Exact bytes keep Go's +// accepted 386 compatibility rows (notably ENDBR64 and SWAPGS) object-level +// compatible even when LLVM's mnemonic matcher applies architectural mode +// restrictions. Effects remain typed so traps and non-returning RSM cannot +// accidentally acquire ordinary fallthrough. +var amd64FixedSystemSpecs = map[Op]amd64FixedSystemSpec{ + "CLAC": {encoding: ".byte 0x0f, 0x01, 0xca"}, + "CLI": {encoding: ".byte 0xfa"}, + "CLTS": {encoding: ".byte 0x0f, 0x06"}, + "ENDBR32": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfb", rawOnly: true}, + "ENDBR64": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfa"}, + "ICEBP": {encoding: ".byte 0xf1", effect: amd64FixedSystemTrap}, + "INVD": {encoding: ".byte 0x0f, 0x08"}, + "RSM": {encoding: ".byte 0x0f, 0xaa", effect: amd64FixedSystemControlTransfer}, + "STAC": {encoding: ".byte 0x0f, 0x01, 0xcb"}, + "STI": {encoding: ".byte 0xfb"}, + "SWAPGS": {encoding: ".byte 0x0f, 0x01, 0xf8"}, + "UD1": {encoding: ".byte 0x0f, 0xb9, 0x00", effect: amd64FixedSystemTrap}, + "WBINVD": {encoding: ".byte 0x0f, 0x09"}, +} + +func (c *amd64Ctx) lowerFixedSystem(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64FixedSystemSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if spec.rawOnly && !ins.x86Encoded { + return true, false, fmt.Errorf("%s %s has no Go assembler mnemonic; use its raw encoding: %q", c.goarch, baseOp, ins.Raw) + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", + spec.encoding, "~{memory},~{dirflag},~{fpsr},~{flags}") + if spec.effect == amd64FixedSystemContinue { + return true, false, nil + } + c.b.WriteString(" unreachable\n") + return true, true, nil +} diff --git a/amd64_lower_fixed_system_test.go b/amd64_lower_fixed_system_test.go new file mode 100644 index 000000000..0598f9072 --- /dev/null +++ b/amd64_lower_fixed_system_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64FixedSystemGrammarIsComplete(t *testing.T) { + want := map[Op]amd64FixedSystemSpec{ + "CLAC": {encoding: ".byte 0x0f, 0x01, 0xca"}, + "CLI": {encoding: ".byte 0xfa"}, + "CLTS": {encoding: ".byte 0x0f, 0x06"}, + "ENDBR32": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfb", rawOnly: true}, + "ENDBR64": {encoding: ".byte 0xf3, 0x0f, 0x1e, 0xfa"}, + "ICEBP": {encoding: ".byte 0xf1", effect: amd64FixedSystemTrap}, + "INVD": {encoding: ".byte 0x0f, 0x08"}, + "RSM": {encoding: ".byte 0x0f, 0xaa", effect: amd64FixedSystemControlTransfer}, + "STAC": {encoding: ".byte 0x0f, 0x01, 0xcb"}, + "STI": {encoding: ".byte 0xfb"}, + "SWAPGS": {encoding: ".byte 0x0f, 0x01, 0xf8"}, + "UD1": {encoding: ".byte 0x0f, 0xb9, 0x00", effect: amd64FixedSystemTrap}, + "WBINVD": {encoding: ".byte 0x0f, 0x09"}, + } + if !reflect.DeepEqual(amd64FixedSystemSpecs, want) { + t.Fatalf("fixed-system grammar = %+v, want %+v", amd64FixedSystemSpecs, want) + } +} + +func TestTranslateX86FixedSystemCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT fixedsystemforms(SB),$0-0 + CLAC + CLI + CLTS + ENDBR64 + INVD + STAC + STI + SWAPGS + WBINVD + RET +TEXT fixedsystemicebp(SB),$0-0 + ICEBP +TEXT fixedsystemrsm(SB),$0-0 + RSM +TEXT fixedsystemud1(SB),$0-0 + UD1 +` + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "fixedsystemforms": {Name: "fixedsystemforms", Ret: Void}, + "fixedsystemicebp": {Name: "fixedsystemicebp", Ret: Void}, + "fixedsystemrsm": {Name: "fixedsystemrsm", Ret: Void}, + "fixedsystemud1": {Name: "fixedsystemud1", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `.byte 0x0f, 0x01, 0xca`, + `.byte 0xfa`, + `.byte 0x0f, 0x06`, + `.byte 0xf3, 0x0f, 0x1e, 0xfa`, + `.byte 0x0f, 0x08`, + `.byte 0x0f, 0x01, 0xcb`, + `.byte 0xfb`, + `.byte 0x0f, 0x01, 0xf8`, + `.byte 0x0f, 0x09`, + `.byte 0xf1`, + `.byte 0x0f, 0xaa`, + `.byte 0x0f, 0xb9, 0x00`, + `unreachable`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if got := strings.Count(ir, "unreachable"); got != 3 { + t.Errorf("unreachable count = %d, want 3", got) + } + compileLLVMToObject(t, llc, target.triple, "fixed-system-"+target.name+".ll", "fixed-system-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FixedSystemRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "CLAC AX"}, + {goarch: "amd64", instruction: "CLI.P"}, + {goarch: "amd64", instruction: "CLTS AX"}, + {goarch: "amd64", instruction: "ENDBR64 AX"}, + {goarch: "amd64", instruction: "ICEBP AX"}, + {goarch: "amd64", instruction: "INVD AX"}, + {goarch: "amd64", instruction: "RSM AX"}, + {goarch: "amd64", instruction: "STAC AX"}, + {goarch: "amd64", instruction: "STI AX"}, + {goarch: "amd64", instruction: "SWAPGS AX"}, + {goarch: "amd64", instruction: "UD1 AX"}, + {goarch: "amd64", instruction: "WBINVD AX"}, + {goarch: "386", instruction: "ENDBR64.P"}, + {goarch: "386", instruction: "SWAPGS AX"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's fixed-system tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_fixupimm.go b/amd64_lower_fixupimm.go new file mode 100644 index 000000000..652afc026 --- /dev/null +++ b/amd64_lower_fixupimm.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FixupImmediateSpec struct { + laneBits int + scalar bool +} + +var amd64FixupImmediateSpecs = map[Op]amd64FixupImmediateSpec{ + "VFIXUPIMMPS": {laneBits: 32}, + "VFIXUPIMMPD": {laneBits: 64}, + "VFIXUPIMMSS": {laneBits: 32, scalar: true}, + "VFIXUPIMMSD": {laneBits: 64, scalar: true}, +} + +func (c *amd64Ctx) lowerFixupImmediate(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FixupImmediateSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + shape := amd64UnaryImmediateFloatingShape{laneBits: spec.laneBits, scalar: spec.scalar, twoSource: true} + form, err := c.parseUnaryImmediateFloatingForm(baseOp, suffix, shape, ins) + if err != nil { + return true, false, err + } + daz := c.loadMXCSRDAZ() + if spec.scalar { + return c.lowerScalarFixupImmediate(spec, form, daz) + } + return c.lowerPackedFixupImmediate(spec, form, daz) +} + +func (c *amd64Ctx) lowerPackedFixupImmediate(spec amd64FixupImmediateSpec, form amd64UnaryImmediateFloatingForm, daz string) (bool, bool, error) { + lanes := form.byteWidth * 8 / spec.laneBits + table, err := c.loadPackedCompareLanes(form.source, form.byteWidth, spec.laneBits, form.properties.broadcast) + if err != nil { + return true, false, err + } + classified, err := c.loadPackedCompareLanes(form.passthrough, form.byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(form.destination, form.byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, spec.laneBits, oldBytes) + result := c.emitFixupImmediateBits(lanes, spec.laneBits, classified, table, old, daz) + if form.mask != "" { + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, form.mask, form.properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, form.byteWidth) + return true, false, c.storeVectorBytes(form.destination.Reg, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarFixupImmediate(spec amd64FixupImmediateSpec, form amd64UnaryImmediateFloatingForm, daz string) (bool, bool, error) { + tableBits, err := c.loadFloatingScalarBits(form.source, spec.laneBits) + if err != nil { + return true, false, err + } + classifiedBytes, err := c.loadX(form.passthrough.Reg) + if err != nil { + return true, false, err + } + classifiedVector := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, classifiedBytes) + classifiedLow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 0\n", classifiedLow, 128/spec.laneBits, spec.laneBits, classifiedVector) + oldLow, err := c.loadXLowInteger(form.destination.Reg, spec.laneBits) + if err != nil { + return true, false, err + } + tableVector, classifiedOne, oldVector := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", tableVector, spec.laneBits, spec.laneBits, tableBits) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %%%s, i32 0\n", classifiedOne, spec.laneBits, spec.laneBits, classifiedLow) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", oldVector, spec.laneBits, spec.laneBits, oldLow) + result := c.emitFixupImmediateBits(1, spec.laneBits, "%"+classifiedOne, "%"+tableVector, "%"+oldVector, daz) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + return true, false, c.storeVectorScalarRegister(form.destination.Reg, spec.laneBits, "%"+low, classifiedVector, form.mask, form.properties.zeroing) +} + +// emitFixupImmediateBits implements the data-result portion of Intel's +// FIXUPIMM_SP/FIXUPIMM_DP grammar. imm8 controls only MXCSR exception +// reporting; plan9asm does not currently expose floating exception state. +func (c *amd64Ctx) emitFixupImmediateBits(lanes, laneBits int, classified, table, old, daz string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + positiveOne := uint64(0x3f800000) + constants := [16]uint64{ + 0, 0, 0, 0xffc00000, + 0xff800000, 0x7f800000, 0, 0x80000000, + 0, 0xbf800000, 0x3f800000, 0x3f000000, + 0x42b40000, 0x3fc90fdb, 0x7f7fffff, 0xff7fffff, + } + if laneBits == 64 { + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + positiveOne = uint64(0x3ff0000000000000) + constants = [16]uint64{ + 0, 0, 0, 0xfff8000000000000, + 0xfff0000000000000, 0x7ff0000000000000, 0, 0x8000000000000000, + 0, 0xbff0000000000000, 0x3ff0000000000000, 0x3fe0000000000000, + 0x4056800000000000, 0x3ff921fb54442d18, 0x7fefffffffffffff, 0xffefffffffffffff, + } + } + result := "poison" + for lane := 0; lane < lanes; lane++ { + raw, tableLane, oldLane := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", raw, vectorType, classified, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", tableLane, vectorType, table, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", oldLane, vectorType, old, lane) + rawExponent, exponentZero, dazZero := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", rawExponent, laneBits, raw, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, rawExponent) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", dazZero, exponentZero, daz) + effective := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %%%s\n", effective, dazZero, laneBits, laneBits, raw) + + exponent, mantissa, sign, absolute := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, effective, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, effective, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", sign, laneBits, effective, signBit) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", absolute, laneBits, effective, signBit-1) + exponentOnes, mantissaZero, mantissaNonzero, negative, zero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", mantissaNonzero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", negative, laneBits, sign) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", zero, laneBits, absolute) + isNaN, isInfinity, quietSet := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + quietValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", quietValue, laneBits, mantissa, quietBit) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", quietSet, laneBits, quietValue) + notQuiet, signalingNaN, quietNaN := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notQuiet, quietSet) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", signalingNaN, isNaN, notQuiet) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", quietNaN, isNaN, quietSet) + negativeInfinity, positiveInfinity, notNegative := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", negativeInfinity, isInfinity, negative) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notNegative, negative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", positiveInfinity, isInfinity, notNegative) + positiveOneValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", positiveOneValue, laneBits, effective, positiveOne) + + token := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 6, i%d 7\n", token, negative, laneBits, laneBits) + for _, selection := range []struct { + predicate string + value int + }{ + {predicate: positiveInfinity, value: 5}, + {predicate: negativeInfinity, value: 4}, + {predicate: positiveOneValue, value: 3}, + {predicate: zero, value: 2}, + {predicate: signalingNaN, value: 1}, + {predicate: quietNaN, value: 0}, + } { + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %%%s\n", selected, selection.predicate, laneBits, selection.value, laneBits, token) + token = selected + } + shift, shiftedTable, response := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, 2\n", shift, laneBits, token) + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %%%s\n", shiftedTable, laneBits, tableLane, shift) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 15\n", response, laneBits, shiftedTable) + + quieted, signedInfinity := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quieted, laneBits, effective, exponentMask|quietBit) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", signedInfinity, laneBits, sign, exponentMask) + responseValues := [16]string{ + "%" + oldLane, "%" + effective, "%" + quieted, + fmt.Sprintf("%d", constants[3]), fmt.Sprintf("%d", constants[4]), fmt.Sprintf("%d", constants[5]), "%" + signedInfinity, + fmt.Sprintf("%d", constants[7]), fmt.Sprintf("%d", constants[8]), fmt.Sprintf("%d", constants[9]), fmt.Sprintf("%d", constants[10]), + fmt.Sprintf("%d", constants[11]), fmt.Sprintf("%d", constants[12]), fmt.Sprintf("%d", constants[13]), fmt.Sprintf("%d", constants[14]), fmt.Sprintf("%d", constants[15]), + } + laneResult := responseValues[0] + for responseValue := 1; responseValue < len(responseValues); responseValue++ { + matches, selected := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", matches, laneBits, response, responseValue) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %s, i%d %s\n", selected, matches, laneBits, responseValues[responseValue], laneBits, laneResult) + laneResult = "%" + selected + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, laneBits, laneResult, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_fixupimm_test.go b/amd64_lower_fixupimm_test.go new file mode 100644 index 000000000..8cd63792e --- /dev/null +++ b/amd64_lower_fixupimm_test.go @@ -0,0 +1,265 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +type fixupImmediateTestSpec struct { + op string + laneBits int + scalar bool +} + +func go127FixupImmediateSpecs() []fixupImmediateTestSpec { + return []fixupImmediateTestSpec{ + {op: "VFIXUPIMMPS", laneBits: 32}, + {op: "VFIXUPIMMPD", laneBits: 64}, + {op: "VFIXUPIMMSS", laneBits: 32, scalar: true}, + {op: "VFIXUPIMMSD", laneBits: 64, scalar: true}, + } +} + +func TestAMD64FixupImmediateGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127FixupImmediateSpecs() + if len(amd64FixupImmediateSpecs) != len(expected) { + t.Fatalf("FIXUPIMM grammar has %d entries, want %d", len(amd64FixupImmediateSpecs), len(expected)) + } + for _, want := range expected { + got, ok := amd64FixupImmediateSpecs[Op(want.op)] + if !ok { + t.Errorf("FIXUPIMM grammar omitted %s", want.op) + continue + } + if got.laneBits != want.laneBits || got.scalar != want.scalar { + t.Errorf("FIXUPIMM grammar %s = %+v, want laneBits=%d scalar=%v", want.op, got, want.laneBits, want.scalar) + } + } +} + +func TestTranslateX86FixupImmediateCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT fixupimmforms(SB),$0-0\n") + for index, spec := range go127FixupImmediateSpecs() { + if spec.scalar { + fmt.Fprintf(&source, "\t%s $%d, X0, X1, X23\n", spec.op, index) + fmt.Fprintf(&source, "\t%s $%d, %d(AX), X1, X23\n", spec.op, index+16, index*8) + fmt.Fprintf(&source, "\t%s $%d, X2, X1, K1, X23\n", spec.op, index+32) + fmt.Fprintf(&source, "\t%s.Z $%d, %d(AX), X1, K2, X23\n", spec.op, index+48, index*8) + fmt.Fprintf(&source, "\t%s.SAE $%d, X3, X1, X23\n", spec.op, index+64) + fmt.Fprintf(&source, "\t%s.SAE.Z $%d, X4, X1, K3, X23\n", spec.op, index+80) + continue + } + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s $%d, %s0, %s1, %s23\n", spec.op, index, width, width, width) + fmt.Fprintf(&source, "\t%s $%d, %d(AX), %s1, %s23\n", spec.op, index+16, index*8, width, width) + fmt.Fprintf(&source, "\t%s.BCST $%d, %d(AX), %s1, %s23\n", spec.op, index+32, index*8, width, width) + fmt.Fprintf(&source, "\t%s $%d, %s2, %s1, K1, %s23\n", spec.op, index+48, width, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z $%d, %d(AX), %s1, K2, %s23\n", spec.op, index+64, index*8, width, width) + } + fmt.Fprintf(&source, "\t%s.SAE $%d, Z3, Z1, Z23\n", spec.op, index+80) + fmt.Fprintf(&source, "\t%s.SAE.Z $%d, Z4, Z1, K3, Z23\n", spec.op, index+96) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: target.triple, + Sigs: map[string]FuncSig{"fixupimmforms": {Name: "fixupimmforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "fixupimm-"+target.name+".ll", "fixupimm-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FixupImmediateRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VFIXUPIMMPS $-1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS.Z $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS $1, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS.SAE $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS.SAE $1, 0(AX), Z1, Z2"}, + {goarch: "amd64", instruction: "VFIXUPIMMPS $1, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMSS.BCST $1, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VFIXUPIMMSS.SAE $1, 0(AX), X1, X2"}, + {goarch: "386", instruction: "VFIXUPIMMPS $1, X0, X1, X2"}, + {goarch: "386", instruction: "VFIXUPIMMSS $1, X0, X1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's FIXUPIMM tables", test.instruction) + } + }) + } +} + +func TestAMD64FixupImmediateRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT fixupimmsemantics(SB),$0-64 + MOVQ out+0(FP), AX + MOVQ src32+8(FP), BX + MOVQ table32+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ src64+32(FP), R8 + MOVQ table64+40(FP), R9 + MOVQ old64+48(FP), R10 + MOVQ mask+56(FP), R11 + KMOVQ R11, K1 + VMOVDQU32 0(BX), Z1 + VMOVDQU32 0(CX), Z0 + VMOVDQU32 0(DX), Z2 + STC + VFIXUPIMMPS $0, Z0, Z1, Z2 + VMOVDQU32 Z2, 0(AX) + VMOVDQU32 0(DX), Z2 + VFIXUPIMMPS $0, Z0, Z1, K1, Z2 + VMOVDQU32 Z2, 64(AX) + VFIXUPIMMPS.Z $0, Z0, Z1, K1, Z3 + VMOVDQU32 Z3, 128(AX) + VMOVDQU64 0(R8), Z5 + VMOVDQU64 0(R9), Z4 + VMOVDQU64 0(R10), Z6 + VFIXUPIMMPD $0, Z4, Z5, Z6 + VMOVDQU64 Z6, 192(AX) + VMOVDQU64 0(R8), X5 + VMOVDQU64 0(R9), X4 + VMOVDQU64 0(R10), X6 + VFIXUPIMMSD $0, X4, X5, X6 + VMOVDQU64 X6, 256(AX) + VMOVDQU32 0(BX), X1 + VMOVDQU32 0(CX), X0 + VMOVDQU32 0(DX), X2 + VFIXUPIMMSS $0, X0, X1, X2 + VMOVDQU32 X2, 272(AX) + SETCS 288(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"fixupimmsemantics": { + Name: "fixupimmsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void fixupimmsemantics(uint8_t *, const uint32_t *, const uint32_t *, const uint32_t *, const uint64_t *, const uint64_t *, const uint64_t *, uint64_t); +static unsigned token32(uint32_t x, int daz) { + if (daz && (x & UINT32_C(0x7f800000)) == 0) x = 0; + uint32_t a=x&UINT32_C(0x7fffffff),m=x&UINT32_C(0x007fffff),e=x&UINT32_C(0x7f800000); + if(e==UINT32_C(0x7f800000)&&m) return m&UINT32_C(0x00400000)?0:1; + if(a==0)return 2;if(x==UINT32_C(0x3f800000))return 3; + if(a==UINT32_C(0x7f800000))return x>>31?4:5;return x>>31?6:7; +} +static unsigned token64(uint64_t x, int daz) { + if (daz && (x & UINT64_C(0x7ff0000000000000)) == 0) x = 0; + uint64_t a=x&UINT64_C(0x7fffffffffffffff),m=x&UINT64_C(0x000fffffffffffff),e=x&UINT64_C(0x7ff0000000000000); + if(e==UINT64_C(0x7ff0000000000000)&&m) return m&UINT64_C(0x0008000000000000)?0:1; + if(a==0)return 2;if(x==UINT64_C(0x3ff0000000000000))return 3; + if(a==UINT64_C(0x7ff0000000000000))return x>>63?4:5;return x>>63?6:7; +} +static uint32_t effective32(uint32_t x,int daz){return daz&&(x&UINT32_C(0x7f800000))==0?0:x;} +static uint64_t effective64(uint64_t x,int daz){return daz&&(x&UINT64_C(0x7ff0000000000000))==0?0:x;} +static uint32_t fix32(uint32_t old,uint32_t x,uint32_t table,int daz){ + x=effective32(x,daz);unsigned r=(table>>(4*token32(x,0)))&15; + static const uint32_t c[16]={0,0,0,UINT32_C(0xffc00000),UINT32_C(0xff800000),UINT32_C(0x7f800000),0,UINT32_C(0x80000000),0,UINT32_C(0xbf800000),UINT32_C(0x3f800000),UINT32_C(0x3f000000),UINT32_C(0x42b40000),UINT32_C(0x3fc90fdb),UINT32_C(0x7f7fffff),UINT32_C(0xff7fffff)}; + if(r==0)return old;if(r==1)return x;if(r==2)return x|UINT32_C(0x7fc00000);if(r==6)return (x&UINT32_C(0x80000000))|UINT32_C(0x7f800000);return c[r]; +} +static uint64_t fix64(uint64_t old,uint64_t x,uint64_t table,int daz){ + x=effective64(x,daz);unsigned r=(table>>(4*token64(x,0)))&15; + static const uint64_t c[16]={0,0,0,UINT64_C(0xfff8000000000000),UINT64_C(0xfff0000000000000),UINT64_C(0x7ff0000000000000),0,UINT64_C(0x8000000000000000),0,UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0x3fe0000000000000),UINT64_C(0x4056800000000000),UINT64_C(0x3ff921fb54442d18),UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff)}; + if(r==0)return old;if(r==1)return x;if(r==2)return x|UINT64_C(0x7ff8000000000000);if(r==6)return (x&UINT64_C(0x8000000000000000))|UINT64_C(0x7ff0000000000000);return c[r]; +} +int main(void){ + const uint32_t src32[16]={UINT32_C(0x7fc12345),1,UINT32_C(0xc0200001),0,UINT32_C(0xff800000),UINT32_C(0x7f800000),UINT32_C(0xc0000000),UINT32_C(0x40000000),UINT32_C(0x80000000),UINT32_C(0x3f800000),UINT32_C(0x7f812345),UINT32_C(0x80000001),UINT32_C(0x40400000),UINT32_C(0xc0800000),UINT32_C(0x3f400000),UINT32_C(0xbf400000)}; + const uint64_t src64[8]={UINT64_C(0x7ff8123456789abc),1,UINT64_C(0xc004000000000001),0,UINT64_C(0xfff0000000000000),UINT64_C(0x7ff0000000000000),UINT64_C(0xc000000000000000),UINT64_C(0x4000000000000000)}; + uint32_t old32[16],table32[16],got32[48],want32[48];uint64_t old64[8],table64[8],got64[10],want64[10];uint8_t out[289]; + const uint64_t mask=UINT64_C(0xa55a);unsigned original=_mm_getcsr(); + for(int i=0;i<16;i++)old32[i]=UINT32_C(0x41000000)+i;for(int i=0;i<8;i++)old64[i]=UINT64_C(0x4020000000000000)+i; + for(int daz=0;daz<2;daz++){ + _mm_setcsr((original&~64u)|(daz?64u:0));memset(out,0,sizeof(out)); + for(int i=0;i<16;i++)table32[i]=(uint32_t)i<<(4*token32(src32[i],daz)); + for(int i=0;i<8;i++)table64[i]=(uint64_t)(i+8)<<(4*token64(src64[i],daz)); + fixupimmsemantics(out,src32,table32,old32,src64,table64,old64,mask);memcpy(got32,out,192);memcpy(got64,out+192,80); + for(int i=0;i<16;i++){uint32_t v=fix32(old32[i],src32[i],table32[i],daz);want32[i]=v;want32[16+i]=(mask>>i)&1?v:old32[i];want32[32+i]=(mask>>i)&1?v:0;} + for(int i=0;i<8;i++)want64[i]=fix64(old64[i],src64[i],table64[i],daz);want64[8]=fix64(old64[0],src64[0],table64[0],daz);want64[9]=src64[1]; + for(int i=0;i<48;i++)if(got32[i]!=want32[i])return 10+daz; + for(int i=0;i<10;i++)if(got64[i]!=want64[i])return 20+daz; + uint32_t scalar32[4];memcpy(scalar32,out+272,16);if(scalar32[0]!=fix32(old32[0],src32[0],table32[0],daz)||scalar32[1]!=src32[1]||scalar32[2]!=src32[2]||scalar32[3]!=src32[3])return 30+daz; + if(out[288]!=1)return 40+daz; + } + _mm_setcsr(original);return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "fixupimm_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_float_add_sub.go b/amd64_lower_float_add_sub.go new file mode 100644 index 000000000..dfb1060bc --- /dev/null +++ b/amd64_lower_float_add_sub.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerFloatAddSub(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := x86FloatAddSubSpecs[Op(base)]; !recognized { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("floating add/sub dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86FloatAddSubProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + return c.lowerFloatBinaryOperands(ins, form.x86FloatBinaryForm, func(lanes int, left, right, mask string) string { + return c.emitFloatAddSub(form, lanes, left, right, mask) + }) +} + +// Shared yxm/VEX/EVEX source loading and alias writes. Each family supplies +// its arithmetic emitter, so a common shape grants no numerical behavior. +func (c *amd64Ctx) lowerFloatBinaryOperands(ins Instr, form x86FloatBinaryForm, emit func(int, string, string, string) string) (bool, bool, error) { + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.scalar && form.memory >= 0 { + return true, false, c.lowerLegacyPackedFloatingMemory(ins, form) + } + if first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask string + var err error + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + bits := form.bits + if form.scalar { + var a string + var err error + if form.literal { + if bits == 32 { + a, err = c.evalF32(first) + } else { + a, err = c.evalF64(first) + } + } else { + a, err = c.loadFloatBinaryScalar(first, bits, mask) + } + if err != nil { + return true, false, err + } + b, err := c.loadFMA3Scalar(second, bits, "") + if err != nil { + return true, false, err + } + value := emit(1, b, a, mask) + integer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", integer, amd64FMA3LLVMType(1, bits), value, bits) + upper, err := c.loadX(second.Reg) + if err != nil { + return true, false, err + } + lanes := 128 / bits + base := c.bitcastVectorBytesToIntegerLanes(16, lanes, bits, upper) + if !form.legacy { + return true, false, c.storeScalarMoveRegister(destination.Reg, bits, "%"+integer, base, mask, form.zeroing) + } + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 0\n", merged, lanes, bits, base, bits, integer) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", bytes, lanes, bits, merged) + return true, false, c.storeLegacyPackedLogical(destination, "%"+bytes) + } + lanes := form.byteWidth * 8 / bits + a, err := c.loadFloatBinaryPacked(first, form.byteWidth, bits, form.broadcast, mask) + if err != nil { + return true, false, err + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + floatType := amd64FMA3LLVMType(lanes, bits) + aFloat, bFloat := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", aFloat, lanes, bits, a, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", bFloat, lanes, bits, b, floatType) + value := emit(lanes, "%"+bFloat, "%"+aFloat, mask) + result := c.bitcastFloatingToIntegerLanes(floatType, lanes, bits, value) + if form.mask >= 0 { + old, err := c.loadPackedCompareLanes(destination, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, bits, result, old, mask, form.zeroing) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, bits, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+bytes) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+bytes) +} + +func (c *amd64Ctx) emitFloatAddSub(form x86FloatAddSubForm, lanes int, left, right, mask string) string { + if form.rounding != "" { + return c.emitRoundedFloatAddSub(lanes, form.spec.bits, left, right, form.spec.subtract, form.rounding) + } + // Do not grant fast-math, reassociation or an ambient-rounding assumption. + // Dynamic strict operations retain x86 rounding/DAZ/FTZ/exception behavior. + operation := "fadd" + if form.spec.subtract { + operation = "fsub" + } + return c.emitStrictFloatBinary(form.x86FloatBinaryForm, operation, lanes, left, right, mask, func(lanes int, left, right, value string) string { + return c.emitFloatAddSubSpecial(form.spec.bits, lanes, left, right, form.spec.subtract, value) + }) +} + +// Strict operations must remain observable under middle-end optimization. +// The correction callback handles each family's native NaN result priority +// without replacing the actual hardware rounding or exception side effects. +func (c *amd64Ctx) emitStrictFloatBinary(form x86FloatBinaryForm, operation string, lanes int, left, right, mask string, correct func(int, string, string, string) string) string { + typ := amd64FMA3LLVMType(lanes, form.bits) + if mask != "" { + // EVEX suppresses arithmetic exceptions as well as memory faults for + // inactive lanes. Selecting after an eager strict fadd is not enough. + scalarType := amd64FMA3LLVMType(1, form.bits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := []string{left, right} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, typ, value, lane) + values[index] = "%" + extracted + } + } + active := amd64MaskBitI1(c, mask, lane) + integer, _ := c.loadX86ScalarIf(form.bits, active, func() (string, error) { + value, bits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.experimental.constrained.%s.f%d(%s %s, %s %s, metadata !\"round.dynamic\", metadata !\"fpexcept.strict\")\n", value, scalarType, operation, form.bits, scalarType, values[0], scalarType, values[1]) + corrected := correct(1, values[0], values[1], "%"+value) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, scalarType, corrected, form.bits) + return "%" + bits, nil + }) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to %s\n", value, form.bits, integer, scalarType) + if lanes == 1 { + return "%" + value + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, typ, result, scalarType, value, lane) + result = "%" + inserted + } + return result + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.experimental.constrained.%s.%s(%s %s, %s %s, metadata !\"round.dynamic\", metadata !\"fpexcept.strict\")\n", result, typ, operation, amd64BinaryFloatingIntrinsicSuffix(lanes, form.bits), typ, left, typ, right) + return correct(lanes, left, right, "%"+result) +} + +// LLVM's commutative add legalization can swap NaN operands even for a +// constrained operation. Preserve its real hardware exception/rounding +// effects, but restore x86 binary source-order payload/indefinite results. +func (c *amd64Ctx) emitFloatAddSubSpecial(bits, lanes int, left, right string, subtract bool, hardware string) string { + scalarType, typ := amd64FMA3LLVMType(1, bits), amd64FMA3LLVMType(lanes, bits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := []string{left, right, hardware} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, typ, value, lane) + values[index] = "%" + extracted + } + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @__plan9asm_add_sub_special_f%d(%s %s, %s %s, i1 %t, %s %s)\n", value, scalarType, bits, scalarType, values[0], scalarType, values[1], subtract, scalarType, values[2]) + if lanes == 1 { + return "%" + value + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, typ, result, scalarType, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_float_add_sub_round.go b/amd64_lower_float_add_sub_round.go new file mode 100644 index 000000000..8eca3ad81 --- /dev/null +++ b/amd64_lower_float_add_sub_round.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Classify binary x86 special values using integer operations only. This is +// also used after a real strict operation without changing its MXCSR effects. +func emitX86FloatAddSubSpecialBits(e *amd64FMA3IntegerEmitter, bits int) (special, result, rightSigned string) { + fraction, exponent := 23, uint64(0x7f800000) + if bits == 64 { + fraction, exponent = 52, 0x7ff0000000000000 + } + typ := amd64FMA3LLVMType(1, bits) + left := e.value("bitcast %s %%left to i%d", typ, bits) + right := e.value("bitcast %s %%right to i%d", typ, bits) + sign := uint64(1) << (bits - 1) + leftAbs := e.value("and i%d %s, %d", bits, left, sign-1) + rightAbs := e.value("and i%d %s, %d", bits, right, sign-1) + leftNaN := e.value("icmp ugt i%d %s, %d", bits, leftAbs, exponent) + rightNaN := e.value("icmp ugt i%d %s, %d", bits, rightAbs, exponent) + leftInf := e.value("icmp eq i%d %s, %d", bits, leftAbs, exponent) + rightInf := e.value("icmp eq i%d %s, %d", bits, rightAbs, exponent) + special = e.value("or i1 %s, %s", leftNaN, rightNaN) + special = e.value("or i1 %s, %s", special, leftInf) + special = e.value("or i1 %s, %s", special, rightInf) + negated := e.value("xor i%d %s, %d", bits, right, sign) + rightSigned = e.value("select i1 %%subtract, i%d %s, i%d %s", bits, negated, bits, right) + quiet := uint64(1) << (fraction - 1) + leftQuiet := e.value("or i%d %s, %d", bits, left, quiet) + rightQuiet := e.value("or i%d %s, %d", bits, right, quiet) + // SUB never toggles a NaN's sign/payload, only its finite/inf operand sign. + nan := e.value("select i1 %s, i%d %s, i%d %s", leftNaN, bits, leftQuiet, bits, rightQuiet) + anyNaN := e.value("or i1 %s, %s", leftNaN, rightNaN) + signs := e.value("xor i%d %s, %s", bits, left, rightSigned) + opposite := e.value("icmp slt i%d %s, 0", bits, signs) + bothInf := e.value("and i1 %s, %s", leftInf, rightInf) + invalid := e.value("and i1 %s, %s", bothInf, opposite) + infinity := e.value("select i1 %s, i%d %s, i%d %s", leftInf, bits, left, bits, rightSigned) + infinity = e.value("select i1 %s, i%d %d, i%d %s", invalid, bits, sign|exponent|quiet, bits, infinity) + result = e.value("select i1 %s, i%d %s, i%d %s", anyNaN, bits, nan, bits, infinity) + return +} + +func emitX86FloatAddSubSpecialHelper(b *strings.Builder, bits int) { + typ := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(b, "define internal %s @__plan9asm_add_sub_special_f%d(%s %%left, %s %%right, i1 %%subtract, %s %%hardware) {\nentry:\n", typ, bits, typ, typ, typ) + e := amd64FMA3IntegerEmitter{b: b} + special, result, _ := emitX86FloatAddSubSpecialBits(&e, bits) + floatResult := e.value("bitcast i%d %s to %s", bits, result, typ) + selected := e.value("select i1 %s, %s %s, %s %%hardware", special, typ, floatResult, typ) + fmt.Fprintf(b, " ret %s %s\n}\n\n", typ, selected) +} + +// Embedded rounding suppresses exceptions. Only finite binary arithmetic +// uses the reviewed exact accumulator (a*1+b); special results remain ADD/SUB +// semantics rather than FMA's operand priority. +func emitX86FloatAddSubRoundingHelper(b *strings.Builder, bits int) { + typ := amd64FMA3LLVMType(1, bits) + fmt.Fprintf(b, "define internal %s @__plan9asm_add_sub_round_f%d(%s %%left, %s %%right, i1 %%subtract, i32 %%mode, i32 %%mxcsr) {\nentry:\n", typ, bits, typ, typ) + e := amd64FMA3IntegerEmitter{b: b} + special, result, rightSigned := emitX86FloatAddSubSpecialBits(&e, bits) + fmt.Fprintf(b, " br i1 %s, label %%special, label %%finite\nspecial:\n", special) + floatResult := e.value("bitcast i%d %s to %s", bits, result, typ) + fmt.Fprintf(b, " ret %s %s\nfinite:\n", typ, floatResult) + rightFloat := e.value("bitcast i%d %s to %s", bits, rightSigned, typ) + finite := e.value("call %s @__plan9asm_fma3_round_f%d(%s %%left, %s 1.0, %s %s, i32 %%mode, i32 %%mxcsr)", typ, bits, typ, typ, typ, rightFloat) + fmt.Fprintf(b, " ret %s %s\n}\n\n", typ, finite) +} + +func (c *amd64Ctx) emitRoundedFloatAddSub(lanes, bits int, left, right string, subtract bool, rounding string) string { + typ, vectorType := amd64FMA3LLVMType(1, bits), amd64FMA3LLVMType(lanes, bits) + mode, mxcsr := c.packedNumericRoundingMode(rounding), c.loadMXCSR() + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := []string{left, right} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, vectorType, value, lane) + values[index] = "%" + extracted + } + } + computed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @__plan9asm_add_sub_round_f%d(%s %s, %s %s, i1 %t, i32 %s, i32 %s)\n", computed, typ, bits, typ, values[0], typ, values[1], subtract, mode, mxcsr) + if lanes == 1 { + return "%" + computed + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, vectorType, result, typ, computed, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_float_movemask_test.go b/amd64_lower_float_movemask_test.go new file mode 100644 index 000000000..995424748 --- /dev/null +++ b/amd64_lower_float_movemask_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86FloatingMoveMaskCompleteGoAssemblerForms(t *testing.T) { + // MOVMSK{PS,PD} uses Go's legacy yxrrl table (X -> Yrl), while the + // VMOVMSK forms use _yvmovmskpd (X/Y -> Yrl). Neither table accepts + // memory, EVEX registers, masks, or opcode suffixes. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastLegacyX := 15 + if target.goarch == "386" { + lastLegacyX = 7 + } + source := fmt.Sprintf(` +TEXT floatingmovemaskforms(SB),NOSPLIT,$0-0 + MOVMSKPS X%d, AX + MOVMSKPD X%d, BX + VMOVMSKPS X15, CX + VMOVMSKPS Y15, DX + VMOVMSKPD X15, SI + VMOVMSKPD Y15, DI + RET +`, lastLegacyX, lastLegacyX) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"floatingmovemaskforms": {Name: "floatingmovemaskforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "floating-movemask-"+target.name+".ll", "floating-movemask-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86FloatingMoveMaskRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "MOVMSKPS Y0, AX", + "MOVMSKPD (AX), BX", + "MOVMSKPS X16, AX", + "VMOVMSKPS Z0, AX", + "VMOVMSKPD X16, AX", + "VMOVMSKPS Y16, AX", + "VMOVMSKPD (AX), BX", + "VMOVMSKPS X0, X1", + "VMOVMSKPD X0, K1, AX", + "MOVMSKPS.Z X0, AX", + "VMOVMSKPD.BCST Y0, AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86FloatingMoveMaskRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86FloatingMoveMaskRejected(t, "386", "i386-unknown-linux-gnu", "MOVMSKPS X8, AX") +} + +func assertX86FloatingMoveMaskRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's floating movemask forms", instruction) + } +} diff --git a/amd64_lower_float_mul_div.go b/amd64_lower_float_mul_div.go new file mode 100644 index 000000000..b4a808e46 --- /dev/null +++ b/amd64_lower_float_mul_div.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerFloatMulDiv(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := x86FloatMulDivSpecs[Op(base)]; !recognized { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("floating mul/div dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86FloatMulDivProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + return c.lowerFloatBinaryOperands(ins, form.x86FloatBinaryForm, func(lanes int, left, right, mask string) string { + if form.rounding != "" { + return c.emitRoundedFloatMulDiv(form, lanes, left, right) + } + operation := "fmul" + if form.spec.divide { + operation = "fdiv" + } + return c.emitStrictFloatBinary(form.x86FloatBinaryForm, operation, lanes, left, right, mask, func(lanes int, left, right, value string) string { + return c.emitFloatBinaryNaNPriority(form.bits, lanes, left, right, value) + }) + }) +} + +// LLVM may commute strict multiplication during legalization. Restore x86's +// first-source NaN payload without executing another floating operation or +// changing the real strict operation's sticky/rounding/DAZ/FTZ effects. +func (c *amd64Ctx) emitFloatBinaryNaNPriority(bits, lanes int, left, right, hardware string) string { + typ := amd64FMA3LLVMType(lanes, bits) + integerType := fmt.Sprintf("<%d x i%d>", lanes, bits) + if lanes == 1 { + integerType = fmt.Sprintf("i%d", bits) + } + exponent, quiet := uint64(0x7f800000), uint64(0x400000) + if bits == 64 { + exponent, quiet = 0x7ff0000000000000, 0x8000000000000 + } + constant := func(value uint64) string { + if lanes == 1 { + return fmt.Sprint(value) + } + values := make([]string, lanes) + for lane := range values { + values[lane] = fmt.Sprintf("i%d %d", bits, value) + } + return "<" + strings.Join(values, ", ") + ">" + } + values, nan, quieted := [2]string{left, right}, [2]string{}, [2]string{} + for index, value := range values { + raw, magnitude, isNaN, withQuiet := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", raw, typ, value, integerType) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", magnitude, integerType, raw, constant(uint64(1)<<(bits-1)-1)) + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", isNaN, integerType, magnitude, constant(exponent)) + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", withQuiet, integerType, raw, constant(quiet)) + nan[index], quieted[index] = "%"+isNaN, "%"+withQuiet + } + predicateType := "i1" + if lanes > 1 { + predicateType = fmt.Sprintf("<%d x i1>", lanes) + } + anyNaN, chosen, selected, rawHardware := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %s, %s\n", anyNaN, predicateType, nan[0], nan[1]) + fmt.Fprintf(c.b, " %%%s = select %s %s, %s %s, %s %s\n", chosen, predicateType, nan[0], integerType, quieted[0], integerType, quieted[1]) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", rawHardware, typ, hardware, integerType) + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %%%s, %s %%%s\n", selected, predicateType, anyNaN, integerType, chosen, integerType, rawHardware) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to %s\n", result, integerType, selected, typ) + return "%" + result +} + +func (c *amd64Ctx) emitRoundedFloatMulDiv(form x86FloatMulDivForm, lanes int, left, right string) string { + typ, vectorType := amd64FMA3LLVMType(1, form.bits), amd64FMA3LLVMType(lanes, form.bits) + mode, mxcsr := c.packedNumericRoundingMode(form.rounding), c.loadMXCSR() + result := "poison" + for lane := 0; lane < lanes; lane++ { + values := [2]string{left, right} + if lanes > 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, vectorType, value, lane) + values[index] = "%" + extracted + } + } + for index, floating := range values { + raw := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", raw, typ, floating, form.bits) + values[index] = "%" + raw + } + raw, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i%d @__plan9asm_mul_div_round_f%d(i%d %s, i%d %s, i1 %t, i32 %s, i32 %s)\n", raw, form.bits, form.bits, form.bits, values[0], form.bits, values[1], form.spec.divide, mode, mxcsr) + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to %s\n", value, form.bits, raw, typ) + if lanes == 1 { + return "%" + value + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, vectorType, result, typ, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_fma3.go b/amd64_lower_fma3.go new file mode 100644 index 000000000..6f22555b6 --- /dev/null +++ b/amd64_lower_fma3.go @@ -0,0 +1,374 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FMA3Mode uint8 + +const ( + amd64FMA3Add amd64FMA3Mode = iota + amd64FMA3Sub + amd64FMA3NegAdd + amd64FMA3NegSub + amd64FMA3AddSub + amd64FMA3SubAdd +) + +type amd64FMA3Spec struct { + laneBits int + order int + mode amd64FMA3Mode + scalar bool + rawOnly bool +} + +// amd64FMA3Specs is the complete Go 1.27 FMA3 family. The 36 packed +// instructions use _yvaddpd and the 24 scalar instructions use _yvaddsd. +// The additional 30 MAP6 FP16 forms are accepted only from raw bytes. +var amd64FMA3Specs = map[Op]amd64FMA3Spec{ + "VFMADD132PS": {laneBits: 32, order: 132, mode: amd64FMA3Add}, + "VFMADD132PD": {laneBits: 64, order: 132, mode: amd64FMA3Add}, + "VFMADD132SS": {laneBits: 32, order: 132, mode: amd64FMA3Add, scalar: true}, + "VFMADD132SD": {laneBits: 64, order: 132, mode: amd64FMA3Add, scalar: true}, + "VFMADD213PS": {laneBits: 32, order: 213, mode: amd64FMA3Add}, + "VFMADD213PD": {laneBits: 64, order: 213, mode: amd64FMA3Add}, + "VFMADD213SS": {laneBits: 32, order: 213, mode: amd64FMA3Add, scalar: true}, + "VFMADD213SD": {laneBits: 64, order: 213, mode: amd64FMA3Add, scalar: true}, + "VFMADD231PS": {laneBits: 32, order: 231, mode: amd64FMA3Add}, + "VFMADD231PD": {laneBits: 64, order: 231, mode: amd64FMA3Add}, + "VFMADD231SS": {laneBits: 32, order: 231, mode: amd64FMA3Add, scalar: true}, + "VFMADD231SD": {laneBits: 64, order: 231, mode: amd64FMA3Add, scalar: true}, + "VFMSUB132PS": {laneBits: 32, order: 132, mode: amd64FMA3Sub}, + "VFMSUB132PD": {laneBits: 64, order: 132, mode: amd64FMA3Sub}, + "VFMSUB132SS": {laneBits: 32, order: 132, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB132SD": {laneBits: 64, order: 132, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB213PS": {laneBits: 32, order: 213, mode: amd64FMA3Sub}, + "VFMSUB213PD": {laneBits: 64, order: 213, mode: amd64FMA3Sub}, + "VFMSUB213SS": {laneBits: 32, order: 213, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB213SD": {laneBits: 64, order: 213, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB231PS": {laneBits: 32, order: 231, mode: amd64FMA3Sub}, + "VFMSUB231PD": {laneBits: 64, order: 231, mode: amd64FMA3Sub}, + "VFMSUB231SS": {laneBits: 32, order: 231, mode: amd64FMA3Sub, scalar: true}, + "VFMSUB231SD": {laneBits: 64, order: 231, mode: amd64FMA3Sub, scalar: true}, + "VFNMADD132PS": {laneBits: 32, order: 132, mode: amd64FMA3NegAdd}, + "VFNMADD132PD": {laneBits: 64, order: 132, mode: amd64FMA3NegAdd}, + "VFNMADD132SS": {laneBits: 32, order: 132, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD132SD": {laneBits: 64, order: 132, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD213PS": {laneBits: 32, order: 213, mode: amd64FMA3NegAdd}, + "VFNMADD213PD": {laneBits: 64, order: 213, mode: amd64FMA3NegAdd}, + "VFNMADD213SS": {laneBits: 32, order: 213, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD213SD": {laneBits: 64, order: 213, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD231PS": {laneBits: 32, order: 231, mode: amd64FMA3NegAdd}, + "VFNMADD231PD": {laneBits: 64, order: 231, mode: amd64FMA3NegAdd}, + "VFNMADD231SS": {laneBits: 32, order: 231, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMADD231SD": {laneBits: 64, order: 231, mode: amd64FMA3NegAdd, scalar: true}, + "VFNMSUB132PS": {laneBits: 32, order: 132, mode: amd64FMA3NegSub}, + "VFNMSUB132PD": {laneBits: 64, order: 132, mode: amd64FMA3NegSub}, + "VFNMSUB132SS": {laneBits: 32, order: 132, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB132SD": {laneBits: 64, order: 132, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB213PS": {laneBits: 32, order: 213, mode: amd64FMA3NegSub}, + "VFNMSUB213PD": {laneBits: 64, order: 213, mode: amd64FMA3NegSub}, + "VFNMSUB213SS": {laneBits: 32, order: 213, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB213SD": {laneBits: 64, order: 213, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB231PS": {laneBits: 32, order: 231, mode: amd64FMA3NegSub}, + "VFNMSUB231PD": {laneBits: 64, order: 231, mode: amd64FMA3NegSub}, + "VFNMSUB231SS": {laneBits: 32, order: 231, mode: amd64FMA3NegSub, scalar: true}, + "VFNMSUB231SD": {laneBits: 64, order: 231, mode: amd64FMA3NegSub, scalar: true}, + "VFMADDSUB132PS": {laneBits: 32, order: 132, mode: amd64FMA3AddSub}, + "VFMADDSUB132PD": {laneBits: 64, order: 132, mode: amd64FMA3AddSub}, + "VFMADDSUB213PS": {laneBits: 32, order: 213, mode: amd64FMA3AddSub}, + "VFMADDSUB213PD": {laneBits: 64, order: 213, mode: amd64FMA3AddSub}, + "VFMADDSUB231PS": {laneBits: 32, order: 231, mode: amd64FMA3AddSub}, + "VFMADDSUB231PD": {laneBits: 64, order: 231, mode: amd64FMA3AddSub}, + "VFMSUBADD132PS": {laneBits: 32, order: 132, mode: amd64FMA3SubAdd}, + "VFMSUBADD132PD": {laneBits: 64, order: 132, mode: amd64FMA3SubAdd}, + "VFMSUBADD213PS": {laneBits: 32, order: 213, mode: amd64FMA3SubAdd}, + "VFMSUBADD213PD": {laneBits: 64, order: 213, mode: amd64FMA3SubAdd}, + "VFMSUBADD231PS": {laneBits: 32, order: 231, mode: amd64FMA3SubAdd}, + "VFMSUBADD231PD": {laneBits: 64, order: 231, mode: amd64FMA3SubAdd}, + "VFMADD132PH": {laneBits: 16, order: 132, mode: amd64FMA3Add, rawOnly: true}, + "VFMADD132SH": {laneBits: 16, order: 132, mode: amd64FMA3Add, scalar: true, rawOnly: true}, + "VFMSUB132PH": {laneBits: 16, order: 132, mode: amd64FMA3Sub, rawOnly: true}, + "VFMSUB132SH": {laneBits: 16, order: 132, mode: amd64FMA3Sub, scalar: true, rawOnly: true}, + "VFNMADD132PH": {laneBits: 16, order: 132, mode: amd64FMA3NegAdd, rawOnly: true}, + "VFNMADD132SH": {laneBits: 16, order: 132, mode: amd64FMA3NegAdd, scalar: true, rawOnly: true}, + "VFNMSUB132PH": {laneBits: 16, order: 132, mode: amd64FMA3NegSub, rawOnly: true}, + "VFNMSUB132SH": {laneBits: 16, order: 132, mode: amd64FMA3NegSub, scalar: true, rawOnly: true}, + "VFMADDSUB132PH": {laneBits: 16, order: 132, mode: amd64FMA3AddSub, rawOnly: true}, + "VFMSUBADD132PH": {laneBits: 16, order: 132, mode: amd64FMA3SubAdd, rawOnly: true}, + "VFMADD213PH": {laneBits: 16, order: 213, mode: amd64FMA3Add, rawOnly: true}, + "VFMADD213SH": {laneBits: 16, order: 213, mode: amd64FMA3Add, scalar: true, rawOnly: true}, + "VFMSUB213PH": {laneBits: 16, order: 213, mode: amd64FMA3Sub, rawOnly: true}, + "VFMSUB213SH": {laneBits: 16, order: 213, mode: amd64FMA3Sub, scalar: true, rawOnly: true}, + "VFNMADD213PH": {laneBits: 16, order: 213, mode: amd64FMA3NegAdd, rawOnly: true}, + "VFNMADD213SH": {laneBits: 16, order: 213, mode: amd64FMA3NegAdd, scalar: true, rawOnly: true}, + "VFNMSUB213PH": {laneBits: 16, order: 213, mode: amd64FMA3NegSub, rawOnly: true}, + "VFNMSUB213SH": {laneBits: 16, order: 213, mode: amd64FMA3NegSub, scalar: true, rawOnly: true}, + "VFMADDSUB213PH": {laneBits: 16, order: 213, mode: amd64FMA3AddSub, rawOnly: true}, + "VFMSUBADD213PH": {laneBits: 16, order: 213, mode: amd64FMA3SubAdd, rawOnly: true}, + "VFMADD231PH": {laneBits: 16, order: 231, mode: amd64FMA3Add, rawOnly: true}, + "VFMADD231SH": {laneBits: 16, order: 231, mode: amd64FMA3Add, scalar: true, rawOnly: true}, + "VFMSUB231PH": {laneBits: 16, order: 231, mode: amd64FMA3Sub, rawOnly: true}, + "VFMSUB231SH": {laneBits: 16, order: 231, mode: amd64FMA3Sub, scalar: true, rawOnly: true}, + "VFNMADD231PH": {laneBits: 16, order: 231, mode: amd64FMA3NegAdd, rawOnly: true}, + "VFNMADD231SH": {laneBits: 16, order: 231, mode: amd64FMA3NegAdd, scalar: true, rawOnly: true}, + "VFNMSUB231PH": {laneBits: 16, order: 231, mode: amd64FMA3NegSub, rawOnly: true}, + "VFNMSUB231SH": {laneBits: 16, order: 231, mode: amd64FMA3NegSub, scalar: true, rawOnly: true}, + "VFMADDSUB231PH": {laneBits: 16, order: 231, mode: amd64FMA3AddSub, rawOnly: true}, + "VFMSUBADD231PH": {laneBits: 16, order: 231, mode: amd64FMA3SubAdd, rawOnly: true}, +} + +type amd64FMA3Suffix struct { + broadcast bool + zeroing bool + rounding string +} + +func parseAMD64FMA3Suffix(suffix string) (amd64FMA3Suffix, bool) { + switch suffix { + case "": + return amd64FMA3Suffix{}, true + case "Z": + return amd64FMA3Suffix{zeroing: true}, true + case "BCST": + return amd64FMA3Suffix{broadcast: true}, true + case "BCST.Z": + return amd64FMA3Suffix{broadcast: true, zeroing: true}, true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + return amd64FMA3Suffix{rounding: suffix}, true + case "RN_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", "RZ_SAE.Z": + return amd64FMA3Suffix{zeroing: true, rounding: strings.TrimSuffix(suffix, ".Z")}, true + default: + return amd64FMA3Suffix{}, false + } +} + +func (c *amd64Ctx) lowerFMA3(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := amd64FMA3Specs[Op(base)]; !recognized { + return false, false, nil + } + // Bind only a local copy for legacy helpers. A dispatch/source conflict + // must be rejected before reading a register or emitting any LLVM IR. + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("FMA3 dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := amd64FMA3Properties(ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + spec, properties := form.spec, form.suffix + dstArg, byteWidth := ins.Args[form.destination], form.byteWidth + + var mask string + if form.mask >= 0 { + loadedMask, loadErr := c.loadK(ins.Args[form.mask].Reg) + if loadErr != nil { + return true, false, loadErr + } + mask = loadedMask + } + if spec.scalar { + return c.lowerScalarFMA3(spec, properties, ins, dstArg, mask) + } + return c.lowerPackedFMA3(spec, properties, ins, dstArg, byteWidth, mask) +} + +func (c *amd64Ctx) lowerPackedFMA3(spec amd64FMA3Spec, properties amd64FMA3Suffix, ins Instr, dstArg Operand, byteWidth int, mask string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + llvmType := amd64FMA3LLVMType(lanes, spec.laneBits) + load := func(arg Operand, allowBroadcast bool) (string, error) { + sourceMask := "" + if allowBroadcast { + sourceMask = mask + } + bits, err := c.loadMaskedPackedCompareLanes(arg, byteWidth, spec.laneBits, allowBroadcast && properties.broadcast, sourceMask) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", value, lanes, spec.laneBits, bits, llvmType) + return "%" + value, nil + } + first, err := load(ins.Args[0], true) + if err != nil { + return true, false, err + } + second, err := load(ins.Args[1], false) + if err != nil { + return true, false, err + } + old, err := load(dstArg, false) + if err != nil { + return true, false, err + } + computed := c.emitFMA3(spec, llvmType, lanes, first, second, old, properties.rounding) + if mask != "" { + computedBits := c.newTmp() + oldBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i%d>\n", computedBits, llvmType, computed, lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i%d>\n", oldBits, llvmType, old, lanes, spec.laneBits) + masked := amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, "%"+computedBits, "%"+oldBits, mask, properties.zeroing) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to %s\n", back, lanes, spec.laneBits, masked, llvmType) + computed = "%" + back + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, llvmType, computed, byteWidth) + return true, false, c.storePackedMoveOperand(dstArg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarFMA3(spec amd64FMA3Spec, properties amd64FMA3Suffix, ins Instr, dstArg Operand, mask string) (bool, bool, error) { + first, err := c.loadFMA3Scalar(ins.Args[0], spec.laneBits, mask) + if err != nil { + return true, false, err + } + second, err := c.loadFMA3Scalar(ins.Args[1], spec.laneBits, "") + if err != nil { + return true, false, err + } + old, err := c.loadFMA3Scalar(dstArg, spec.laneBits, "") + if err != nil { + return true, false, err + } + llvmType := amd64FMA3LLVMType(1, spec.laneBits) + computed := c.emitFMA3(spec, llvmType, 1, first, second, old, properties.rounding) + destinationBytes, err := c.loadX(dstArg.Reg) + if err != nil { + return true, false, err + } + lanes := 128 / spec.laneBits + base := c.bitcastVectorBytesToIntegerLanes(16, lanes, spec.laneBits, destinationBytes) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, llvmType, computed, spec.laneBits) + return true, false, c.storeScalarMoveRegister(dstArg.Reg, spec.laneBits, "%"+bits, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) loadFMA3Scalar(arg Operand, laneBits int, mask string) (string, error) { + var bits string + var err error + if arg.Kind == OpReg { + bits, err = c.loadXLowInteger(arg.Reg, laneBits) + } else if mask != "" { + bits, err = c.loadVectorScalarMemory(arg, laneBits, mask) + } else { + bits, err = c.evalIntSized(arg, amd64IntegerTypeForBits(laneBits)) + } + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %s to %s\n", value, laneBits, bits, amd64FMA3LLVMType(1, laneBits)) + return "%" + value, nil +} + +func amd64FMA3LLVMType(lanes, laneBits int) string { + element := "float" + if laneBits == 16 { + element = "half" + } else if laneBits == 64 { + element = "double" + } + if lanes == 1 { + return element + } + return fmt.Sprintf("<%d x %s>", lanes, element) +} + +func (c *amd64Ctx) splatFMA3Scalar(lanes, laneBits int, scalar string) string { + typeName := amd64FMA3LLVMType(lanes, laneBits) + element := amd64FMA3LLVMType(1, laneBits) + seed := c.newTmp() + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s poison, %s %s, i32 0\n", seed, typeName, element, scalar) + fmt.Fprintf(c.b, " %%%s = shufflevector %s %%%s, %s poison, <%d x i32> zeroinitializer\n", splat, typeName, seed, typeName, lanes) + return "%" + splat +} + +func (c *amd64Ctx) emitFMA3(spec amd64FMA3Spec, llvmType string, lanes int, first, second, old, rounding string) string { + a, b, addend := old, first, second + switch spec.order { + case 213: + a, b, addend = second, old, first + case 231: + a, b, addend = second, first, old + } + if spec.mode == amd64FMA3NegAdd || spec.mode == amd64FMA3NegSub { + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg %s %s\n", negated, llvmType, a) + a = "%" + negated + } + switch spec.mode { + case amd64FMA3Sub, amd64FMA3NegSub: + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg %s %s\n", negated, llvmType, addend) + addend = "%" + negated + case amd64FMA3AddSub, amd64FMA3SubAdd: + addend = c.alternateFMA3Addend(llvmType, lanes, addend, spec.mode == amd64FMA3AddSub) + } + if rounding != "" && !spec.rawOnly { + return c.emitRoundedFMA3(lanes, spec.laneBits, a, b, addend, rounding) + } + result := c.newTmp() + intrinsic := amd64FMA3IntrinsicName(lanes, spec.laneBits, rounding != "") + if rounding == "" { + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, %s %s)\n", result, llvmType, intrinsic, llvmType, a, llvmType, b, llvmType, addend) + } else { + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, %s %s, metadata !\"%s\", metadata !\"fpexcept.ignore\")\n", result, llvmType, intrinsic, llvmType, a, llvmType, b, llvmType, addend, amd64FMA3RoundingMetadata(rounding)) + } + return "%" + result +} + +func (c *amd64Ctx) alternateFMA3Addend(llvmType string, lanes int, addend string, negateEven bool) string { + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg %s %s\n", negated, llvmType, addend) + var mask strings.Builder + mask.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane > 0 { + mask.WriteString(", ") + } + selected := lane%2 == 0 + if !negateEven { + selected = !selected + } + fmt.Fprintf(&mask, "i1 %t", selected) + } + mask.WriteByte('>') + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %s, %s %%%s, %s %s\n", result, lanes, mask.String(), llvmType, negated, llvmType, addend) + return "%" + result +} + +func amd64FMA3IntrinsicName(lanes, laneBits int, constrained bool) string { + prefix := "llvm.fma." + if constrained { + prefix = "llvm.experimental.constrained.fma." + } + suffix := "f32" + if laneBits == 16 { + suffix = "f16" + } else if laneBits == 64 { + suffix = "f64" + } + if lanes > 1 { + suffix = fmt.Sprintf("v%d%s", lanes, suffix) + } + return prefix + suffix +} + +func amd64FMA3RoundingMetadata(rounding string) string { + switch rounding { + case "RD_SAE": + return "round.downward" + case "RU_SAE": + return "round.upward" + case "RZ_SAE": + return "round.towardzero" + default: + return "round.tonearest" + } +} diff --git a/amd64_lower_fma3_round.go b/amd64_lower_fma3_round.go new file mode 100644 index 000000000..fdb46cf6f --- /dev/null +++ b/amd64_lower_fma3_round.go @@ -0,0 +1,289 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// LLVM 22 can lower fixed-rounding constrained FMA to an ambient-rounding +// libcall. Accumulate the exact product and addend with guard/sticky bits +// instead. The product needs at most 106 bits; a 128-bit accumulator retains +// every cancellation bit and enough low bits for one final IEEE rounding. +type amd64FMA3IntegerEmitter struct { + b *strings.Builder + next int +} + +func (e *amd64FMA3IntegerEmitter) value(format string, args ...any) string { + name := fmt.Sprintf("%%fma%d", e.next) + e.next++ + fmt.Fprintf(e.b, " %s = %s\n", name, fmt.Sprintf(format, args...)) + return name +} + +type amd64FMA3IntegerInput struct { + negative, magnitude, exponent, finite, floating string +} + +func (e *amd64FMA3IntegerEmitter) input(bits int, value, daz string) amd64FMA3IntegerInput { + fractionBits, bias := 23, 127 + if bits == 64 { + fractionBits, bias = 52, 1023 + } + fractionMask := uint64(1)< 0 || len(mulDivNeeded) > 0 { + b.WriteString("declare i128 @llvm.ctlz.i128(i128, i1)\n") + } + for _, bits := range []int{32, 64} { + if mulDivNeeded[bits] { + emitX86FloatMulDivRoundingHelper(b, bits) + } + if addSubSpecial[bits] { + emitX86FloatAddSubSpecialHelper(b, bits) + } + if needed[bits] { + emitAMD64FMA3RoundingHelper(b, bits) + if addSubNeeded[bits] { + emitX86FloatAddSubRoundingHelper(b, bits) + } + } + } +} + +func emitAMD64FMA3RoundingHelper(b *strings.Builder, bits int) { + fractionBits, bias := 23, 127 + if bits == 64 { + fractionBits, bias = 52, 1023 + } + floatType := amd64FMA3LLVMType(1, bits) + name := fmt.Sprintf("__plan9asm_fma3_round_f%d", bits) + fmt.Fprintf(b, "define internal %s @%s(%s %%a, %s %%b, %s %%c, i32 %%mode, i32 %%mxcsr) {\nentry:\n", floatType, name, floatType, floatType, floatType) + e := amd64FMA3IntegerEmitter{b: b} + daz := e.value("and i32 %%mxcsr, 64") + daz = e.value("icmp ne i32 %s, 0", daz) + a, multiplier, addend := e.input(bits, "%a", daz), e.input(bits, "%b", daz), e.input(bits, "%c", daz) + finite := e.value("and i1 %s, %s", a.finite, multiplier.finite) + finite = e.value("and i1 %s, %s", finite, addend.finite) + fmt.Fprintf(b, " br i1 %s, label %%finite, label %%special\nspecial:\n", finite) + // Preserve the existing target's NaN/infinity behavior without executing + // a floating operation on the finite software path or changing host fenv. + fmt.Fprintf(b, " %%special_result = call %s @llvm.fma.f%d(%s %s, %s %s, %s %s)\n ret %s %%special_result\nfinite:\n", floatType, bits, floatType, a.floating, floatType, multiplier.floating, floatType, addend.floating, floatType) + product := e.value("mul i128 %s, %s", a.magnitude, multiplier.magnitude) + productExponent := e.value("add i32 %s, %s", a.exponent, multiplier.exponent) + productSign := e.value("xor i1 %s, %s", a.negative, multiplier.negative) + productTop := e.value("add i32 %s, %d", productExponent, 2*fractionBits+1) + addendTop := e.value("add i32 %s, %d", addend.exponent, fractionBits) + largerTop := e.value("icmp sgt i32 %s, %s", productTop, addendTop) + top := e.value("select i1 %s, i32 %s, i32 %s", largerTop, productTop, addendTop) + // A zero contribution has no exponent. In particular, a huge operand + // multiplied by zero must not erase a tiny nonzero addend into sticky. + originalTop := top + productZero := e.value("icmp eq i128 %s, 0", product) + addendZero := e.value("icmp eq i128 %s, 0", addend.magnitude) + top = e.value("select i1 %s, i32 %s, i32 %s", productZero, addendTop, top) + top = e.value("select i1 %s, i32 %s, i32 %s", addendZero, productTop, top) + bothZero := e.value("and i1 %s, %s", productZero, addendZero) + top = e.value("select i1 %s, i32 %s, i32 %s", bothZero, originalTop, top) + unit := e.value("sub i32 %s, 126", top) + product = e.align(product, productExponent, unit) + add := e.align(addend.magnitude, addend.exponent, unit) + sameSign := e.value("icmp eq i1 %s, %s", productSign, addend.negative) + productLarger := e.value("icmp uge i128 %s, %s", product, add) + sum := e.value("add i128 %s, %s", product, add) + difference := e.value("sub i128 %s, %s", product, add) + reverseDifference := e.value("sub i128 %s, %s", add, product) + difference = e.value("select i1 %s, i128 %s, i128 %s", productLarger, difference, reverseDifference) + magnitude := e.value("select i1 %s, i128 %s, i128 %s", sameSign, sum, difference) + negative := e.value("select i1 %s, i1 %s, i1 %s", productLarger, productSign, addend.negative) + zero := e.value("icmp eq i128 %s, 0", magnitude) + downward := e.value("icmp eq i32 %%mode, 1") + zeroSign := e.value("select i1 %s, i1 %s, i1 %s", sameSign, productSign, downward) + negative = e.value("select i1 %s, i1 %s, i1 %s", zero, zeroSign, negative) + leading := e.value("call i128 @llvm.ctlz.i128(i128 %s, i1 false)", magnitude) + leading = e.value("trunc i128 %s to i32", leading) + highest := e.value("sub i32 127, %s", leading) + normalShift := e.value("sub i32 %s, %d", highest, fractionBits) + subnormalShift := e.value("sub i32 %d, %s", 1-bias-fractionBits, unit) + isSubnormal := e.value("icmp sgt i32 %s, %s", subnormalShift, normalShift) + shift := e.value("select i1 %s, i32 %s, i32 %s", isSubnormal, subnormalShift, normalShift) + positiveShift := e.value("icmp sgt i32 %s, 0", shift) + far := e.value("icmp sge i32 %s, 128", shift) + safeRight := e.value("select i1 %s, i32 %s, i32 0", positiveShift, shift) + safeRight = e.value("select i1 %s, i32 127, i32 %s", far, safeRight) + safeRight = e.value("zext i32 %s to i128", safeRight) + right := e.value("lshr i128 %s, %s", magnitude, safeRight) + right = e.value("select i1 %s, i128 0, i128 %s", far, right) + leftCount := e.value("sub i32 0, %s", shift) + leftCount = e.value("select i1 %s, i32 0, i32 %s", positiveShift, leftCount) + leftCount = e.value("zext i32 %s to i128", leftCount) + left := e.value("shl i128 %s, %s", magnitude, leftCount) + unrounded := e.value("select i1 %s, i128 %s, i128 %s", positiveShift, right, left) + mask := e.value("shl i128 1, %s", safeRight) + mask = e.value("sub i128 %s, 1", mask) + remainder := e.value("and i128 %s, %s", magnitude, mask) + remainder = e.value("select i1 %s, i128 %s, i128 %s", far, magnitude, remainder) + inexact := e.value("icmp ne i128 %s, 0", remainder) + halfCount := e.value("sub i32 %s, 1", shift) + halfCount = e.value("select i1 %s, i32 %s, i32 0", positiveShift, halfCount) + halfFar := e.value("icmp ugt i32 %s, 127", halfCount) + halfCount = e.value("select i1 %s, i32 127, i32 %s", halfFar, halfCount) + halfCount = e.value("zext i32 %s to i128", halfCount) + half := e.value("shl i128 1, %s", halfCount) + above := e.value("icmp ugt i128 %s, %s", remainder, half) + tied := e.value("icmp eq i128 %s, %s", remainder, half) + odd := e.value("and i128 %s, 1", unrounded) + odd = e.value("icmp ne i128 %s, 0", odd) + tied = e.value("and i1 %s, %s", tied, odd) + nearest := e.value("or i1 %s, %s", above, tied) + validHalf := e.value("xor i1 %s, true", halfFar) + validHalf = e.value("and i1 %s, %s", validHalf, positiveShift) + nearest = e.value("and i1 %s, %s", nearest, validHalf) + upward := e.value("icmp eq i32 %%mode, 2") + positive := e.value("xor i1 %s, true", negative) + directed := e.value("and i1 %s, %s", downward, negative) + upwardPositive := e.value("and i1 %s, %s", upward, positive) + directed = e.value("or i1 %s, %s", directed, upwardPositive) + awayFromZero := directed + directed = e.value("and i1 %s, %s", directed, inexact) + isNearest := e.value("icmp eq i32 %%mode, 0") + increment := e.value("select i1 %s, i1 %s, i1 %s", isNearest, nearest, directed) + increment = e.value("zext i1 %s to i128", increment) + rounded := e.value("add i128 %s, %s", unrounded, increment) + carry := e.value("icmp uge i128 %s, %d", rounded, uint64(1)<<(fractionBits+1)) + carried := e.value("lshr i128 %s, 1", rounded) + rounded = e.value("select i1 %s, i128 %s, i128 %s", carry, carried, rounded) + carry32 := e.value("zext i1 %s to i32", carry) + exponent := e.value("add i32 %s, %s", unit, shift) + exponent = e.value("add i32 %s, %d", exponent, fractionBits+bias) + exponent = e.value("add i32 %s, %s", exponent, carry32) + stillSubnormal := e.value("icmp ult i128 %s, %d", rounded, uint64(1)< 1 { + for index, value := range values { + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, vectorType, value, lane) + values[index] = "%" + extracted + } + } + computed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @__plan9asm_fma3_round_f%d(%s %s, %s %s, %s %s, i32 %s, i32 %s)\n", computed, floatType, bits, floatType, values[0], floatType, values[1], floatType, values[2], mode, mxcsr) + if lanes == 1 { + return "%" + computed + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %%%s, i32 %d\n", inserted, vectorType, result, floatType, computed, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_four_source.go b/amd64_lower_four_source.go new file mode 100644 index 000000000..416f66b9f --- /dev/null +++ b/amd64_lower_four_source.go @@ -0,0 +1,343 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FourSourceMode uint8 + +const ( + amd64FourSourceFMA amd64FourSourceMode = iota + amd64FourSourceDotWords +) + +type amd64FourSourceSpec struct { + mode amd64FourSourceMode + scalar bool + negative bool + saturating bool +} + +// amd64FourSourceSpecs is the complete Go 1.27 AVX512_4FMAPS/AVX512_4VNNIW +// grammar. The opcode table records only semantic axes; memory, a consecutive +// four-register range, optional K mask, zeroing, and architecture rules are +// parsed once below. +var amd64FourSourceSpecs = map[Op]amd64FourSourceSpec{ + "V4FMADDPS": {mode: amd64FourSourceFMA}, + "V4FMADDSS": {mode: amd64FourSourceFMA, scalar: true}, + "V4FNMADDPS": {mode: amd64FourSourceFMA, negative: true}, + "V4FNMADDSS": {mode: amd64FourSourceFMA, scalar: true, negative: true}, + "VP4DPWSSD": {mode: amd64FourSourceDotWords}, + "VP4DPWSSDS": {mode: amd64FourSourceDotWords, saturating: true}, +} + +type amd64FourSourceForm struct { + memory Operand + sourceBase int + destination Operand + mask string + zeroing bool +} + +func (c *amd64Ctx) lowerFourSource(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FourSourceSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + form, err := c.parseFourSourceForm(baseOp, suffix, spec, ins) + if err != nil { + return true, false, err + } + if spec.mode == amd64FourSourceFMA { + return c.lowerFourSourceFMA(spec, form) + } + return c.lowerFourSourceDotWords(spec, form) +} + +func (c *amd64Ctx) parseFourSourceForm(baseOp, suffix string, spec amd64FourSourceSpec, ins Instr) (amd64FourSourceForm, error) { + var form amd64FourSourceForm + if suffix != "" && suffix != "Z" { + return form, fmt.Errorf("%s %s has a suffix absent from Go 1.27's four-source tables: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return form, fmt.Errorf("%s %s expects memory, four-register range, [K1-K7,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return form, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if suffix == "Z" && !masked { + return form, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if !isAMD64MemoryOperand(ins.Args[0]) { + return form, fmt.Errorf("%s %s first source must be 16-byte memory: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpMem && !x86MemoryRegistersValidForArch(ins.Args[0].Mem, c.goarch) { + return form, fmt.Errorf("%s %s memory source uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + + width := 64 + if spec.scalar { + width = 16 + } + rangeOperand := ins.Args[1] + if rangeOperand.Kind != OpRegList || !rangeOperand.RegListRange || len(rangeOperand.RegList) != 4 { + return form, fmt.Errorf("%s %s second source must be one consecutive four-register range: %q", c.goarch, baseOp, ins.Raw) + } + first := -1 + for index, register := range rangeOperand.RegList { + registerIndex, valid := amd64VectorRegisterIndex(register, width) + if !valid || c.goarch == "386" && registerIndex >= 8 || index != 0 && registerIndex != first+index { + return form, fmt.Errorf("%s %s source range is outside Go 1.27's four-register class: %q", c.goarch, baseOp, ins.Raw) + } + if index == 0 { + first = registerIndex + } + } + + destination := ins.Args[len(ins.Args)-1] + destinationIndex, valid := amd64VectorRegisterIndex(destination.Reg, width) + if destination.Kind != OpReg || !valid || c.goarch == "386" && destinationIndex >= 8 { + return form, fmt.Errorf("%s %s destination must be an in-range %s register: %q", c.goarch, baseOp, map[bool]string{true: "X", false: "Z"}[spec.scalar], ins.Raw) + } + mask := "" + if masked { + if !amd64NonzeroKOperand(ins.Args[2]) { + return form, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + loadedMask, loadErr := c.loadK(ins.Args[2].Reg) + if loadErr != nil { + return form, loadErr + } + mask = loadedMask + } + + // The hardware ignores the low two encoded source-register bits. Preserve + // that behavior for Go spellings such as [Z10-Z13], which the assembler + // accepts even though the architectural block begins at Z8. + return amd64FourSourceForm{ + memory: ins.Args[0], sourceBase: first &^ 3, destination: destination, + mask: mask, zeroing: suffix == "Z", + }, nil +} + +func (c *amd64Ctx) lowerFourSourceFMA(spec amd64FourSourceSpec, form amd64FourSourceForm) (bool, bool, error) { + activeMaskBits := 16 + if spec.scalar { + activeMaskBits = 1 + } + memoryBits, err := c.loadFourSourceMemoryI32(form, activeMaskBits) + if err != nil { + return true, false, err + } + memoryName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %s to <4 x float>\n", memoryName, memoryBits) + memory := "%" + memoryName + if spec.scalar { + return c.lowerFourSourceScalarFMA(spec, form, memory) + } + + accumulator, err := c.loadFloatingVectorOperand(form.destination, 64, 16, 32) + if err != nil { + return true, false, err + } + original := accumulator + for sourceIndex := 0; sourceIndex < 4; sourceIndex++ { + sourceReg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Z%d", form.sourceBase+sourceIndex))} + source, loadErr := c.loadFloatingVectorOperand(sourceReg, 64, 16, 32) + if loadErr != nil { + return true, false, loadErr + } + factor := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x float> %s, i32 %d\n", factor, memory, sourceIndex) + factors := c.splatFMA3Scalar(16, 32, "%"+factor) + if spec.negative { + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg <16 x float> %s\n", negated, source) + source = "%" + negated + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <16 x float> @llvm.fma.v16f32(<16 x float> %s, <16 x float> %s, <16 x float> %s)\n", result, source, factors, accumulator) + accumulator = "%" + result + } + if form.mask != "" { + computedBits := c.newTmp() + originalBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x float> %s to <16 x i32>\n", computedBits, accumulator) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x float> %s to <16 x i32>\n", originalBits, original) + masked := amd64ApplyI32LaneMask(c, 16, "%"+computedBits, "%"+originalBits, form.mask, form.zeroing) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i32> %s to <16 x float>\n", back, masked) + accumulator = "%" + back + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x float> %s to <64 x i8>\n", out, accumulator) + return true, false, c.storeZ(form.destination.Reg, "%"+out) +} + +func (c *amd64Ctx) lowerFourSourceScalarFMA(spec amd64FourSourceSpec, form amd64FourSourceForm, memory string) (bool, bool, error) { + accumulator, err := c.loadXLowF32(form.destination.Reg) + if err != nil { + return true, false, err + } + original := accumulator + for sourceIndex := 0; sourceIndex < 4; sourceIndex++ { + sourceReg := Reg(fmt.Sprintf("X%d", form.sourceBase+sourceIndex)) + source, loadErr := c.loadXLowF32(sourceReg) + if loadErr != nil { + return true, false, loadErr + } + if spec.negative { + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg float %s\n", negated, source) + source = "%" + negated + } + factor := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x float> %s, i32 %d\n", factor, memory, sourceIndex) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call float @llvm.fma.f32(float %s, float %%%s, float %s)\n", result, source, factor, accumulator) + accumulator = "%" + result + } + if form.mask != "" { + bit := amd64MaskBitI1(c, form.mask, 0) + fallback := original + if form.zeroing { + fallback = "0.000000e+00" + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, float %s, float %s\n", selected, bit, accumulator, fallback) + accumulator = "%" + selected + } + destination, err := c.loadX(form.destination.Reg) + if err != nil { + return true, false, err + } + words := c.newTmp() + bits := c.newTmp() + inserted := c.newTmp() + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, destination) + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", bits, accumulator) + fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> %%%s, i32 %%%s, i32 0\n", inserted, words, bits) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, inserted) + return true, false, c.storeX(form.destination.Reg, "%"+out) +} + +func (c *amd64Ctx) lowerFourSourceDotWords(spec amd64FourSourceSpec, form amd64FourSourceForm) (bool, bool, error) { + memoryDwords, err := c.loadFourSourceMemoryI32(form, 16) + if err != nil { + return true, false, err + } + memoryName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %s to <8 x i16>\n", memoryName, memoryDwords) + memory := "%" + memoryName + destinationBytes, err := c.loadZ(form.destination.Reg) + if err != nil { + return true, false, err + } + original := c.bitcastVectorBytesToIntegerLanes(64, 16, 32, destinationBytes) + accumulator := original + for sourceIndex := 0; sourceIndex < 4; sourceIndex++ { + sourceReg := Reg(fmt.Sprintf("Z%d", form.sourceBase+sourceIndex)) + sourceBytes, loadErr := c.loadZ(sourceReg) + if loadErr != nil { + return true, false, loadErr + } + source := c.bitcastVectorBytesToIntegerLanes(64, 32, 16, sourceBytes) + accumulator = c.emitFourSourceDotStep(accumulator, source, memory, sourceIndex, spec.saturating) + } + if form.mask != "" { + accumulator = amd64ApplyI32LaneMask(c, 16, accumulator, original, form.mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i32> %s to <64 x i8>\n", out, accumulator) + return true, false, c.storeZ(form.destination.Reg, "%"+out) +} + +func (c *amd64Ctx) loadFourSourceMemoryI32(form amd64FourSourceForm, activeMaskBits int) (string, error) { + if form.mask == "" { + bytes, err := c.loadPackedCompareBytes(form.memory, 16) + if err != nil { + return "", err + } + return c.bitcastVectorBytesToIntegerLanes(16, 4, 32, bytes), nil + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", masked, form.mask, (uint64(1)< poison, i1 %%%s, i32 0\n", seed, active) + fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i1> %%%s, <4 x i1> poison, <4 x i32> zeroinitializer\n", predicate, seed) + pointer, pointerType, intrinsicSuffix, err := c.packedVectorMemoryPointer(form.memory) + if err != nil { + return "", err + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x i32> @llvm.masked.load.v4i32%s(%s align 1 %s, <4 x i1> %%%s, <4 x i32> zeroinitializer)\n", + loaded, intrinsicSuffix, pointerType, pointer, predicate) + return "%" + loaded, nil +} + +func (c *amd64Ctx) emitFourSourceDotStep(accumulator, source, memory string, sourceIndex int, saturating bool) string { + result := "poison" + for lane := 0; lane < 16; lane++ { + accumulatorLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <16 x i32> %s, i32 %d\n", accumulatorLane, accumulator, lane) + total := "" + if saturating { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i32 %%%s to i64\n", wide, accumulatorLane) + total = "%" + wide + } else { + total = "%" + accumulatorLane + } + for word := 0; word < 2; word++ { + sourceLane := c.newTmp() + memoryLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <32 x i16> %s, i32 %d\n", sourceLane, source, lane*2+word) + fmt.Fprintf(c.b, " %%%s = extractelement <8 x i16> %s, i32 %d\n", memoryLane, memory, sourceIndex*2+word) + wideSource := c.newTmp() + wideMemory := c.newTmp() + bits := 32 + if saturating { + bits = 64 + } + fmt.Fprintf(c.b, " %%%s = sext i16 %%%s to i%d\n", wideSource, sourceLane, bits) + fmt.Fprintf(c.b, " %%%s = sext i16 %%%s to i%d\n", wideMemory, memoryLane, bits) + product := c.newTmp() + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i%d %%%s, %%%s\n", product, bits, wideSource, wideMemory) + fmt.Fprintf(c.b, " %%%s = add i%d %s, %%%s\n", next, bits, total, product) + total = "%" + next + } + if saturating { + total = c.saturateFourSourceI64ToI32(total) + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <16 x i32> %s, i32 %s, i32 %d\n", inserted, result, total, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) saturateFourSourceI64ToI32(value string) string { + above := c.newTmp() + capped := c.newTmp() + below := c.newTmp() + bounded := c.newTmp() + narrowed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt i64 %s, 2147483647\n", above, value) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 2147483647, i64 %s\n", capped, above, value) + fmt.Fprintf(c.b, " %%%s = icmp slt i64 %%%s, -2147483648\n", below, capped) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 -2147483648, i64 %%%s\n", bounded, below, capped) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", narrowed, bounded) + return "%" + narrowed +} diff --git a/amd64_lower_four_source_test.go b/amd64_lower_four_source_test.go new file mode 100644 index 000000000..0448085af --- /dev/null +++ b/amd64_lower_four_source_test.go @@ -0,0 +1,242 @@ +package plan9asm + +import ( + "runtime" + "sort" + "strings" + "testing" +) + +func TestAMD64FourSourceGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64FourSourceSpecs)) + for op := range amd64FourSourceSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{"V4FMADDPS", "V4FMADDSS", "V4FNMADDPS", "V4FNMADDSS", "VP4DPWSSD", "VP4DPWSSDS"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("four-source grammar opcodes = %v, want %v", got, want) + } +} + +func TestTranslateX86FourSourceCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT foursourceforms(SB),$0-0 + V4FMADDPS (BX), [Z0-Z3], Z4 + V4FNMADDPS 32(BX), [Z0-Z3], Z5 + V4FMADDSS 64(BX), [X0-X3], X4 + V4FNMADDSS 96(BX), [X0-X3], X5 + VP4DPWSSD 128(BX), [Z0-Z3], Z6 + VP4DPWSSDS 160(BX), [Z0-Z3], Z7 +` + if target.goarch == "amd64" { + source += ` V4FMADDPS 8(BX), [Z2-Z5], K7, Z4 + V4FMADDPS.Z 16(BX), [Z2-Z5], K1, Z4 + V4FNMADDPS 40(BX), [Z2-Z5], K7, Z5 + V4FNMADDPS.Z 48(BX), [Z2-Z5], K2, Z5 + V4FMADDSS 72(BX), [X2-X5], K7, X4 + V4FMADDSS.Z 80(BX), [X2-X5], K3, X4 + V4FNMADDSS 104(BX), [X2-X5], K7, X5 + V4FNMADDSS.Z 112(BX), [X2-X5], K4, X5 + VP4DPWSSD 136(BX), [Z2-Z5], K7, Z6 + VP4DPWSSD.Z 144(BX), [Z2-Z5], K5, Z6 + VP4DPWSSDS 168(BX), [Z2-Z5], K7, Z7 + VP4DPWSSDS.Z 176(BX), [Z2-Z5], K6, Z7 +` + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"foursourceforms": {Name: "foursourceforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "four-source-"+target.name+".ll", "four-source-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FourSourceRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "V4FMADDPS Z0, [Z0-Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [Z0-Z2], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [Z0,Z1,Z2,Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [X0-X3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS (AX), [Z0-Z3], K0, Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS.Z (AX), [Z0-Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDPS.BCST (AX), [Z0-Z3], Z4"}, + {goarch: "amd64", instruction: "V4FMADDSS (AX), [X0-X3], Z4"}, + {goarch: "amd64", instruction: "VP4DPWSSD (AX), [Z0-Z3], X4"}, + {goarch: "386", instruction: "VP4DPWSSD (AX), [Z6-Z9], Z4"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's four-source tables", test.instruction) + } + }) + } +} + +func TestAMD64FourSourceRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT foursourcesemantics(SB),NOSPLIT,$0-24 + MOVQ in+0(FP), AX + MOVQ out+8(FP), BX + MOVQ mask+16(FP), CX + VBROADCASTSS 0(AX), Z0 + VBROADCASTSS 16(AX), Z1 + VBROADCASTSS 32(AX), Z2 + VBROADCASTSS 48(AX), Z3 + VBROADCASTSS 64(AX), Z4 + V4FMADDPS 128(AX), [Z2-Z5], Z4 + VMOVUPS Z4, 0(BX) + VBROADCASTSS 64(AX), Z5 + KMOVQ CX, K1 + V4FNMADDPS.Z 128(AX), [Z0-Z3], K1, Z5 + VMOVUPS Z5, 64(BX) + MOVUPS 0(AX), X0 + MOVUPS 16(AX), X1 + MOVUPS 32(AX), X2 + MOVUPS 48(AX), X3 + MOVUPS 64(AX), X6 + V4FMADDSS 128(AX), [X0-X3], X6 + MOVUPS X6, 128(BX) + VMOVDQU64 192(AX), Z0 + VMOVDQU64 256(AX), Z1 + VMOVDQU64 320(AX), Z2 + VMOVDQU64 384(AX), Z3 + VMOVDQU64 448(AX), Z7 + VP4DPWSSD 144(AX), [Z0-Z3], Z7 + VMOVDQU64 Z7, 144(BX) + VMOVDQU64 448(AX), Z7 + VP4DPWSSDS 144(AX), [Z0-Z3], Z7 + VMOVDQU64 Z7, 208(BX) + RET + +TEXT foursourcefaultsuppression(SB),NOSPLIT,$0-24 + MOVQ memory+0(FP), AX + MOVQ out+8(FP), BX + MOVQ mask+16(FP), CX + KMOVQ CX, K1 + V4FMADDPS.Z (AX), [Z0-Z3], K1, Z4 + VMOVUPS Z4, 0(BX) + V4FMADDSS.Z (AX), [X0-X3], K1, X4 + MOVUPS X4, 64(BX) + VP4DPWSSD.Z (AX), [Z0-Z3], K1, Z5 + VMOVUPS Z5, 80(BX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{ + "foursourcesemantics": { + Name: "foursourcesemantics", Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + "foursourcefaultsuppression": { + Name: "foursourcefaultsuppression", Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void foursourcesemantics(void *, void *, uint64_t); +extern void foursourcefaultsuppression(void *, void *, uint64_t); +static void putf(uint8_t *p, float v) { memcpy(p, &v, 4); } +static float getf(const uint8_t *p) { float v; memcpy(&v, p, 4); return v; } +static void put16(uint8_t *p, int16_t v) { memcpy(p, &v, 2); } +static void put32(uint8_t *p, int32_t v) { memcpy(p, &v, 4); } +static int32_t get32(const uint8_t *p) { int32_t v; memcpy(&v, p, 4); return v; } +int main(void) { + uint8_t in[512] = {0}, out[272] = {0}, faultOut[144] = {1}; + for (int r=0;r<4;r++) for (int lane=0;lane<4;lane++) putf(in+r*16+lane*4, (float)(r+1)); + for (int lane=0;lane<4;lane++) putf(in+64+lane*4, 10.0f+(float)lane); + putf(in+128, 1); putf(in+132, 2); putf(in+136, 3); putf(in+140, 4); + for (int r=0;r<4;r++) for (int lane=0;lane<32;lane++) put16(in+192+r*64+lane*2, (int16_t)(r+1)); + put16(in+144, 1); put16(in+146, 2); put16(in+148, 3); put16(in+150, 4); + put16(in+152, 5); put16(in+154, 6); put16(in+156, 7); put16(in+158, 8); + for (int lane=0;lane<16;lane++) put32(in+448+lane*4, lane==0 ? INT_MAX-5 : lane); + foursourcesemantics(in, out, 5); + foursourcefaultsuppression(0, faultOut, 0); + if (getf(out) != 40 || getf(out+4) != 40 || getf(out+8) != 40 || getf(out+12) != 40) return 10; + if (getf(out+64) != -20) return 21; + if (getf(out+68) != 0) return 22; + if (getf(out+72) != -20) return 23; + if (getf(out+76) != 0) return 24; + if (getf(out+128) != 40 || getf(out+132) != 11 || getf(out+136) != 12 || getf(out+140) != 13) return 12; + if (get32(out+144) != INT_MIN+104 || get32(out+148) != 111) return 13; + if (get32(out+208) != INT_MAX || get32(out+212) != 111) return 14; + for (int i=0;i<144;i++) if (faultOut[i] != 0) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "four_source_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_fp.go b/amd64_lower_fp.go index 58f7fdf0c..a363f4f28 100644 --- a/amd64_lower_fp.go +++ b/amd64_lower_fp.go @@ -2,55 +2,346 @@ package plan9asm import ( "fmt" + "math" "strconv" "strings" ) func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, term, err := c.lowerLegacyPackedMove(op, ins); known { + return known, term, err + } + if ok, term, err := c.lowerSignedIntegerScalarFloat(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFloatMulDiv(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFloatAddSub(op, ins); ok { + return ok, term, err + } + if spec, recognized := amd64ScalarVectorMoveSpecs[op]; recognized && spec.legacy { + if _, _, err := amd64ScalarVectorMoveProperties(ins, c.goarch); err != nil { + return true, false, err + } + } + if ok, term, err := c.lowerApproximateMath(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFixupImmediate(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerScaledRound(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerGetMant(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerGetExp(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFPClass(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFloatingRange(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerReciprocal14(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerReciprocalEstimate(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerRound(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerScalarFloatToInteger(op, ins); ok { + return ok, term, err + } + if raw := strings.ToUpper(string(op)); raw != "" { + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + if _, recognized := amd64HorizontalFloatingSpecs[Op(base)]; recognized { + return c.lowerHorizontalFloating(op, ins) + } + } + if raw := strings.ToUpper(string(op)); raw != "" { + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + if _, recognized := amd64BinaryFloatingSpecs[Op(base)]; recognized { + return c.lowerBinaryFloating(op, ins) + } + } + if raw := strings.ToUpper(string(op)); raw != "" { + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + if _, recognized := amd64FMA3Specs[Op(base)]; recognized { + return c.lowerFMA3(op, ins) + } + } + if raw := strings.ToUpper(string(op)); raw != "" { + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + if _, recognized := x86PackedLogicalSpecs[Op(base)]; recognized { + return c.lowerPackedFloatingLogical(op, ins) + } + } + if raw := strings.ToUpper(string(op)); raw != "" { + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + if base == "VCVTSS2SD" || base == "VCVTSD2SS" { + return c.lowerVEXScalarFloatConvert(op, ins) + } + } switch op { - case "MOVSD", "MOVAPD", "ANDPD", "ANDNPD", "ORPD", "XORPS", - "ADDSD", "SUBSD", "MULSD", "DIVSD", "MAXSD", "MINSD", "SQRTSD", - "COMISD", "CMPSD", "VADDSD", "VFMADD213SD", "VFNMADD231SD", - "CVTSD2SL", "CVTSL2SD", "CVTSQ2SD", "CVTTSD2SQ": + case "MOVSS", "MOVSD", "MOVAPD", "MOVUPD", + "MULSS", "DIVSS", "MAXSS", "MINSS", "SQRTSS", + "MULPS", "DIVPS", "MAXPS", "MINPS", + "MULPD", "DIVPD", "MAXPD", "MINPD", + "SQRTPS", "SQRTPD", + "MOVLHPS", "MOVHLPS", "MOVLPS", "MOVHPS", "MOVLPD", "MOVHPD", + "VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD", "SHUFPD", "UNPCKLPS", "UNPCKHPD", + "MULSD", "DIVSD", "MAXSD", "MINSD", "SQRTSD", + "COMISS", "UCOMISS", "COMISD", "UCOMISD", "CMPSD", + "CVTSS2SD", "VCVTSS2SD", "CVTSD2SS", "VCVTSD2SS", "CVTSL2SD", "CVTSQ2SD": // handled below default: return false, false, nil } switch op { - case "MOVAPD", "ANDPD", "ANDNPD", "ORPD", "XORPS": + case "VCVTSS2SD": + return c.lowerVEXScalarFloatConvert(op, ins) + + case "VCVTSD2SS": + return c.lowerVEXScalarFloatConvert(op, ins) + + case "MOVSS": + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("amd64 MOVSS expects src, dst: %q", ins.Raw) + } + v, err := c.evalF32(ins.Args[0]) + if err != nil { + return true, false, err + } + switch ins.Args[1].Kind { + case OpReg: + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return true, false, fmt.Errorf("amd64 MOVSS expects X-reg destination: %q", ins.Raw) + } + return true, false, c.storeLegacyScalarMoveRegister(ins.Args[1].Reg, v, 32, ins.Args[0].Kind == OpReg) + case OpFP: + return true, false, c.storeScalarFPFloat(ins.Args[1].FPOffset, 32, v) + case OpMem: + addr, err := c.addrFromMem(ins.Args[1].Mem) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store float %s, ptr %s, align 1\n", v, c.ptrFromAddrI64(addr)) + return true, false, nil + case OpSym: + p, err := c.ptrFromSB(ins.Args[1].Sym) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store float %s, ptr %s, align 1\n", v, p) + return true, false, nil + default: + return true, false, fmt.Errorf("amd64 MOVSS unsupported destination: %q", ins.Raw) + } + + case "MULSS", "DIVSS", "MAXSS", "MINSS": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } - dst := ins.Args[1].Reg - srcv, err := c.loadXVecOperand(ins.Args[0]) + src, err := c.evalF32(ins.Args[0]) if err != nil { return true, false, err } - if op == "MOVAPD" { - return true, false, c.storeX(dst, srcv) - } - dstv, err := c.loadX(dst) + dst, err := c.loadXLowF32(ins.Args[1].Reg) if err != nil { return true, false, err } - t := c.newTmp() + value := c.newTmp() switch op { - case "ANDPD": - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %s, %s\n", t, dstv, srcv) - case "ORPD": - fmt.Fprintf(c.b, " %%%s = or <16 x i8> %s, %s\n", t, dstv, srcv) - case "XORPS": - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, dstv, srcv) - case "ANDNPD": - notv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", notv, dstv, llvmAllOnesI8Vec(16)) - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %%%s, %s\n", t, notv, srcv) - } - return true, false, c.storeX(dst, "%"+t) + case "MULSS": + fmt.Fprintf(c.b, " %%%s = fmul float %s, %s\n", value, dst, src) + case "DIVSS": + fmt.Fprintf(c.b, " %%%s = fdiv float %s, %s\n", value, dst, src) + case "MAXSS", "MINSS": + cmp := c.newTmp() + pred := "ogt" + if op == "MINSS" { + pred = "olt" + } + fmt.Fprintf(c.b, " %%%s = fcmp %s float %s, %s\n", cmp, pred, dst, src) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, float %s, float %s\n", value, cmp, dst, src) + } + return true, false, c.storeXLowF32(ins.Args[1].Reg, "%"+value) + + case "SQRTSS": + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 SQRTSS expects src, Xdst: %q", ins.Raw) + } + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return false, false, nil + } + src, err := c.evalF32(ins.Args[0]) + if err != nil { + return true, false, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call float @llvm.sqrt.f32(float %s)\n", value, src) + return true, false, c.storeXLowF32(ins.Args[1].Reg, "%"+value) + + case "MULPS", "DIVPS", "MAXPS", "MINPS", + "MULPD", "DIVPD", "MAXPD", "MINPD": + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) + } + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return false, false, nil + } + vectorType := "<4 x float>" + if strings.HasSuffix(string(op), "PD") { + vectorType = "<2 x double>" + } + src, err := c.loadXTypedVectorOperand(ins.Args[0], vectorType) + if err != nil { + return true, false, err + } + dst, err := c.loadXTypedVectorOperand(ins.Args[1], vectorType) + if err != nil { + return true, false, err + } + value := c.newTmp() + switch { + case strings.HasPrefix(string(op), "MUL"): + fmt.Fprintf(c.b, " %%%s = fmul %s %s, %s\n", value, vectorType, dst, src) + case strings.HasPrefix(string(op), "DIV"): + fmt.Fprintf(c.b, " %%%s = fdiv %s %s, %s\n", value, vectorType, dst, src) + default: + cmp := c.newTmp() + pred := "ogt" + if strings.HasPrefix(string(op), "MIN") { + pred = "olt" + } + lanes := 4 + if vectorType == "<2 x double>" { + lanes = 2 + } + fmt.Fprintf(c.b, " %%%s = fcmp %s %s %s, %s\n", cmp, pred, vectorType, dst, src) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", value, lanes, cmp, vectorType, dst, vectorType, src) + } + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytesValue, vectorType, value) + return true, false, c.storeX(ins.Args[1].Reg, "%"+bytesValue) + + case "SQRTPS", "SQRTPD": + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) + } + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return false, false, nil + } + vectorType := "<4 x float>" + intrinsic := "@llvm.sqrt.v4f32" + if op == "SQRTPD" { + vectorType = "<2 x double>" + intrinsic = "@llvm.sqrt.v2f64" + } + src, err := c.loadXTypedVectorOperand(ins.Args[0], vectorType) + if err != nil { + return true, false, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s %s(%s %s)\n", value, vectorType, intrinsic, vectorType, src) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytesValue, vectorType, value) + return true, false, c.storeX(ins.Args[1].Reg, "%"+bytesValue) + + case "MOVLHPS", "MOVHLPS": + if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) + } + src, err := c.loadXAsI64x2(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + dst, err := c.loadXAsI64x2(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + mask := "" + if op == "MOVHLPS" { + mask = "" + } + shuffled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <2 x i32> %s\n", shuffled, dst, src, mask) + return true, false, c.storeXFromI64x2(ins.Args[1].Reg, "%"+shuffled) + + case "MOVLPS", "MOVHPS", "MOVLPD", "MOVHPD", "VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD": + return c.lowerMOVHalf(op, ins) + + case "SHUFPD": + if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 SHUFPD expects $imm, Xsrc, Xdst: %q", ins.Raw) + } + src, err := c.loadXAsI64x2(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + dst, err := c.loadXAsI64x2(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + lo := ins.Args[0].Imm & 1 + hi := 2 + ((ins.Args[0].Imm >> 1) & 1) + shuffled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <2 x i32> \n", shuffled, dst, src, lo, hi) + return true, false, c.storeXFromI64x2(ins.Args[2].Reg, "%"+shuffled) + + case "UNPCKLPS", "UNPCKHPD": + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) + } + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return false, false, nil + } + vectorType := "<4 x float>" + maskType := "<4 x i32>" + mask := "" + if op == "UNPCKHPD" { + vectorType = "<2 x double>" + maskType = "<2 x i32>" + mask = "" + } + src, err := c.loadXTypedVectorOperand(ins.Args[0], vectorType) + if err != nil { + return true, false, err + } + dst, err := c.loadXTypedVectorOperand(ins.Args[1], vectorType) + if err != nil { + return true, false, err + } + shuffled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector %s %s, %s %s, %s %s\n", shuffled, vectorType, dst, vectorType, src, maskType, mask) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytesValue, vectorType, shuffled) + return true, false, c.storeX(ins.Args[1].Reg, "%"+bytesValue) case "MOVSD": if len(ins.Args) != 2 { @@ -65,9 +356,9 @@ func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err erro if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return true, false, fmt.Errorf("amd64 MOVSD expects X-reg destination: %q", ins.Raw) } - return true, false, c.storeXLowF64(ins.Args[1].Reg, v) + return true, false, c.storeLegacyScalarMoveRegister(ins.Args[1].Reg, v, 64, ins.Args[0].Kind == OpReg) case OpFP: - return true, false, c.storeFPResult(ins.Args[1].FPOffset, LLVMType("double"), v) + return true, false, c.storeScalarFPFloat(ins.Args[1].FPOffset, 64, v) case OpMem: addr, err := c.addrFromMem(ins.Args[1].Mem) if err != nil { @@ -87,7 +378,7 @@ func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err erro return true, false, fmt.Errorf("amd64 MOVSD unsupported destination: %q", ins.Raw) } - case "ADDSD", "SUBSD", "MULSD", "DIVSD", "MAXSD", "MINSD": + case "MULSD", "DIVSD", "MAXSD", "MINSD": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { return true, false, fmt.Errorf("amd64 %s expects src, Xdst: %q", op, ins.Raw) } @@ -104,10 +395,6 @@ func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err erro } t := c.newTmp() switch op { - case "ADDSD": - fmt.Fprintf(c.b, " %%%s = fadd double %s, %s\n", t, dst, src) - case "SUBSD": - fmt.Fprintf(c.b, " %%%s = fsub double %s, %s\n", t, dst, src) case "MULSD": fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", t, dst, src) case "DIVSD": @@ -138,28 +425,8 @@ func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err erro fmt.Fprintf(c.b, " %%%s = call double @llvm.sqrt.f64(double %s)\n", t, src) return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+t) - case "COMISD": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 COMISD expects src, dst: %q", ins.Raw) - } - src, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err - } - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp ueq double %s, %s\n", z, dst, src) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", z, c.flagsZSlot) - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp ult double %s, %s\n", cf, dst, src) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - lt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp olt double %s, %s\n", lt, dst, src) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", lt, c.flagsSltSlot) - return true, false, nil + case "COMISS", "UCOMISS", "COMISD", "UCOMISD": + return c.lowerVectorScalarFlagCompare(op, ins) case "CMPSD": if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg { @@ -213,136 +480,36 @@ func (c *amd64Ctx) lowerFP(op Op, ins Instr) (ok bool, terminated bool, err erro fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 -1, i64 0\n", mask, cmp) return true, false, c.storeXLowI64(ins.Args[1].Reg, "%"+mask) - case "CVTTSD2SQ": + case "CVTSS2SD": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTTSD2SQ expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fptosi double %s to i64\n", t, src) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+t) - - case "CVTSQ2SD": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTSQ2SD expects src, Xdst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 CVTSS2SD expects src, Xdst: %q", ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } - src, err := c.evalI64(ins.Args[0]) + src, err := c.evalF32(ins.Args[0]) if err != nil { return true, false, err } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sitofp i64 %s to double\n", t, src) - return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+t) + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", converted, src) + return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+converted) - case "CVTSD2SL": + case "CVTSD2SS": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTSD2SL expects src, dstReg: %q", ins.Raw) - } - src, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - round := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call double @llvm.rint.f64(double %s)\n", round, src) - i32v := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fptosi double %%%s to i32\n", i32v, round) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, i32v) - return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - - case "CVTSL2SD": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 CVTSL2SD expects srcReg, Xdst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 CVTSD2SS expects src, Xdst: %q", ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } - src, err := c.evalI64(ins.Args[0]) - if err != nil { - return true, false, err - } - i32v := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", i32v, src) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sitofp i32 %%%s to double\n", t, i32v) - return true, false, c.storeXLowF64(ins.Args[1].Reg, "%"+t) - - case "VADDSD": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VADDSD expects src1, src2, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - s1, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - s2, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fadd double %s, %s\n", t, s2, s1) - return true, false, c.storeXLowF64(ins.Args[2].Reg, "%"+t) - - case "VFMADD213SD": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VFMADD213SD expects src1, src2, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - s1, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - s2, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err - } - dst, err := c.loadXLowF64(ins.Args[2].Reg) + src, err := c.evalF64(ins.Args[0]) if err != nil { return true, false, err } - mul := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", mul, dst, s2) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fadd double %%%s, %s\n", t, mul, s1) - return true, false, c.storeXLowF64(ins.Args[2].Reg, "%"+t) + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fptrunc double %s to float\n", converted, src) + return true, false, c.storeXLowF32(ins.Args[1].Reg, "%"+converted) - case "VFNMADD231SD": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VFNMADD231SD expects src1, src2, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - s1, err := c.evalF64(ins.Args[0]) - if err != nil { - return true, false, err - } - s2, err := c.evalF64(ins.Args[1]) - if err != nil { - return true, false, err - } - dst, err := c.loadXLowF64(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - mul := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", mul, s1, s2) - neg := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fneg double %%%s\n", neg, mul) - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fadd double %%%s, %s\n", t, neg, dst) - return true, false, c.storeXLowF64(ins.Args[2].Reg, "%"+t) } return false, false, nil } @@ -371,11 +538,43 @@ func (c *amd64Ctx) loadXVecOperand(op Operand) (string, error) { t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", t, p) return "%" + t, nil + case OpFP: + // ABI0 fields need not be two contiguous uint64 values. Assemble + // every declared byte, including adjacent float32 and narrow fields. + return c.loadFPVectorBytes(op.FPOffset, 16) default: return "", fmt.Errorf("amd64: unsupported X-vector operand %s", op.String()) } } +func (c *amd64Ctx) storeXVecOperand(op Operand, value string) error { + switch op.Kind { + case OpReg: + if !isAMD64XReg(op.Reg) { + return fmt.Errorf("expected X register, got %s", op.String()) + } + return c.storeX(op.Reg, value) + case OpMem: + addr, err := c.addrFromMem(op.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", value, c.ptrFromAddrI64(addr)) + return nil + case OpSym: + p, err := c.ptrFromSB(op.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", value, p) + return nil + case OpFP: + return c.storeFPVectorBytes(op.FPOffset, 16, value, "") + default: + return fmt.Errorf("unsupported X-vector destination %s", op.String()) + } +} + func (c *amd64Ctx) loadYVecOperand(op Operand) (string, error) { switch op.Kind { case OpReg: @@ -456,6 +655,20 @@ func (c *amd64Ctx) loadXLowF64(r Reg) (string, error) { return "%" + t, nil } +func (c *amd64Ctx) loadXLowF32(r Reg) (string, error) { + xv, err := c.loadX(r) + if err != nil { + return "", err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, xv) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 0\n", low, words) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", value, low) + return "%" + value, nil +} + func (c *amd64Ctx) storeXLowI64(r Reg, low string) error { cur, err := c.loadX(r) if err != nil { @@ -480,6 +693,484 @@ func (c *amd64Ctx) storeXLowF64(r Reg, v string) error { return c.storeXLowI64(r, "%"+t) } +// Legacy scalar moves preserve bits above 127 in every physical alias. The +// source class separately determines whether the upper XMM lanes survive. +// Do not use this helper for VEX/EVEX writes, which zero their wider views. +func (c *amd64Ctx) storeLegacyScalarMoveRegister(r Reg, value string, bits int, preserve bool) error { + var store func(Reg, string) error + switch bits { + case 32: + store = c.storeXLowF32ClearingUpper + if preserve { + store = c.storeXLowF32 + } + case 64: + store = c.storeXLowF64ClearingUpper + if preserve { + store = c.storeXLowF64 + } + default: + return fmt.Errorf("invalid legacy scalar move width %d", bits) + } + if err := store(r, value); err != nil { + return err + } + xmm, err := c.loadX(r) + if err != nil { + return err + } + return c.storeLegacyXViews(r, xmm) +} + +func (c *amd64Ctx) storeXLowF64ClearingUpper(r Reg, v string) error { + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", bits, v) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> zeroinitializer, i64 %%%s, i32 0\n", inserted, bits) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", back, inserted) + return c.storeX(r, "%"+back) +} + +func (c *amd64Ctx) storeXLowF32(r Reg, v string) error { + cur, err := c.loadX(r) + if err != nil { + return err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, cur) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", bits, v) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> %%%s, i32 %%%s, i32 0\n", inserted, words, bits) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", back, inserted) + return c.storeX(r, "%"+back) +} + +func (c *amd64Ctx) storeXLowF32ClearingUpper(r Reg, v string) error { + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %s to i32\n", bits, v) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> zeroinitializer, i32 %%%s, i32 0\n", inserted, bits) + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", back, inserted) + return c.storeX(r, "%"+back) +} + +func (c *amd64Ctx) loadXTypedVectorOperand(op Operand, vectorType string) (string, error) { + bytesValue, err := c.loadXVecOperand(op) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", value, bytesValue, vectorType) + return "%" + value, nil +} + +func (c *amd64Ctx) loadXAsI64x2(r Reg) (string, error) { + if _, ok := amd64ParseXReg(r); !ok { + return "", fmt.Errorf("amd64: expected X register, got %s", r) + } + bytesValue, err := c.loadX(r) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", value, bytesValue) + return "%" + value, nil +} + +func (c *amd64Ctx) storeXFromI64x2(r Reg, value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %s to <16 x i8>\n", bytesValue, value) + return c.storeX(r, "%"+bytesValue) +} + +type amd64MoveHalfSpec struct { + high bool + vector bool +} + +// amd64MoveHalfSpecs is the complete Go 1.27 yxmov/_yvmovhpd family. Legacy +// register forms alias MOVHLPS/MOVLHPS; VEX/EVEX forms deliberately accept +// only memory loads (memory, upper-lane X, destination X) and memory stores. +var amd64MoveHalfSpecs = map[Op]amd64MoveHalfSpec{ + "MOVLPS": {}, + "MOVHPS": {high: true}, + "MOVLPD": {}, + "MOVHPD": {high: true}, + "VMOVLPS": {vector: true}, + "VMOVHPS": {high: true, vector: true}, + "VMOVLPD": {vector: true}, + "VMOVHPD": {high: true, vector: true}, +} + +func (c *amd64Ctx) lowerMOVHalf(op Op, ins Instr) (bool, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64MoveHalfSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if spec.vector { + return c.lowerVEXMOVHalf(baseOp, spec, ins) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", baseOp, ins.Raw) + } + destinationLane := int64(0) + if spec.high { + destinationLane = 1 + } + if ins.Args[1].Kind == OpReg { + if !c.isGoLegacyXReg(ins.Args[1].Reg) { + return true, false, fmt.Errorf("amd64 %s destination is outside Go 1.27's X register class: %q", baseOp, ins.Raw) + } + bits := "" + if ins.Args[0].Kind == OpReg { + if !c.isGoLegacyXReg(ins.Args[0].Reg) { + return true, false, fmt.Errorf("amd64 %s source is outside Go 1.27's X register class: %q", baseOp, ins.Raw) + } + source, err := c.loadXAsI64x2(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + // Register encodings of the low/high moves alias MOVHLPS and + // MOVLHPS respectively, regardless of the PS/PD prefix. + sourceLane := int64(0) + if destinationLane == 0 { + sourceLane = 1 + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %s, i32 %d\n", extracted, source, sourceLane) + bits = "%" + extracted + } else { + if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("amd64 %s source must be X or memory: %q", baseOp, ins.Raw) + } + value, err := c.evalF64(ins.Args[0]) + if err != nil { + return true, false, err + } + loadedBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", loadedBits, value) + bits = "%" + loadedBits + } + cur, err := c.loadXAsI64x2(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> %s, i64 %s, i32 %d\n", inserted, cur, bits, destinationLane) + return true, false, c.storeXFromI64x2(ins.Args[1].Reg, "%"+inserted) + } + if ins.Args[0].Kind != OpReg || !c.isGoLegacyXReg(ins.Args[0].Reg) || !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s store expects X source and memory destination: %q", baseOp, ins.Raw) + } + src, err := c.loadXAsI64x2(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %s, i32 %d\n", bits, src, destinationLane) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %%%s to double\n", value, bits) + switch ins.Args[1].Kind { + case OpFP: + return true, false, c.storeFPResult(ins.Args[1].FPOffset, LLVMType("double"), "%"+value) + case OpMem: + addr, err := c.addrFromMem(ins.Args[1].Mem) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store double %%%s, ptr %s, align 1\n", value, c.ptrFromAddrI64(addr)) + return true, false, nil + case OpSym: + p, err := c.ptrFromSB(ins.Args[1].Sym) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store double %%%s, ptr %s, align 1\n", value, p) + return true, false, nil + default: + return true, false, fmt.Errorf("amd64 %s unsupported destination: %q", baseOp, ins.Raw) + } +} + +func (c *amd64Ctx) lowerVEXMOVHalf(baseOp string, spec amd64MoveHalfSpec, ins Instr) (bool, bool, error) { + lane := int64(0) + if spec.high { + lane = 1 + } + if len(ins.Args) == 2 { + if !c.isGoPackedVectorMoveRegister(ins.Args[0], 16) || !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s store expects X source and memory destination: %q", c.goarch, baseOp, ins.Raw) + } + source, err := c.loadXAsI64x2(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %s, i32 %d\n", bits, source, lane) + return true, false, c.storeMOVHalfBits(ins.Args[1], "%"+bits) + } + if len(ins.Args) != 3 || !isAMD64MemoryOperand(ins.Args[0]) || + !c.isGoPackedVectorMoveRegister(ins.Args[1], 16) || !c.isGoPackedVectorMoveRegister(ins.Args[2], 16) { + return true, false, fmt.Errorf("%s %s load expects memory, upper-lane X, destination X: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.evalF64(ins.Args[0]) + if err != nil { + return true, false, err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", bits, value) + base, err := c.loadXAsI64x2(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> %s, i64 %%%s, i32 %d\n", inserted, base, bits, lane) + return true, false, c.storeXFromI64x2(ins.Args[2].Reg, "%"+inserted) +} + +func (c *amd64Ctx) storeMOVHalfBits(destination Operand, bits string) error { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", value, bits) + switch destination.Kind { + case OpFP: + return c.storeFPResult(destination.FPOffset, LLVMType("double"), "%"+value) + case OpMem: + addr, err := c.addrFromMem(destination.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store double %%%s, ptr %s, align 1\n", value, c.ptrFromAddrI64(addr)) + return nil + case OpSym: + p, err := c.ptrFromSB(destination.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store double %%%s, ptr %s, align 1\n", value, p) + return nil + default: + return fmt.Errorf("unsupported half-vector destination %s", destination.String()) + } +} + +func (c *amd64Ctx) lowerVEXScalarFloatConvert(op Op, ins Instr) (bool, bool, error) { + rawOp := strings.ToUpper(string(op)) + parts := strings.Split(rawOp, ".") + baseOp := parts[0] + rounding := "" + sae, zeroing := false, false + for _, suffix := range parts[1:] { + switch suffix { + case "Z": + if zeroing { + return true, false, fmt.Errorf("%s %s repeats .Z: %q", c.goarch, baseOp, ins.Raw) + } + zeroing = true + case "SAE": + if baseOp != "VCVTSS2SD" || sae || rounding != "" { + return true, false, fmt.Errorf("%s %s suffix .%s is absent from its Go 1.27 EVEX form: %q", c.goarch, baseOp, suffix, ins.Raw) + } + sae = true + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + if baseOp != "VCVTSD2SS" || sae || rounding != "" { + return true, false, fmt.Errorf("%s %s suffix .%s is absent from its Go 1.27 EVEX form: %q", c.goarch, baseOp, suffix, ins.Raw) + } + rounding = suffix + default: + return true, false, fmt.Errorf("%s %s has unsupported suffix .%s: %q", c.goarch, baseOp, suffix, ins.Raw) + } + } + if baseOp != "VCVTSS2SD" && baseOp != "VCVTSD2SS" { + return false, false, nil + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects src, upper Xsrc, [K mask,] Xdst: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's operand limit: %q", baseOp, ins.Raw) + } + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if !c.isGoEVEXVectorRegister(ins.Args[1], 16) { + return true, false, fmt.Errorf("%s %s upper source must be a Go 1.27 X register: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoEVEXVectorRegister(destination, 16) { + return true, false, fmt.Errorf("%s %s destination must be a Go 1.27 X register: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if !c.isGoEVEXVectorRegister(ins.Args[0], 16) { + return true, false, fmt.Errorf("%s %s register source must be a Go 1.27 X register: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s source must be an X register or memory operand: %q", c.goarch, baseOp, ins.Raw) + } + if (sae || rounding != "") && ins.Args[0].Kind != OpReg { + return true, false, fmt.Errorf("%s %s embedded rounding/SAE requires a register source: %q", c.goarch, baseOp, ins.Raw) + } + var converted string + var err error + if baseOp == "VCVTSS2SD" { + src, e := c.evalF32(ins.Args[0]) + if e != nil { + return true, false, e + } + converted = c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", converted, src) + } else { + src, e := c.evalF64(ins.Args[0]) + if e != nil { + return true, false, e + } + converted = c.emitRoundedF64ToF32(src, rounding) + } + upperBytes, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + mask := "" + if masked { + mask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + if baseOp == "VCVTSS2SD" { + upperWords := c.bitcastVectorBytesToIntegerLanes(16, 2, 64, upperBytes) + lowBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast double %%%s to i64\n", lowBits, converted) + return true, false, c.storeVectorScalarRegister(destination.Reg, 64, "%"+lowBits, upperWords, mask, zeroing) + } + + upperWords := c.bitcastVectorBytesToIntegerLanes(16, 4, 32, upperBytes) + lowBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %%%s to i32\n", lowBits, converted) + return true, false, c.storeVectorScalarRegister(destination.Reg, 32, "%"+lowBits, upperWords, mask, zeroing) +} + +// emitRoundedF64ToF32 implements EVEX embedded rounding independently of the +// host floating-point environment. LLVM's scalar constrained fptrunc may lower +// to the ambient MXCSR mode on targets without AVX-512, so compute the nearest +// result first and step one IEEE-754 float toward the requested direction only +// when extending it back to double proves that adjustment is necessary. +func (c *amd64Ctx) emitRoundedF64ToF32(source, rounding string) string { + nearest := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fptrunc double %s to float\n", nearest, source) + if rounding == "" || rounding == "RN_SAE" { + return nearest + } + expanded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %%%s to double\n", expanded, nearest) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast float %%%s to i32\n", bits, nearest) + negativeBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %%%s, 0\n", negativeBits, bits) + incremented := c.newTmp() + decremented := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, 1\n", incremented, bits) + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, 1\n", decremented, bits) + nextUpBits := c.newTmp() + nextDownBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %%%s\n", nextUpBits, negativeBits, decremented, incremented) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %%%s\n", nextDownBits, negativeBits, incremented, decremented) + nextUp := c.newTmp() + nextDown := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", nextUp, nextUpBits) + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", nextDown, nextDownBits) + needsUp := c.newTmp() + needsDown := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fcmp olt double %%%s, %s\n", needsUp, expanded, source) + fmt.Fprintf(c.b, " %%%s = fcmp ogt double %%%s, %s\n", needsDown, expanded, source) + upResult := c.newTmp() + downResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, float %%%s, float %%%s\n", upResult, needsUp, nextUp, nearest) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, float %%%s, float %%%s\n", downResult, needsDown, nextDown, nearest) + switch rounding { + case "RD_SAE": + return downResult + case "RU_SAE": + return upResult + case "RZ_SAE": + negativeSource := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fcmp olt double %s, 0.000000e+00\n", negativeSource, source) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, float %%%s, float %%%s\n", result, negativeSource, upResult, downResult) + return result + default: + return nearest + } +} + +func (c *amd64Ctx) evalF32(op Operand) (string, error) { + switch op.Kind { + case OpReg: + if _, ok := amd64ParseXReg(op.Reg); !ok { + return "", fmt.Errorf("amd64: expected X register for f32 operand, got %s", op.String()) + } + return c.loadXLowF32(op.Reg) + case OpImm: + bits := c.newTmp() + if op.ImmIsFloat { + // The parser retains the original float64 constant. Go's + // Float32Sym rounds its numeric value, not its low 32 bits. + rounded := math.Float32bits(float32(math.Float64frombits(uint64(op.Imm)))) + fmt.Fprintf(c.b, " %%%s = bitcast i32 %d to float\n", bits, rounded) + return "%" + bits, nil + } + fmt.Fprintf(c.b, " %%%s = trunc i64 %d to i32\n", bits, op.Imm) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %%%s to float\n", value, bits) + return "%" + value, nil + case OpFP: + return c.evalFPToF32(op.FPOffset) + case OpMem: + p, ptrType, err := c.ptrFromMem(op.Mem) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load float, %s %s, align 1\n", value, ptrType, p) + return "%" + value, nil + case OpSym: + p, err := c.ptrFromSB(op.Sym) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load float, ptr %s, align 1\n", value, p) + return "%" + value, nil + default: + return "", fmt.Errorf("amd64: unsupported f32 operand %s", op.String()) + } +} + +func (c *amd64Ctx) evalFPToF32(off int64) (string, error) { + bits, err := c.loadScalarFPBits(off, 32) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i32 %s to float\n", value, bits) + return "%" + value, nil +} + func (c *amd64Ctx) evalF64(op Operand) (string, error) { switch op.Kind { case OpReg: @@ -515,34 +1206,11 @@ func (c *amd64Ctx) evalF64(op Operand) (string, error) { } func (c *amd64Ctx) evalFPToF64(off int64) (string, error) { - slot, ok := c.fpParam(off) - if !ok { - return "", fmt.Errorf("unsupported FP read slot for f64: +%d(FP)", off) - } - idx := slot.Index - if idx < 0 || idx >= len(c.sig.Args) { - return "", fmt.Errorf("FP read slot for f64: invalid arg index %d at +%d(FP)", idx, off) - } - arg := fmt.Sprintf("%%arg%d", idx) - ty := slot.Type - if c.classicFrame != "" { - loaded := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", loaded, ty, c.classicFramePtr(off)) - arg = "%" + loaded - } else if slot.Field >= 0 { - aggTy := c.sig.Args[idx] - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue %s %s, %d\n", t, aggTy, arg, slot.Field) - arg = "%" + t - } - switch ty { - case LLVMType("double"): - return arg, nil - case I64: - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", t, arg) - return "%" + t, nil - default: - return "", fmt.Errorf("FP read unsupported type %q for f64 at +%d(FP)", ty, off) + bits, err := c.loadScalarFPBits(off, 64) + if err != nil { + return "", err } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to double\n", value, bits) + return "%" + value, nil } diff --git a/amd64_lower_fpclass.go b/amd64_lower_fpclass.go new file mode 100644 index 000000000..ded8fefe0 --- /dev/null +++ b/amd64_lower_fpclass.go @@ -0,0 +1,204 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FPClassSpec struct { + laneBits int + bytes int + scalar bool +} + +// amd64FPClassSpecs mirrors the width-specialized opcode names in Go 1.27's +// _yvfpclasspdx/_yvfpclasspdy/_yvfpclasspdz tables. Classification itself is +// shared across every packed and scalar form. +var amd64FPClassSpecs = map[Op]amd64FPClassSpec{ + "VFPCLASSPDX": {laneBits: 64, bytes: 16}, + "VFPCLASSPDY": {laneBits: 64, bytes: 32}, + "VFPCLASSPDZ": {laneBits: 64, bytes: 64}, + "VFPCLASSPSX": {laneBits: 32, bytes: 16}, + "VFPCLASSPSY": {laneBits: 32, bytes: 32}, + "VFPCLASSPSZ": {laneBits: 32, bytes: 64}, + "VFPCLASSSD": {laneBits: 64, bytes: 16, scalar: true}, + "VFPCLASSSS": {laneBits: 32, bytes: 16, scalar: true}, +} + +func (c *amd64Ctx) lowerFPClass(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FPClassSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + broadcast := suffix == "BCST" + if suffix != "" && !broadcast { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's FPCLASS optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects immediate, source, [K mask,] K destination: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("%s %s immediate must be an unsigned byte: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s input mask must be K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if !amd64IsKOperand(destination) { + return true, false, fmt.Errorf("%s %s destination must be K0-K7: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[1] + if broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, spec.bytes) { + return true, false, fmt.Errorf("%s %s source register does not match its opcode width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be its named vector width or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := spec.bytes * 8 / spec.laneBits + var bits string + if spec.scalar { + scalarBits, loadErr := c.loadFloatingScalarBits(source, spec.laneBits) + if loadErr != nil { + return true, false, loadErr + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", inserted, spec.laneBits, spec.laneBits, scalarBits) + bits = "%" + inserted + lanes = 1 + } else { + bits, err = c.loadPackedCompareLanes(source, spec.bytes, spec.laneBits, broadcast) + if err != nil { + return true, false, err + } + } + classified := c.emitFPClassPredicate(lanes, spec.laneBits, bits, uint8(ins.Args[0].Imm)) + var result string + if lanes == 1 { + low := c.newTmp() + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i1> %s, i32 0\n", low, classified) + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i64\n", wide, low) + result = "%" + wide + } else { + result = c.packFloatingCompareMask(classified, lanes) + } + if masked { + writeMask, loadErr := c.loadK(ins.Args[2].Reg) + if loadErr != nil { + return true, false, loadErr + } + maskedResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", maskedResult, result, writeMask) + result = "%" + maskedResult + } + return true, false, c.storeK(destination.Reg, result) +} + +func (c *amd64Ctx) emitFPClassPredicate(lanes, laneBits int, bits string, immediate uint8) string { + integerType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + predicateType := fmt.Sprintf("<%d x i1>", lanes) + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + if laneBits == 64 { + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + } + exponent, mantissa, quiet := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", exponent, integerType, bits, llvmSplatInteger(lanes, laneBits, exponentMask)) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", mantissa, integerType, bits, llvmSplatInteger(lanes, laneBits, mantissaMask)) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", quiet, integerType, bits, llvmSplatInteger(lanes, laneBits, quietBit)) + exponentOnes, exponentNotOnes := c.newTmp(), c.newTmp() + exponentZero, exponentNotZero := c.newTmp(), c.newTmp() + mantissaZero, mantissaNonzero := c.newTmp(), c.newTmp() + quietSet, quietClear := c.newTmp(), c.newTmp() + negative, nonnegative := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %s\n", exponentOnes, integerType, exponent, llvmSplatInteger(lanes, laneBits, exponentMask)) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, %s\n", exponentNotOnes, integerType, exponent, llvmSplatInteger(lanes, laneBits, exponentMask)) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", exponentZero, integerType, exponent) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", exponentNotZero, integerType, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", mantissaZero, integerType, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", mantissaNonzero, integerType, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", quietSet, integerType, quiet) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", quietClear, integerType, quiet) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, zeroinitializer\n", negative, integerType, bits) + fmt.Fprintf(c.b, " %%%s = icmp sge %s %s, zeroinitializer\n", nonnegative, integerType, bits) + daz := c.loadMXCSRDAZ() + dazClear := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i1 %s, false\n", dazClear, daz) + dazSetVector := amd64SplatI1(c, lanes, daz) + dazClearVector := amd64SplatI1(c, lanes, "%"+dazClear) + and := func(first, second string) string { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", value, predicateType, first, second) + return "%" + value + } + categories := [8]string{} + nan := and("%"+exponentOnes, "%"+mantissaNonzero) + categories[0] = and(nan, "%"+quietSet) + effectiveMantissaZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", effectiveMantissaZero, predicateType, mantissaZero, dazSetVector) + zero := and("%"+exponentZero, "%"+effectiveMantissaZero) + categories[1] = and(zero, "%"+nonnegative) + categories[2] = and(zero, "%"+negative) + infinity := and("%"+exponentOnes, "%"+mantissaZero) + categories[3] = and(infinity, "%"+nonnegative) + categories[4] = and(infinity, "%"+negative) + effectiveMantissaNonzero := and("%"+mantissaNonzero, dazClearVector) + categories[5] = and("%"+exponentZero, effectiveMantissaNonzero) + negativeFinite := and("%"+negative, "%"+exponentNotOnes) + effectiveNonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", effectiveNonzero, predicateType, exponentNotZero, effectiveMantissaNonzero) + categories[6] = and(negativeFinite, "%"+effectiveNonzero) + categories[7] = and(nan, "%"+quietClear) + result := "zeroinitializer" + for category, predicate := range categories { + if immediate&(1< +#include +#include +extern void fpclasssemantics(uint8_t *, const uint32_t *, const uint64_t *, uint64_t); +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + const uint32_t source32[16]={ + 0x7fc12345,0x00000000,0x80000000,0x7f800000, + 0xff800000,0x00000001,0xbf800000,0x7f812345, + 0x3f800000,0x40000000,0x00800000,0x7f7fffff, + 0x3f000000,0x40400000,0x3f800001,0x3f800002}; + const uint64_t source64[8]={ + UINT64_C(0x7ff8123456789abc),UINT64_C(0),UINT64_C(0x8000000000000000),UINT64_C(0x7ff0000000000000), + UINT64_C(0xfff0000000000000),UINT64_C(1),UINT64_C(0xbff0000000000000),UINT64_C(0x7ff0123456789abc)}; + uint8_t out[273]={0}, outDaz[273]={0}; + fpclasssemantics(out,source32,source64,UINT64_C(0xa5)); + unsigned oldCsr=_mm_getcsr(); + _mm_setcsr(oldCsr|UINT32_C(0x40)); + fpclasssemantics(outDaz,source32,source64,UINT64_C(0xa5)); + _mm_setcsr(oldCsr); + for (int category=0;category<8;category++) { + uint64_t bit=UINT64_C(1)<= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FunnelShiftSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's funnel-shift encoding: %q", c.goarch, baseOp, ins.Raw) + } + if spec.laneBits == 16 && properties.broadcast { + return true, false, fmt.Errorf("%s %s word forms do not enable EVEX broadcast: %q", c.goarch, baseOp, ins.Raw) + } + + unmaskedArgs := 4 + maskedArgs := 5 + if spec.variable { + unmaskedArgs, maskedArgs = 3, 4 + } + if len(ins.Args) != unmaskedArgs && len(ins.Args) != maskedArgs { + return true, false, fmt.Errorf("%s %s has an operand count outside Go 1.27's encoding table: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == maskedArgs + if !spec.variable && c.goarch == "386" { + return true, false, fmt.Errorf("386 %s is rejected by the Go assembler frontend's operand limit: %q", baseOp, ins.Raw) + } + if spec.variable && c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + + immediateOffset := 0 + if !spec.variable { + if !amd64UnsignedImmediate(ins.Args[0], 8) { + return true, false, fmt.Errorf("%s %s first operand must be Go 1.27's unsigned-imm8 class: %q", c.goarch, baseOp, ins.Raw) + } + immediateOffset = 1 + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || + !c.isGoPackedVectorMoveRegister(destination, byteWidth) || + !c.isGoPackedVectorMoveRegister(ins.Args[immediateOffset+1], byteWidth) { + return true, false, fmt.Errorf("%s %s second vector source and destination must be matching Go vector registers: %q", c.goarch, baseOp, ins.Raw) + } + rmSource := ins.Args[immediateOffset] + if properties.broadcast { + if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory first vector source: %q", c.goarch, baseOp, ins.Raw) + } + } else if rmSource.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(rmSource, byteWidth) { + return true, false, fmt.Errorf("%s %s first vector source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s first vector source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + maskOperandIndex := len(ins.Args) - 2 + if masked && !amd64NonzeroKOperand(ins.Args[maskOperandIndex]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7 before the destination: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth * 8 / spec.laneBits + first, err := c.loadPackedCompareLanes(rmSource, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + second, err := c.loadPackedCompareLanes(ins.Args[immediateOffset+1], byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + old := "" + if spec.variable || masked { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + } + + count := first + leftInput, rightInput := old, second + if !spec.variable { + count = amd64SplatInteger(c, lanes, spec.laneBits, fmt.Sprintf("%d", uint8(ins.Args[0].Imm))) + leftInput, rightInput = second, first + } + intrinsic := "fshl" + firstArgument, secondArgument := leftInput, rightInput + if !spec.left { + intrinsic = "fshr" + firstArgument, secondArgument = rightInput, leftInput + } + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + computedName := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.%s.v%di%d(%s %s, %s %s, %s %s)\n", + computedName, vectorType, intrinsic, lanes, spec.laneBits, + vectorType, firstArgument, vectorType, secondArgument, vectorType, count) + computed := "%" + computedName + if masked { + mask, loadErr := c.loadK(ins.Args[maskOperandIndex].Reg) + if loadErr != nil { + return true, false, loadErr + } + computed = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, vectorType, computed, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} diff --git a/amd64_lower_funnel_shift_test.go b/amd64_lower_funnel_shift_test.go new file mode 100644 index 000000000..deb4d4da2 --- /dev/null +++ b/amd64_lower_funnel_shift_test.go @@ -0,0 +1,285 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86FunnelShiftCompleteGo127Forms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT funnel_shift_forms(SB),$0-0\n") + if target.goarch == "amd64" { + for _, op := range []string{"VPSHLDW", "VPSHLDD", "VPSHLDQ", "VPSHRDW", "VPSHRDD", "VPSHRDQ"} { + broadcast := !strings.HasSuffix(op, "W") + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s $0, %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s $255, 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&source, "\t%s $1, %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z $2, 16(BX), %s23, K7, %s24\n", op, width, width) + if broadcast { + fmt.Fprintf(&source, "\t%s.BCST $3, 24(BX), %s25, %s26\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(BX), %s27, K2, %s28\n", op, width, width) + } + } + } + } + for _, op := range []string{"VPSHLDVW", "VPSHLDVD", "VPSHLDVQ", "VPSHRDVW", "VPSHRDVD", "VPSHRDVQ"} { + broadcast := !strings.HasSuffix(op, "VW") + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(BX), %s3, %s4\n", op, width, width) + if broadcast { + fmt.Fprintf(&source, "\t%s.BCST 16(BX), %s5, %s6\n", op, width, width) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 24(BX), %s23, K7, %s24\n", op, width, width) + if broadcast { + fmt.Fprintf(&source, "\t%s.BCST.Z 32(BX), %s25, K2, %s26\n", op, width, width) + } + } + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "funnel_shift_forms": {Name: "funnel_shift_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "funnel-shift-"+target.name+".ll", "funnel-shift-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FunnelShiftRejectsFormsOutsideGo127Tables(t *testing.T) { + for _, instruction := range []string{ + "VPSHRDQ $0, X0, X1", + "VPSHLDW $-1, X0, X1, X2", + "VPSHRDD $256, X0, X1, X2", + "VPSHLDQ AX, X0, X1, X2", + "VPSHRDW $0, Y0, X1, X2", + "VPSHLDVD Y0, X1, X2", + "VPSHRDVQ X0, Y1, X2", + "VPSHLDVW X0, X1, Y2", + "VPSHRDQ $0, X0, 8(BX), X2", + "VPSHLDVD X0, 8(BX), X2", + "VPSHRDVQ X0, X1, 8(BX)", + "VPSHLDQ $0, X0, X1, K0, X2", + "VPSHRDVD X0, K1, X1, X2", + "VPSHLDW.Z $0, X0, X1, X2", + "VPSHRDVW.Z X0, X1, X2", + "VPSHLDW.BCST $0, 8(BX), X1, X2", + "VPSHRDVW.BCST 8(BX), X1, X2", + "VPSHLDQ.BCST X0, X1, X2", + "VPSHRDVD.BCST X0, X1, X2", + "VPSHLDQ.Z.BCST $0, 8(BX), X1, K1, X2", + "VPSHRDVQ.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's funnel-shift tables", instruction) + } + }) + } + for _, instruction := range []string{ + "VPSHRDQ $0, X0, X1, X2", + "VPSHLDW $0, 8(BX), Y1, Y2", + "VPSHRDVD X0, X1, K1, X2", + "VPSHLDVQ.Z 8(BX), Y1, K7, Y2", + "VPSHRDVD.BCST.Z 16(BX), Z1, K2, Z2", + "VPSHLDVW Z8, Z0, Z1", + "VPSHRDVQ Z0, Z8, Z1", + "VPSHLDVD Z0, Z1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted 386 %q outside Go 1.27's frontend boundary", instruction) + } + }) + } +} + +func TestAMD64FunnelShiftRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT funnelShiftSemantics(SB),$0-64 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ old+24(FP), DX + MOVQ counts+32(FP), SI + MOVQ scalarCount+40(FP), R8 + MOVQ scalarData+48(FP), R9 + VMOVDQU64 (BX), X0 + VMOVDQU64 (CX), X1 + VMOVDQU64 (SI), X3 + VPSHLDW $4, X0, X1, X4 + VMOVDQU64 X4, 0(AX) + VPSHRDW $4, X0, X1, X5 + VMOVDQU64 X5, 16(AX) + VPSHLDD $8, X0, X1, X6 + VMOVDQU64 X6, 32(AX) + VPSHRDD $8, X0, X1, X7 + VMOVDQU64 X7, 48(AX) + VPSHLDQ $13, X0, X1, X8 + VMOVDQU64 X8, 64(AX) + VPSHRDQ $32, X0, X1, X9 + VMOVDQU64 X9, 80(AX) + VMOVDQU64 (DX), X10 + VPSHLDVW X3, X1, X10 + VMOVDQU64 X10, 96(AX) + VMOVDQU64 (DX), X11 + VPSHRDVD X3, X1, X11 + VMOVDQU64 X11, 112(AX) + VMOVDQU64 (DX), X12 + VPSHLDVQ X3, X1, X12 + VMOVDQU64 X12, 128(AX) + KMOVQ mask+56(FP), K1 + VMOVDQU64 (DX), X13 + VPSHRDQ $32, X0, X1, K1, X13 + VMOVDQU64 X13, 144(AX) + VMOVDQU64 (DX), X14 + VPSHRDQ.Z $32, X0, X1, K1, X14 + VMOVDQU64 X14, 160(AX) + VMOVDQU64 (DX), X15 + VPSHLDVD.BCST (R8), X1, X15 + VMOVDQU64 X15, 176(AX) + VPSHRDD.BCST $8, (R9), X1, X16 + VMOVDQU64 X16, 192(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "funnelShiftSemantics": { + Name: "funnelShiftSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +typedef union { uint8_t b[16]; uint16_t w[8]; uint32_t d[4]; uint64_t q[2]; } V; +extern void funnelShiftSemantics(V *, const V *, const V *, const V *, const V *, const uint32_t *, const uint32_t *, uint64_t); +static uint16_t shl16(uint16_t a, uint16_t b, unsigned c) { c &= 15; return c ? (uint16_t)((a << c) | (b >> (16-c))) : a; } +static uint16_t shr16(uint16_t a, uint16_t b, unsigned c) { c &= 15; return c ? (uint16_t)((a >> c) | (b << (16-c))) : a; } +static uint32_t shl32(uint32_t a, uint32_t b, unsigned c) { c &= 31; return c ? (a << c) | (b >> (32-c)) : a; } +static uint32_t shr32(uint32_t a, uint32_t b, unsigned c) { c &= 31; return c ? (a >> c) | (b << (32-c)) : a; } +static uint64_t shl64(uint64_t a, uint64_t b, unsigned c) { c &= 63; return c ? (a << c) | (b >> (64-c)) : a; } +static uint64_t shr64(uint64_t a, uint64_t b, unsigned c) { c &= 63; return c ? (a >> c) | (b << (64-c)) : a; } +int main(void) { + V first = {.q = {UINT64_C(0x0123456789abcdef), UINT64_C(0xfedcba9876543210)}}; + V second = {.q = {UINT64_C(0x8877665544332211), UINT64_C(0x1020304050607080)}}; + V old = {.q = {UINT64_C(0x0f1e2d3c4b5a6978), UINT64_C(0x8796a5b4c3d2e1f0)}}; + V counts = {.q = {UINT64_C(0x001f000800030000), UINT64_C(0x0041002000110004)}}; + uint32_t scalarCount = 37; + uint32_t scalarData = UINT32_C(0xa1b2c3d4); + V out[13] = {0}, want[13] = {0}; + for (int i = 0; i < 8; i++) { + want[0].w[i] = shl16(second.w[i], first.w[i], 4); + want[1].w[i] = shr16(second.w[i], first.w[i], 4); + want[6].w[i] = shl16(old.w[i], second.w[i], counts.w[i]); + } + for (int i = 0; i < 4; i++) { + want[2].d[i] = shl32(second.d[i], first.d[i], 8); + want[3].d[i] = shr32(second.d[i], first.d[i], 8); + want[7].d[i] = shr32(old.d[i], second.d[i], counts.d[i]); + want[11].d[i] = shl32(old.d[i], second.d[i], scalarCount); + want[12].d[i] = shr32(second.d[i], scalarData, 8); + } + for (int i = 0; i < 2; i++) { + want[4].q[i] = shl64(second.q[i], first.q[i], 13); + want[5].q[i] = shr64(second.q[i], first.q[i], 32); + want[8].q[i] = shl64(old.q[i], second.q[i], counts.q[i]); + } + want[9].q[0] = want[5].q[0]; want[9].q[1] = old.q[1]; + want[10].q[0] = want[5].q[0]; want[10].q[1] = 0; + funnelShiftSemantics(out, &first, &second, &old, &counts, &scalarCount, &scalarData, 1); + return memcmp(out, want, sizeof(out)) == 0 ? 0 : 1; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "funnel_shift", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_gather.go b/amd64_lower_gather.go new file mode 100644 index 000000000..698170821 --- /dev/null +++ b/amd64_lower_gather.go @@ -0,0 +1,380 @@ +package plan9asm + +import "fmt" + +type amd64GatherTable uint8 + +const ( + amd64GatherDPS amd64GatherTable = iota + amd64GatherDPD + amd64GatherQPS +) + +type amd64GatherSpec struct { + table amd64GatherTable + elemBits int + indexBits int +} + +// amd64GatherSpecs is the complete Go 1.27 data-gather family. The eight +// spellings share _yvgatherdps, _yvgatherdpd, and _yvgatherqps. +var amd64GatherSpecs = map[Op]amd64GatherSpec{ + "VGATHERDPS": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VGATHERQPD": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VPGATHERDD": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VPGATHERQQ": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VGATHERDPD": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VPGATHERDQ": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VGATHERQPS": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, + "VPGATHERQD": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, +} + +type amd64VSIBPrefetchDirection uint8 + +const ( + amd64VSIBPrefetchGather amd64VSIBPrefetchDirection = iota + amd64VSIBPrefetchScatter +) + +type amd64VSIBPrefetchSpec struct { + direction amd64VSIBPrefetchDirection + hint int + elemBits int + indexBits int + indexBytes int +} + +// amd64VSIBPrefetchSpecs is the complete Go 1.27 gather/scatter prefetch +// grammar. Direction, cache hint, element width, and VSIB index width are data; +// all sixteen opcodes share one exact operand validator. These instructions +// are architectural hints, so lowering has no observable IR side effect. +var amd64VSIBPrefetchSpecs = map[Op]amd64VSIBPrefetchSpec{ + "VGATHERPF0DPD": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VGATHERPF0DPS": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VGATHERPF0QPD": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VGATHERPF0QPS": {direction: amd64VSIBPrefetchGather, hint: 0, elemBits: 32, indexBits: 64, indexBytes: 64}, + "VGATHERPF1DPD": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VGATHERPF1DPS": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VGATHERPF1QPD": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VGATHERPF1QPS": {direction: amd64VSIBPrefetchGather, hint: 1, elemBits: 32, indexBits: 64, indexBytes: 64}, + "VSCATTERPF0DPD": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VSCATTERPF0DPS": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VSCATTERPF0QPD": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VSCATTERPF0QPS": {direction: amd64VSIBPrefetchScatter, hint: 0, elemBits: 32, indexBits: 64, indexBytes: 64}, + "VSCATTERPF1DPD": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 64, indexBits: 32, indexBytes: 32}, + "VSCATTERPF1DPS": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 32, indexBits: 32, indexBytes: 64}, + "VSCATTERPF1QPD": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 64, indexBits: 64, indexBytes: 64}, + "VSCATTERPF1QPS": {direction: amd64VSIBPrefetchScatter, hint: 1, elemBits: 32, indexBits: 64, indexBytes: 64}, +} + +func (c *amd64Ctx) lowerGather(op Op, ins Instr) error { + if prefetch, ok := amd64VSIBPrefetchSpecs[op]; ok { + return c.lowerVSIBPrefetch(op, prefetch, ins) + } + spec := amd64GatherSpecs[op] + if len(ins.Args) != 3 { + return fmt.Errorf("amd64 %s expects exactly three operands from its Go 1.27 gather table: %q", op, ins.Raw) + } + + var memory Operand + var mask Operand + var destination Operand + evex := false + if ins.Args[0].Kind == OpMem { + evex = true + memory, mask, destination = ins.Args[0], ins.Args[1], ins.Args[2] + } else { + mask, memory, destination = ins.Args[0], ins.Args[1], ins.Args[2] + } + if memory.Kind != OpMem || destination.Kind != OpReg { + return fmt.Errorf("amd64 %s expects vector-mask,memory,vector or memory,K-mask,vector: %q", op, ins.Raw) + } + destinationBytes := amd64VectorByteWidth(destination.Reg) + indexBytes := amd64VectorByteWidth(memory.Mem.Index) + if destinationBytes == 0 || indexBytes == 0 || !amd64GatherTableAllows(spec.table, evex, destinationBytes, indexBytes) { + return fmt.Errorf("amd64 %s vector widths are outside its Go 1.27 gather table: %q", op, ins.Raw) + } + if err := c.validateGatherMemory(memory.Mem, indexBytes, evex); err != nil { + return fmt.Errorf("amd64 %s invalid VSIB memory: %q: %w", op, ins.Raw, err) + } + destinationIndex, ok := amd64GatherRegisterIndex(destination.Reg, destinationBytes, evex, c.goarch, false) + if !ok { + return fmt.Errorf("amd64 %s destination is outside its Go 1.27 register class: %q", op, ins.Raw) + } + indexIndex, ok := amd64GatherRegisterIndex(memory.Mem.Index, indexBytes, evex, c.goarch, true) + if !ok { + return fmt.Errorf("amd64 %s VSIB index is outside its Go 1.27 register class: %q", op, ins.Raw) + } + if destinationIndex == indexIndex { + return fmt.Errorf("amd64 %s index and destination registers must be distinct: %q", op, ins.Raw) + } + + if evex { + maskIndex, ok := amd64ParseKReg(mask.Reg) + if mask.Kind != OpReg || !ok || maskIndex == 0 { + return fmt.Errorf("amd64 %s EVEX form requires K1..K7: %q", op, ins.Raw) + } + } else { + maskBytes := amd64VectorByteWidth(mask.Reg) + maskIndex, ok := amd64GatherRegisterIndex(mask.Reg, destinationBytes, false, c.goarch, false) + if mask.Kind != OpReg || !ok || maskBytes != destinationBytes { + return fmt.Errorf("amd64 %s VEX form requires a destination-width vector mask: %q", op, ins.Raw) + } + if maskIndex == indexIndex || maskIndex == destinationIndex { + return fmt.Errorf("amd64 %s mask, index, and destination registers must be distinct: %q", op, ins.Raw) + } + } + + return c.emitGather(spec, memory.Mem, mask, destination.Reg, destinationBytes, indexBytes, evex) +} + +func (c *amd64Ctx) lowerVSIBPrefetch(op Op, spec amd64VSIBPrefetchSpec, ins Instr) error { + if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpMem { + return fmt.Errorf("amd64 %s expects K1..K7 and VSIB memory: %q", op, ins.Raw) + } + maskIndex, ok := amd64ParseKReg(ins.Args[0].Reg) + if !ok || maskIndex == 0 { + return fmt.Errorf("amd64 %s requires K1..K7: %q", op, ins.Raw) + } + indexBytes := spec.indexBytes + if amd64VectorByteWidth(ins.Args[1].Mem.Index) != indexBytes { + return fmt.Errorf("amd64 %s has the wrong VSIB index width: %q", op, ins.Raw) + } + if err := c.validateGatherMemory(ins.Args[1].Mem, indexBytes, true); err != nil { + return fmt.Errorf("amd64 %s invalid VSIB memory: %q: %w", op, ins.Raw, err) + } + if _, ok := amd64GatherRegisterIndex(ins.Args[1].Mem.Index, indexBytes, true, c.goarch, true); !ok { + return fmt.Errorf("amd64 %s VSIB index is outside its Go 1.27 register class: %q", op, ins.Raw) + } + return nil +} + +func amd64GatherTableAllows(table amd64GatherTable, evex bool, destinationBytes, indexBytes int) bool { + allowed := false + switch table { + case amd64GatherDPS: + allowed = (destinationBytes == 16 && indexBytes == 16) || + (destinationBytes == 32 && indexBytes == 32) || + (destinationBytes == 64 && indexBytes == 64) + case amd64GatherDPD: + allowed = (destinationBytes == 16 && indexBytes == 16) || + (destinationBytes == 32 && indexBytes == 16) || + (destinationBytes == 64 && indexBytes == 32) + case amd64GatherQPS: + allowed = (destinationBytes == 16 && (indexBytes == 16 || indexBytes == 32)) || + (destinationBytes == 32 && indexBytes == 64) + } + if !allowed { + return false + } + if !evex { + return destinationBytes != 64 + } + return true +} + +func (c *amd64Ctx) validateGatherMemory(memory MemRef, indexBytes int, evex bool) error { + if memory.Index == "" || amd64VectorByteWidth(memory.Index) != indexBytes { + return fmt.Errorf("expected a VSIB vector index") + } + if memory.Scale == 0 { + memory.Scale = 1 + } + if memory.Scale != 1 && memory.Scale != 2 && memory.Scale != 4 && memory.Scale != 8 { + return fmt.Errorf("VSIB scale must be 1, 2, 4, or 8") + } + if memory.Sym != "" { + return fmt.Errorf("Go's x86 assembler does not accept an SB base with VSIB") + } + if memory.Base != "" && !isX86YrlRegisterForArch(memory.Base, c.goarch) { + return fmt.Errorf("VSIB base is not a GP register for %s", c.goarch) + } + if memory.Segment != "" && memory.Segment != FS && memory.Segment != GS { + return fmt.Errorf("unsupported VSIB segment %s", memory.Segment) + } + _ = evex + return nil +} + +func amd64GatherRegisterIndex(register Reg, byteWidth int, evex bool, goarch string, index bool) (int, bool) { + var registerIndex int + var ok bool + switch byteWidth { + case 16: + registerIndex, ok = amd64ParseXReg(register) + case 32: + registerIndex, ok = amd64ParseYReg(register) + case 64: + registerIndex, ok = amd64ParseZReg(register) + } + if !ok { + return 0, false + } + limit := 15 + if evex { + limit = 31 + } + if goarch == "386" && (index || byteWidth == 64) { + limit = 7 + } + return registerIndex, registerIndex <= limit +} + +func (c *amd64Ctx) emitGather(spec amd64GatherSpec, memory MemRef, mask Operand, destination Reg, destinationBytes, indexBytes int, evex bool) error { + destinationValue, err := c.loadGatherVector(destination, destinationBytes) + if err != nil { + return err + } + destinationLanes := destinationBytes * 8 / spec.elemBits + destinationType := fmt.Sprintf("<%d x i%d>", destinationLanes, spec.elemBits) + destinationValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", destinationValues, destinationBytes, destinationValue, destinationType) + result := "%" + destinationValues + + indexValue, err := c.loadGatherVector(memory.Index, indexBytes) + if err != nil { + return err + } + indexLanes := indexBytes * 8 / spec.indexBits + indexType := fmt.Sprintf("<%d x i%d>", indexLanes, spec.indexBits) + indexValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", indexValues, indexBytes, indexValue, indexType) + activeLanes := destinationLanes + if indexLanes < activeLanes { + activeLanes = indexLanes + // QPS/QD with an X index has two active dword lanes and architecturally + // zeroes the upper two lanes of the X destination. + for lane := activeLanes; lane < destinationLanes; lane++ { + zeroed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d 0, i32 %d\n", zeroed, destinationType, result, spec.elemBits, lane) + result = "%" + zeroed + } + } + + var maskValues string + var kMask string + if evex { + kMask, err = c.loadK(mask.Reg) + } else { + maskBytes, err := c.loadGatherVector(mask.Reg, destinationBytes) + if err != nil { + return err + } + maskValues = c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", maskValues, destinationBytes, maskBytes, destinationType) + maskValues = "%" + maskValues + } + + base, err := c.gatherBaseAddress(memory) + if err != nil { + return err + } + for lane := 0; lane < activeLanes; lane++ { + indexLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", indexLane, indexType, indexValues, lane) + wideIndex := "%" + indexLane + if spec.indexBits == 32 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i32 %%%s to i64\n", extended, indexLane) + wideIndex = "%" + extended + } + if memory.Scale != 0 && memory.Scale != 1 { + scaled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", scaled, wideIndex, memory.Scale) + wideIndex = "%" + scaled + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", address, base, wideIndex) + pointer, pointerType := c.gatherPointer(memory, "%"+address) + + condition := "" + if evex { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", shifted, kMask, lane) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 1\n", bit, shifted) + set := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", set, bit) + condition = "%" + set + } else { + maskLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", maskLane, destinationType, maskValues, lane) + set := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %%%s, 0\n", set, spec.elemBits, maskLane) + condition = "%" + set + } + + oldLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", oldLane, destinationType, result, lane) + loadLabel := c.newTmp() + "_gather_load" + skipLabel := c.newTmp() + "_gather_skip" + joinLabel := c.newTmp() + "_gather_join" + loaded := c.newTmp() + selected := c.newTmp() + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n", condition, loadLabel, skipLabel) + fmt.Fprintf(c.b, "%s:\n", loadLabel) + fmt.Fprintf(c.b, " %%%s = load i%d, %s %s, align 1\n", loaded, spec.elemBits, pointerType, pointer) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", skipLabel) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", joinLabel) + fmt.Fprintf(c.b, " %%%s = phi i%d [ %%%s, %%%s ], [ %%%s, %%%s ]\n", selected, spec.elemBits, loaded, loadLabel, oldLane, skipLabel) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, destinationType, result, spec.elemBits, selected, lane) + result = "%" + inserted + } + + bytesResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytesResult, destinationType, result, destinationBytes) + if err := c.storeVectorBytes(destination, destinationBytes, "%"+bytesResult); err != nil { + return err + } + if evex { + return c.storeK(mask.Reg, "0") + } + return c.storeVectorBytes(mask.Reg, destinationBytes, "zeroinitializer") +} + +func (c *amd64Ctx) loadGatherVector(register Reg, byteWidth int) (string, error) { + switch byteWidth { + case 16: + return c.loadX(register) + case 32: + return c.loadY(register) + case 64: + return c.loadZ(register) + default: + return "", fmt.Errorf("unsupported gather vector width %d", byteWidth) + } +} + +func (c *amd64Ctx) gatherBaseAddress(memory MemRef) (string, error) { + base := "0" + if memory.Base != "" { + value, err := c.loadReg(memory.Base) + if err != nil { + return "", err + } + base = value + } + if memory.Off != 0 { + adjusted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", adjusted, base, memory.Off) + base = "%" + adjusted + } + return base, nil +} + +func (c *amd64Ctx) gatherPointer(memory MemRef, address string) (string, string) { + if memory.Segment == "" { + return c.ptrFromAddrI64(address), "ptr" + } + addressSpace := 257 + if memory.Segment == GS { + addressSpace = 256 + } + pointerType := fmt.Sprintf("ptr addrspace(%d)", addressSpace) + pointer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to %s\n", pointer, address, pointerType) + return "%" + pointer, pointerType +} diff --git a/amd64_lower_getexp.go b/amd64_lower_getexp.go new file mode 100644 index 000000000..01d31f84f --- /dev/null +++ b/amd64_lower_getexp.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64GetExpSpec struct { + laneBits int + scalar bool +} + +var amd64GetExpSpecs = map[Op]amd64GetExpSpec{ + "VGETEXPPS": {laneBits: 32}, + "VGETEXPPD": {laneBits: 64}, + "VGETEXPSS": {laneBits: 32, scalar: true}, + "VGETEXPSD": {laneBits: 64, scalar: true}, +} + +func (c *amd64Ctx) lowerGetExp(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64GetExpSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's GETEXP optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && properties.broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + wantArgs := 2 + if spec.scalar { + wantArgs = 3 + } + if len(ins.Args) != wantArgs && len(ins.Args) != wantArgs+1 { + return true, false, fmt.Errorf("%s %s has the wrong operand count for its packed/scalar grammar: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == wantArgs+1 + if c.goarch == "386" && spec.scalar && masked { + return true, false, fmt.Errorf("386 %s mask form exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + maskIndex := len(ins.Args) - 2 + if masked && !amd64NonzeroKOperand(ins.Args[maskIndex]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if spec.scalar { + if byteWidth != 16 || !c.isGoEVEXVectorRegister(destination, 16) || !c.isGoEVEXVectorRegister(ins.Args[1], 16) { + return true, false, fmt.Errorf("%s %s scalar pass-through and destination must be X registers: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s packed destination must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae { + if source.Kind != OpReg || !spec.scalar && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s.SAE requires a register source and scalar or Z form: %q", c.goarch, baseOp, ins.Raw) + } + } + + mask := "" + if masked { + mask, err = c.loadK(ins.Args[maskIndex].Reg) + if err != nil { + return true, false, err + } + } + daz := c.loadMXCSRDAZ() + if spec.scalar { + return c.lowerScalarGetExp(spec, properties, source, ins.Args[1], destination, mask, daz) + } + return c.lowerPackedGetExp(spec, properties, source, destination, byteWidth, mask, daz) +} + +func (c *amd64Ctx) lowerPackedGetExp(spec amd64GetExpSpec, properties amd64BinaryFloatingSuffix, source, destination Operand, byteWidth int, mask, daz string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + input, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + result := c.emitGetExpBits(lanes, spec.laneBits, input, daz) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarGetExp(spec amd64GetExpSpec, properties amd64BinaryFloatingSuffix, source, passthrough, destination Operand, mask, daz string) (bool, bool, error) { + sourceBits, err := c.loadFloatingScalarBits(source, spec.laneBits) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", input, spec.laneBits, spec.laneBits, sourceBits) + result := c.emitGetExpBits(1, spec.laneBits, "%"+input, daz) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(passthrough.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitGetExpBits(lanes, laneBits int, input, daz string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + floatType := amd64FloatingScalarType(laneBits) + mantissaBits, bias, denormalOffset := 23, 127, 149 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + if laneBits == 64 { + mantissaBits, bias, denormalOffset = 52, 1023, 1074 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + } + positiveInfinity := exponentMask + negativeInfinity := signBit | exponentMask + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + bits, exponent, mantissa := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", bits, vectorType, input, lane) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, bits, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, bits, mantissaMask) + exponentZero, exponentOnes := c.newTmp(), c.newTmp() + mantissaZero, mantissaNonzero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", mantissaNonzero, laneBits, mantissa) + isNaN, isInfinity, isDenormal := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isDenormal, exponentZero, mantissaNonzero) + zeroOrDAZ, isEffectiveZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %s\n", zeroOrDAZ, mantissaZero, daz) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isEffectiveZero, exponentZero, zeroOrDAZ) + + normalField, normalExponent := c.newTmp(), c.newTmp() + leadingZeros, highestBit, denormalExponent := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", normalField, laneBits, exponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", normalExponent, laneBits, normalField, bias) + fmt.Fprintf(c.b, " %%%s = call i%d @llvm.ctlz.i%d(i%d %%%s, i1 false)\n", leadingZeros, laneBits, laneBits, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = sub i%d %d, %%%s\n", highestBit, laneBits, laneBits-1, leadingZeros) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", denormalExponent, laneBits, highestBit, denormalOffset) + exponentValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", exponentValue, isDenormal, laneBits, denormalExponent, laneBits, normalExponent) + exponentFloat, exponentBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp i%d %%%s to %s\n", exponentFloat, laneBits, exponentValue, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to i%d\n", exponentBits, floatType, exponentFloat, laneBits) + withInfinity, withZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %%%s\n", withInfinity, isInfinity, laneBits, positiveInfinity, laneBits, exponentBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %%%s\n", withZero, isEffectiveZero, laneBits, negativeInfinity, laneBits, withInfinity) + quietNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quietNaN, laneBits, bits, quietBit) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", selected, isNaN, laneBits, quietNaN, laneBits, withZero) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, vectorType, result, laneBits, selected, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_getexp_test.go b/amd64_lower_getexp_test.go new file mode 100644 index 000000000..76033dc65 --- /dev/null +++ b/amd64_lower_getexp_test.go @@ -0,0 +1,274 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64GetExpGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64GetExpSpec{ + "VGETEXPPS": {laneBits: 32}, + "VGETEXPPD": {laneBits: 64}, + "VGETEXPSS": {laneBits: 32, scalar: true}, + "VGETEXPSD": {laneBits: 64, scalar: true}, + } + if len(amd64GetExpSpecs) != len(expected) { + t.Fatalf("GETEXP grammar has %d entries, want %d", len(amd64GetExpSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64GetExpSpecs[op]; !ok { + t.Errorf("GETEXP grammar omitted %s", op) + } else if got != want { + t.Errorf("GETEXP grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86GetExpCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT getexpforms(SB),$0-0\n") + for _, op := range []string{"VGETEXPPS", "VGETEXPPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 24(AX), K2, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(AX), K3, %s%d\n", op, width, last) + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE Z1, Z%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z Z1, K4, Z%d\n", op, last) + } + } + } + for _, op := range []string{"VGETEXPSS", "VGETEXPSD"} { + last := 31 + fmt.Fprintf(&source, "\t%s X1, X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s 40(AX), X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE X1, X2, X%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X2, K5, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.Z 48(AX), X2, K6, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z X1, X2, K7, X%d\n", op, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"getexpforms": {Name: "getexpforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "getexp-"+target.name+".ll", "getexp-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86GetExpRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VGETEXPPS X0"}, + {goarch: "amd64", instruction: "VGETEXPPS X0, Y1"}, + {goarch: "amd64", instruction: "VGETEXPPS.BCST X0, X1"}, + {goarch: "amd64", instruction: "VGETEXPPS.SAE X0, X1"}, + {goarch: "amd64", instruction: "VGETEXPPS.SAE 0(AX), Z1"}, + {goarch: "amd64", instruction: "VGETEXPPS.RN_SAE Z0, Z1"}, + {goarch: "amd64", instruction: "VGETEXPPS.Z X0, X1"}, + {goarch: "amd64", instruction: "VGETEXPPS X0, K0, X1"}, + {goarch: "amd64", instruction: "VGETEXPSS Y0, X1, X2"}, + {goarch: "amd64", instruction: "VGETEXPSS.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VGETEXPSD X0, X1, Y2"}, + {goarch: "386", instruction: "VGETEXPSS X0, X1, K1, X2"}, + {goarch: "386", instruction: "VGETEXPPS Z8, Z0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's GETEXP table", test.instruction) + } + }) + } +} + +func TestAMD64GetExpRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT getexpsemantics(SB),$0-56 + MOVQ out+0(FP), AX + MOVQ source32+8(FP), BX + MOVQ source64+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ old64+32(FP), R8 + MOVQ upper+40(FP), R9 + MOVQ mask+48(FP), R10 + KMOVQ R10, K1 + STC + VGETEXPPS 0(BX), Z0 + VMOVDQU32 Z0, 0(AX) + VMOVDQU32 0(DX), Z1 + VGETEXPPS 0(BX), K1, Z1 + VMOVDQU32 Z1, 64(AX) + VGETEXPPS.Z 0(BX), K1, Z1 + VMOVDQU32 Z1, 128(AX) + VGETEXPPS.BCST 0(BX), Z1 + VMOVDQU32 Z1, 192(AX) + VGETEXPPD 0(CX), Z2 + VMOVDQU64 Z2, 256(AX) + VMOVDQU64 0(R8), Z3 + VGETEXPPD 0(CX), K1, Z3 + VMOVDQU64 Z3, 320(AX) + VGETEXPPD.Z 0(CX), K1, Z3 + VMOVDQU64 Z3, 384(AX) + VGETEXPPD.BCST 0(CX), Z3 + VMOVDQU64 Z3, 448(AX) + VMOVDQU64 0(R9), X4 + VGETEXPSS 40(BX), X4, X5 + VMOVDQU64 X5, 512(AX) + VGETEXPSD 40(CX), X4, X6 + VMOVDQU64 X6, 528(AX) + SETCS 544(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"getexpsemantics": { + Name: "getexpsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: I64, Index: 6, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void getexpsemantics(uint8_t *, const uint32_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t f32bits(float f) { uint32_t u; memcpy(&u,&f,4); return u; } +static uint64_t f64bits(double f) { uint64_t u; memcpy(&u,&f,8); return u; } +static uint32_t getexp32(uint32_t u, int daz) { + uint32_t exp=u&UINT32_C(0x7f800000), mant=u&UINT32_C(0x007fffff); + if (exp==UINT32_C(0x7f800000)) return mant?(u|UINT32_C(0x00400000)):UINT32_C(0x7f800000); + if (exp==0 && (mant==0||daz)) return UINT32_C(0xff800000); + int e; + if (exp) e=(int)(exp>>23)-127; + else { int p=31-__builtin_clz(mant); e=p-149; } + return f32bits((float)e); +} +static uint64_t getexp64(uint64_t u, int daz) { + uint64_t exp=u&UINT64_C(0x7ff0000000000000), mant=u&UINT64_C(0x000fffffffffffff); + if (exp==UINT64_C(0x7ff0000000000000)) return mant?(u|UINT64_C(0x0008000000000000)):UINT64_C(0x7ff0000000000000); + if (exp==0 && (mant==0||daz)) return UINT64_C(0xfff0000000000000); + int e; + if (exp) e=(int)(exp>>52)-1023; + else { int p=63-__builtin_clzll(mant); e=p-1074; } + return f64bits((double)e); +} +int main(void) { + const uint32_t source32[16]={ + 0x3f800000,0x40000000,0x3f400000,0xc1000000,0,0x80000000,0x7f800000,0xff800000, + 0x7fc12345,0x7f812345,1,0x007fffff,0x80000001,0x00800000,0x7f7fffff,0x3e800000}; + const uint64_t source64[8]={ + UINT64_C(0x3ff0000000000000),UINT64_C(0x4000000000000000),UINT64_C(0x3fe8000000000000),UINT64_C(0xc020000000000000), + 0,1,UINT64_C(0x000fffffffffffff),UINT64_C(0x8000000000000001)}; + uint32_t old32[16]; uint64_t old64[8]; + const uint64_t upper[2]={UINT64_C(0x1122334455667788),UINT64_C(0x99aabbccddeeff00)}; + uint8_t out[545]={0}, outDaz[545]={0}; const uint64_t mask=UINT64_C(0xa55a); + for (int i=0;i<16;i++) old32[i]=UINT32_C(0x41000000)+(uint32_t)i; + for (int i=0;i<8;i++) old64[i]=UINT64_C(0x4020000000000000)+(uint64_t)i; + getexpsemantics(out,source32,source64,old32,old64,upper,mask); + unsigned csr=_mm_getcsr(); _mm_setcsr(csr|UINT32_C(0x40)); + getexpsemantics(outDaz,source32,source64,old32,old64,upper,mask); _mm_setcsr(csr); + for (int i=0;i<16;i++) { + uint32_t want=getexp32(source32[i],0), got; memcpy(&got,out+4*i,4); if (got!=want) return 10; + memcpy(&got,out+64+4*i,4); if (got!=(((mask>>i)&1)?want:old32[i])) return 11; + memcpy(&got,out+128+4*i,4); if (got!=(((mask>>i)&1)?want:0)) return 12; + memcpy(&got,out+192+4*i,4); if (got!=getexp32(source32[0],0)) return 13; + } + for (int i=0;i<8;i++) { + uint64_t want=getexp64(source64[i],0), got; memcpy(&got,out+256+8*i,8); if (got!=want) return 20; + memcpy(&got,out+320+8*i,8); if (got!=(((mask>>i)&1)?want:old64[i])) return 21; + memcpy(&got,out+384+8*i,8); if (got!=(((mask>>i)&1)?want:0)) return 22; + memcpy(&got,out+448+8*i,8); if (got!=getexp64(source64[0],0)) return 23; + } + uint32_t scalar32; memcpy(&scalar32,out+512,4); if (scalar32!=getexp32(source32[10],0)) return 30; + if (memcmp(out+516,(uint8_t*)upper+4,12)) return 31; + uint64_t scalar64; memcpy(&scalar64,out+528,8); if (scalar64!=getexp64(source64[5],0)) return 32; + if (memcmp(out+536,(uint8_t*)upper+8,8)) return 33; + if (out[544]!=1) return 34; + uint32_t daz32; memcpy(&daz32,outDaz+40,4); if (daz32!=UINT32_C(0xff800000)) return 40; + uint64_t daz64; memcpy(&daz64,outDaz+296,8); if (daz64!=UINT64_C(0xfff0000000000000)) return 41; + memcpy(&daz32,outDaz+512,4); if (daz32!=UINT32_C(0xff800000)) return 42; + memcpy(&daz64,outDaz+528,8); if (daz64!=UINT64_C(0xfff0000000000000)) return 43; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "getexp_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_getmant.go b/amd64_lower_getmant.go new file mode 100644 index 000000000..edc07c92f --- /dev/null +++ b/amd64_lower_getmant.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64GetMantSpec = amd64UnaryImmediateFloatingShape + +var amd64GetMantSpecs = map[Op]amd64GetMantSpec{ + "VGETMANTPS": {laneBits: 32}, + "VGETMANTPD": {laneBits: 64}, + "VGETMANTSS": {laneBits: 32, scalar: true}, + "VGETMANTSD": {laneBits: 64, scalar: true}, +} + +func (c *amd64Ctx) lowerGetMant(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64GetMantSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + form, err := c.parseUnaryImmediateFloatingForm(baseOp, suffix, spec, ins) + if err != nil { + return true, false, err + } + daz := c.loadMXCSRDAZ() + if spec.scalar { + return c.lowerScalarGetMant(spec, form.properties, form.source, form.passthrough, form.destination, form.mask, daz, form.immediate) + } + return c.lowerPackedGetMant(spec, form.properties, form.source, form.destination, form.byteWidth, form.mask, daz, form.immediate) +} + +func (c *amd64Ctx) lowerPackedGetMant(spec amd64GetMantSpec, properties amd64BinaryFloatingSuffix, source, destination Operand, byteWidth int, mask, daz string, immediate uint8) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + input, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + result := c.emitGetMantBits(lanes, spec.laneBits, input, daz, immediate) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarGetMant(spec amd64GetMantSpec, properties amd64BinaryFloatingSuffix, source, passthrough, destination Operand, mask, daz string, immediate uint8) (bool, bool, error) { + sourceBits, err := c.loadFloatingScalarBits(source, spec.laneBits) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", input, spec.laneBits, spec.laneBits, sourceBits) + result := c.emitGetMantBits(1, spec.laneBits, "%"+input, daz, immediate) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(passthrough.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitGetMantBits(lanes, laneBits int, input, daz string, immediate uint8) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + mantissaBits, bias := 23, 127 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + indefinite := uint64(0xffc00000) + if laneBits == 64 { + mantissaBits, bias = 52, 1023 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + indefinite = uint64(0xfff8000000000000) + } + interval := immediate & 3 + signControl := immediate >> 2 & 3 + positiveOne := uint64(bias) << mantissaBits + signedOne := positiveOne + if signControl&1 == 0 { + signedOne |= signBit + } + dazClear := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i1 %s, false\n", dazClear, daz) + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + bits, exponent, mantissa, sign := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", bits, vectorType, input, lane) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, bits, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, bits, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", sign, laneBits, bits, signBit) + exponentZero, exponentOnes := c.newTmp(), c.newTmp() + mantissaZero, mantissaNonzero, negative := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", mantissaNonzero, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", negative, laneBits, sign) + isNaN, isInfinity, isDenormal := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isDenormal, exponentZero, mantissaNonzero) + activeDenormal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", activeDenormal, isDenormal, dazClear) + zeroOrDAZ, effectiveZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %s\n", zeroOrDAZ, mantissaZero, daz) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", effectiveZero, exponentZero, zeroOrDAZ) + + leadingZeros, shiftCount, shifted, normalizedFraction := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i%d @llvm.ctlz.i%d(i%d %%%s, i1 false)\n", leadingZeros, laneBits, laneBits, laneBits, mantissa) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", shiftCount, laneBits, leadingZeros, laneBits-mantissaBits-1) + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %%%s\n", shifted, laneBits, mantissa, shiftCount) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", normalizedFraction, laneBits, shifted, mantissaMask) + fraction := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", fraction, activeDenormal, laneBits, normalizedFraction, laneBits, mantissa) + exponentField, unbiasedNormal, unbiasedDenormal, unbiased := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", exponentField, laneBits, exponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, %d\n", unbiasedNormal, laneBits, exponentField, bias) + fmt.Fprintf(c.b, " %%%s = sub i%d 0, %%%s\n", unbiasedDenormal, laneBits, shiftCount) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", unbiased, activeDenormal, laneBits, unbiasedDenormal, laneBits, unbiasedNormal) + oddValue, odd := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 1\n", oddValue, laneBits, unbiased) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", odd, laneBits, oddValue) + fractionTop, topSet := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", fractionTop, laneBits, fraction, quietBit) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", topSet, laneBits, fractionTop) + normalizedExponent := fmt.Sprintf("%d", bias) + switch interval { + case 1: + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %d\n", selected, odd, laneBits, bias-1, laneBits, bias) + normalizedExponent = "%" + selected + case 2: + normalizedExponent = fmt.Sprintf("%d", bias-1) + case 3: + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %d\n", selected, topSet, laneBits, bias-1, laneBits, bias) + normalizedExponent = "%" + selected + } + exponentBits, magnitude := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %s, %d\n", exponentBits, laneBits, normalizedExponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", magnitude, laneBits, exponentBits, fraction) + computed := "%" + magnitude + if signControl&1 == 0 { + withSign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", withSign, laneBits, magnitude, sign) + computed = "%" + withSign + } + zeroOrInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", zeroOrInfinity, effectiveZero, isInfinity) + specialOne := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %d\n", specialOne, negative, laneBits, signedOne, laneBits, positiveOne) + withSpecial := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %s\n", withSpecial, zeroOrInfinity, laneBits, specialOne, laneBits, computed) + selected := "%" + withSpecial + if signControl&2 != 0 { + notZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notZero, effectiveZero) + nonzeroNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", nonzeroNegative, negative, notZero) + indefiniteResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %s\n", indefiniteResult, nonzeroNegative, laneBits, indefinite, laneBits, selected) + selected = "%" + indefiniteResult + } + quietNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quietNaN, laneBits, bits, quietBit) + final := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %s\n", final, isNaN, laneBits, quietNaN, laneBits, selected) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, vectorType, result, laneBits, final, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_getmant_test.go b/amd64_lower_getmant_test.go new file mode 100644 index 000000000..c7fbd93f8 --- /dev/null +++ b/amd64_lower_getmant_test.go @@ -0,0 +1,306 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64GetMantGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64GetMantSpec{ + "VGETMANTPS": {laneBits: 32}, + "VGETMANTPD": {laneBits: 64}, + "VGETMANTSS": {laneBits: 32, scalar: true}, + "VGETMANTSD": {laneBits: 64, scalar: true}, + } + if len(amd64GetMantSpecs) != len(expected) { + t.Fatalf("GETMANT grammar has %d entries, want %d", len(amd64GetMantSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64GetMantSpecs[op]; !ok { + t.Errorf("GETMANT grammar omitted %s", op) + } else if got != want { + t.Errorf("GETMANT grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86GetMantCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT getmantforms(SB),$0-0\n") + for _, op := range []string{"VGETMANTPS", "VGETMANTPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST $1, 16(AX), %s%d\n", op, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $2, %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z $3, 24(AX), K2, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(AX), K3, %s%d\n", op, width, last) + } + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE $5, Z1, Z%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.SAE.Z $6, Z1, K4, Z%d\n", op, last) + } + } + } + } + if target.goarch == "amd64" { + for _, op := range []string{"VGETMANTSS", "VGETMANTSD"} { + fmt.Fprintf(&source, "\t%s $0, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $255, 40(AX), X2, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE $1, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $2, X1, X2, K5, X31\n", op) + fmt.Fprintf(&source, "\t%s.Z $3, 48(AX), X2, K6, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE.Z $4, X1, X2, K7, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"getmantforms": {Name: "getmantforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "getmant-"+target.name+".ll", "getmant-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86GetMantRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VGETMANTPS $-1, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS $256, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS $0, X0, Y1"}, + {goarch: "amd64", instruction: "VGETMANTPS.BCST $0, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS.SAE $0, Z0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS.SAE $0, 0(AX), Z1"}, + {goarch: "amd64", instruction: "VGETMANTPS.RN_SAE $0, Z0, Z1"}, + {goarch: "amd64", instruction: "VGETMANTPS.Z $0, X0, X1"}, + {goarch: "amd64", instruction: "VGETMANTPS $0, X0, K0, X1"}, + {goarch: "amd64", instruction: "VGETMANTSS $0, Y0, X1, X2"}, + {goarch: "amd64", instruction: "VGETMANTSS.BCST $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VGETMANTSD $0, X0, X1, Y2"}, + {goarch: "386", instruction: "VGETMANTSS $0, X0, X1, X2"}, + {goarch: "386", instruction: "VGETMANTPS $0, X0, K1, X2"}, + {goarch: "386", instruction: "VGETMANTPS $0, Z8, Z0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's GETMANT table", test.instruction) + } + }) + } +} + +func TestAMD64GetMantRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + source.WriteString(`TEXT getmantsemantics(SB),$0-56 + MOVQ out+0(FP), AX + MOVQ source32+8(FP), BX + MOVQ source64+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ old64+32(FP), R8 + MOVQ upper+40(FP), R9 + MOVQ mask+48(FP), R10 + KMOVQ R10, K1 + STC +`) + for immediate := 0; immediate < 16; immediate++ { + fmt.Fprintf(&source, "\tVGETMANTPS $%d, 0(BX), Z0\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU32 Z0, %d(AX)\n", immediate*64) + fmt.Fprintf(&source, "\tVGETMANTPD $%d, 0(CX), Z1\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU64 Z1, %d(AX)\n", 1024+immediate*64) + } + source.WriteString(` VMOVDQU32 0(DX), Z2 + VGETMANTPS $3, 0(BX), K1, Z2 + VMOVDQU32 Z2, 2048(AX) + VGETMANTPS.Z $3, 0(BX), K1, Z2 + VMOVDQU32 Z2, 2112(AX) + VGETMANTPS.BCST $5, 0(BX), Z2 + VMOVDQU32 Z2, 2176(AX) + VMOVDQU64 0(R8), Z3 + VGETMANTPD $3, 0(CX), K1, Z3 + VMOVDQU64 Z3, 2240(AX) + VGETMANTPD.Z $3, 0(CX), K1, Z3 + VMOVDQU64 Z3, 2304(AX) + VGETMANTPD.BCST $5, 0(CX), Z3 + VMOVDQU64 Z3, 2368(AX) + VMOVDQU64 0(R9), X4 + VGETMANTSS $11, 12(BX), X4, X5 + VMOVDQU64 X5, 2432(AX) + VGETMANTSD $11, 24(CX), X4, X6 + VMOVDQU64 X6, 2448(AX) + SETCS 2464(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"getmantsemantics": { + Name: "getmantsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: I64, Index: 6, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void getmantsemantics(uint8_t *, const uint32_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t getmant32(uint32_t u, unsigned imm, int daz) { + const uint32_t em=UINT32_C(0x7f800000), fm=UINT32_C(0x007fffff), qb=UINT32_C(0x00400000), sb=UINT32_C(0x80000000); + uint32_t exp=u&em, frac=u&fm, sign=u&sb; unsigned sc=(imm>>2)&3, interval=imm&3; + int nan=exp==em&&frac!=0, inf=exp==em&&frac==0, zero=exp==0&&(frac==0||daz), neg=sign!=0; + if (nan) return u|qb; + uint32_t one=UINT32_C(0x3f800000), signedOne=one|((sc&1)?0:sign); + if (neg && !zero && (sc&2)) return UINT32_C(0xffc00000); + if (zero||inf) return neg?signedOne:one; + int unbiased; + if (exp==0) { int shift=__builtin_clz(frac)-8; frac=(frac<>23)-127; + uint32_t outExp=127; + if (interval==1) outExp=(unbiased&1)?126:127; + else if (interval==2) outExp=126; + else if (interval==3) outExp=(frac&qb)?126:127; + return (outExp<<23)|frac|((sc&1)?0:sign); +} +static uint64_t getmant64(uint64_t u, unsigned imm, int daz) { + const uint64_t em=UINT64_C(0x7ff0000000000000), fm=UINT64_C(0x000fffffffffffff), qb=UINT64_C(0x0008000000000000), sb=UINT64_C(0x8000000000000000); + uint64_t exp=u&em, frac=u&fm, sign=u&sb; unsigned sc=(imm>>2)&3, interval=imm&3; + int nan=exp==em&&frac!=0, inf=exp==em&&frac==0, zero=exp==0&&(frac==0||daz), neg=sign!=0; + if (nan) return u|qb; + uint64_t one=UINT64_C(0x3ff0000000000000), signedOne=one|((sc&1)?0:sign); + if (neg && !zero && (sc&2)) return UINT64_C(0xfff8000000000000); + if (zero||inf) return neg?signedOne:one; + int unbiased; + if (exp==0) { int shift=__builtin_clzll(frac)-11; frac=(frac<>52)-1023; + uint64_t outExp=1023; + if (interval==1) outExp=(unbiased&1)?1022:1023; + else if (interval==2) outExp=1022; + else if (interval==3) outExp=(frac&qb)?1022:1023; + return (outExp<<52)|frac|((sc&1)?0:sign); +} +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + const uint32_t source32[16]={ + 0x3f800000,0xbf800000,0x40400000,0xc0400000,0x3f400000,0xbf400000,0,0x80000000, + 0x7f800000,0xff800000,0x7fc12345,0xff812345,1,0x807fffff,0x00800000,0x7f7fffff}; + const uint64_t source64[8]={ + UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x4008000000000000),UINT64_C(0xc008000000000000), + 0,UINT64_C(0x8000000000000000),UINT64_C(1),UINT64_C(0xfff0000000000000)}; + uint32_t old32[16]; uint64_t old64[8]; + const uint64_t upper[2]={UINT64_C(0x1122334455667788),UINT64_C(0x99aabbccddeeff00)}; + uint8_t out[2465]={0}, outDaz[2465]={0}; const uint64_t mask=UINT64_C(0xa55a); + for (int i=0;i<16;i++) old32[i]=UINT32_C(0x41000000)+(uint32_t)i; + for (int i=0;i<8;i++) old64[i]=UINT64_C(0x4020000000000000)+(uint64_t)i; + getmantsemantics(out,source32,source64,old32,old64,upper,mask); + unsigned csr=_mm_getcsr(); _mm_setcsr(csr|UINT32_C(0x40)); + getmantsemantics(outDaz,source32,source64,old32,old64,upper,mask); _mm_setcsr(csr); + for (unsigned imm=0;imm<16;imm++) { + for (int i=0;i<16;i++) { + if (u32(out+imm*64+i*4)!=getmant32(source32[i],imm,0)) return 10; + if (u32(outDaz+imm*64+i*4)!=getmant32(source32[i],imm,1)) return 11; + } + for (int i=0;i<8;i++) { + if (u64(out+1024+imm*64+i*8)!=getmant64(source64[i],imm,0)) return 12; + if (u64(outDaz+1024+imm*64+i*8)!=getmant64(source64[i],imm,1)) return 13; + } + } + for (int i=0;i<16;i++) { + uint32_t want=getmant32(source32[i],3,0); + if (u32(out+2048+i*4)!=(((mask>>i)&1)?want:old32[i])) return 20; + if (u32(out+2112+i*4)!=(((mask>>i)&1)?want:0)) return 21; + if (u32(out+2176+i*4)!=getmant32(source32[0],5,0)) return 22; + } + for (int i=0;i<8;i++) { + uint64_t want=getmant64(source64[i],3,0); + if (u64(out+2240+i*8)!=(((mask>>i)&1)?want:old64[i])) return 23; + if (u64(out+2304+i*8)!=(((mask>>i)&1)?want:0)) return 24; + if (u64(out+2368+i*8)!=getmant64(source64[0],5,0)) return 25; + } + if (u32(out+2432)!=getmant32(source32[3],11,0)) return 30; + if (memcmp(out+2436,(const uint8_t *)upper+4,12)) return 31; + if (u64(out+2448)!=getmant64(source64[3],11,0)) return 32; + if (memcmp(out+2456,(const uint8_t *)upper+8,8)) return 33; + if (out[2464]!=1) return 34; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "getmant_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_gfni.go b/amd64_lower_gfni.go new file mode 100644 index 000000000..7eaa56c1b --- /dev/null +++ b/amd64_lower_gfni.go @@ -0,0 +1,248 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64GFNIMode uint8 + +const ( + amd64GFNIMultiply amd64GFNIMode = iota + amd64GFNIAffine + amd64GFNIAffineInverse +) + +type amd64GFNISpec struct { + mode amd64GFNIMode + immediate bool + broadcast bool +} + +// amd64GFNISpecs models the whole Go 1.27 GFNI family. VGF2P8MULB uses the +// eight _yvandnpd rows; both affine instructions use the eight +// _yvgf2p8affineinvqb rows, including qword broadcast for their matrix source. +var amd64GFNISpecs = map[Op]amd64GFNISpec{ + "VGF2P8MULB": {mode: amd64GFNIMultiply}, + "VGF2P8AFFINEQB": {mode: amd64GFNIAffine, immediate: true, broadcast: true}, + "VGF2P8AFFINEINVQB": {mode: amd64GFNIAffineInverse, immediate: true, broadcast: true}, +} + +func (c *amd64Ctx) lowerGFNI(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64GFNISpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's GFNI table: %q", c.goarch, baseOp, ins.Raw) + } + if broadcast && !spec.broadcast { + return true, false, fmt.Errorf("%s %s does not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + // The 386 assembler frontend can encode the three-operand multiply forms, + // but cannot represent the affine family's four-operand minimum. + if c.goarch == "386" && spec.immediate && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + + minimumArgs := 3 + if spec.immediate { + minimumArgs = 4 + } + if len(ins.Args) != minimumArgs && len(ins.Args) != minimumArgs+1 { + return true, false, fmt.Errorf("%s %s has the wrong operand count for Go 1.27's GFNI table: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == minimumArgs+1 + if c.goarch == "386" && masked && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s masked form exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + + operandOffset := 0 + immediate := uint8(0) + if spec.immediate { + if !amd64UnsignedImmediate(ins.Args[0], 8) { + return true, false, fmt.Errorf("%s %s expects an unsigned 8-bit immediate: %q", c.goarch, baseOp, ins.Raw) + } + immediate = uint8(ins.Args[0].Imm) + operandOffset = 1 + } + first := ins.Args[operandOffset] + second := ins.Args[operandOffset+1] + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's vector-register class: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoPackedVectorMoveRegister(second, byteWidth) { + return true, false, fmt.Errorf("%s %s second source must be a same-width vector register: %q", c.goarch, baseOp, ins.Raw) + } + if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("%s %s first source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s first source must be a vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if broadcast && !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory matrix source: %q", c.goarch, baseOp, ins.Raw) + } + + var mask string + if masked { + maskArg := ins.Args[len(ins.Args)-2] + if !amd64NonzeroKOperand(maskArg) { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + firstBytes, err := c.loadGFNIFirstSource(first, byteWidth, broadcast) + if err != nil { + return true, false, err + } + secondBytes, err := c.loadPackedCompareBytes(second, byteWidth) + if err != nil { + return true, false, err + } + computed := "" + switch spec.mode { + case amd64GFNIMultiply: + computed = c.emitGF2P8Multiply(byteWidth, firstBytes, secondBytes) + case amd64GFNIAffine, amd64GFNIAffineInverse: + if spec.mode == amd64GFNIAffineInverse { + secondBytes = c.emitGF2P8Inverse(byteWidth, secondBytes) + } + computed = c.emitGF2P8Affine(byteWidth, firstBytes, secondBytes, immediate) + } + if masked { + old, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + computed = amd64ApplyIntegerLaneMask(c, byteWidth, 8, computed, old, mask, zeroing) + } + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, computed) +} + +func (c *amd64Ctx) loadGFNIFirstSource(source Operand, byteWidth int, broadcast bool) (string, error) { + if !broadcast { + return c.loadPackedCompareBytes(source, byteWidth) + } + qwords, err := c.loadPackedCompareLanes(source, byteWidth, 64, true) + if err != nil { + return "", err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytes, byteWidth/8, qwords, byteWidth) + return "%" + bytes, nil +} + +// emitGF2P8Multiply performs lane-wise carry-less multiplication reduced by +// x^8+x^4+x^3+x+1 (0x11b). Generic IR keeps translated programs runnable on +// hosts without GFNI while preserving exactly the instruction's byte lanes. +func (c *amd64Ctx) emitGF2P8Multiply(lanes int, first, second string) string { + vectorType := fmt.Sprintf("<%d x i8>", lanes) + result := "zeroinitializer" + multiplicand, multiplier := first, second + for bit := 0; bit < 8; bit++ { + lowBit := c.newTmp() + active := c.newTmp() + selected := c.newTmp() + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", lowBit, vectorType, multiplier, llvmSplatInteger(lanes, 8, 1)) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", active, vectorType, lowBit) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s zeroinitializer\n", selected, lanes, active, vectorType, multiplicand, vectorType) + fmt.Fprintf(c.b, " %%%s = xor %s %s, %%%s\n", combined, vectorType, result, selected) + result = "%" + combined + if bit == 7 { + break + } + highBit := c.newTmp() + reduce := c.newTmp() + shifted := c.newTmp() + reduced := c.newTmp() + nextMultiplicand := c.newTmp() + nextMultiplier := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", highBit, vectorType, multiplicand, llvmSplatInteger(lanes, 8, 0x80)) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", reduce, vectorType, highBit) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", shifted, vectorType, multiplicand, llvmSplatInteger(lanes, 8, 1)) + fmt.Fprintf(c.b, " %%%s = xor %s %%%s, %s\n", reduced, vectorType, shifted, llvmSplatInteger(lanes, 8, 0x1b)) + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %%%s, %s %%%s\n", nextMultiplicand, lanes, reduce, vectorType, reduced, vectorType, shifted) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", nextMultiplier, vectorType, multiplier, llvmSplatInteger(lanes, 8, 1)) + multiplicand, multiplier = "%"+nextMultiplicand, "%"+nextMultiplier + } + return result +} + +func (c *amd64Ctx) emitGF2P8Inverse(lanes int, source string) string { + powers := make([]string, 7) + powers[0] = c.emitGF2P8Multiply(lanes, source, source) + for i := 1; i < len(powers); i++ { + powers[i] = c.emitGF2P8Multiply(lanes, powers[i-1], powers[i-1]) + } + result := powers[0] + for i := 1; i < len(powers); i++ { + result = c.emitGF2P8Multiply(lanes, result, powers[i]) + } + return result +} + +func (c *amd64Ctx) emitGF2P8Affine(lanes int, matrix, data string, immediate uint8) string { + vectorType := fmt.Sprintf("<%d x i8>", lanes) + result := "zeroinitializer" + for bit := 0; bit < 8; bit++ { + row := c.newTmp() + masked := c.newTmp() + parities := c.newTmp() + oneBit := c.newTmp() + positioned := c.newTmp() + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector %s %s, %s poison, <%d x i32> %s\n", row, vectorType, matrix, vectorType, lanes, llvmGFNIMatrixRowMask(lanes, bit)) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", masked, vectorType, data, row) + fmt.Fprintf(c.b, " %%%s = call %s @llvm.ctpop.v%di8(%s %%%s)\n", parities, vectorType, lanes, vectorType, masked) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", oneBit, vectorType, parities, llvmSplatInteger(lanes, 8, 1)) + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", positioned, vectorType, oneBit, llvmSplatInteger(lanes, 8, uint64(bit))) + fmt.Fprintf(c.b, " %%%s = or %s %s, %%%s\n", combined, vectorType, result, positioned) + result = "%" + combined + } + withImmediate := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", withImmediate, vectorType, result, llvmSplatInteger(lanes, 8, uint64(immediate))) + return "%" + withImmediate +} + +func llvmGFNIMatrixRowMask(lanes, bit int) string { + var b strings.Builder + b.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "i32 %d", lane/8*8+7-bit) + } + b.WriteByte('>') + return b.String() +} diff --git a/amd64_lower_gfni_test.go b/amd64_lower_gfni_test.go new file mode 100644 index 000000000..8c54ce2d2 --- /dev/null +++ b/amd64_lower_gfni_test.go @@ -0,0 +1,263 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64GFNIGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64GFNISpec{ + "VGF2P8MULB": {mode: amd64GFNIMultiply}, + "VGF2P8AFFINEQB": {mode: amd64GFNIAffine, immediate: true, broadcast: true}, + "VGF2P8AFFINEINVQB": {mode: amd64GFNIAffineInverse, immediate: true, broadcast: true}, + } + if len(amd64GFNISpecs) != len(expected) { + t.Fatalf("GFNI grammar has %d entries, want %d", len(amd64GFNISpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64GFNISpecs[op]; !ok { + t.Errorf("GFNI grammar omitted %s", op) + } else if got != want { + t.Errorf("GFNI grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86GFNICompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT gfniforms(SB),$0-0\n") + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\tVGF2P8MULB %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8MULB 8(AX), %s2, %s%d\n", width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB $0x63, %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB $0x63, 16(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB $0xa5, %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB $0xa5, 24(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB.BCST $0x63, 32(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB.BCST $0xa5, 40(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8MULB %s1, %s2, K1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8MULB.Z 48(AX), %s2, K2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB $0x63, %s1, %s2, K3, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB.Z $0x63, 56(AX), %s2, K4, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB $0xa5, %s1, %s2, K5, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB.Z $0xa5, 64(AX), %s2, K6, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEQB.BCST.Z $0x63, 72(AX), %s2, K7, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVGF2P8AFFINEINVQB.BCST.Z $0xa5, 80(AX), %s2, K1, %s%d\n", width, width, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"gfniforms": {Name: "gfniforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "gfni-"+target.name+".ll", "gfni-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86GFNIRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VGF2P8MULB X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VGF2P8MULB X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VGF2P8MULB.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8MULB.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8MULB X0, X1, 0(AX)"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEINVQB $1, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB.BCST $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEINVQB.Z.BCST $1, 0(AX), X1, K1, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB $1, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VGF2P8AFFINEQB $1, X0, X1"}, + {goarch: "386", instruction: "VGF2P8MULB X0, X1, K1, X2"}, + {goarch: "386", instruction: "VGF2P8AFFINEQB $1, X0, X1, X2"}, + {goarch: "386", instruction: "VGF2P8MULB Z0, Z1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's GFNI table", test.instruction) + } + }) + } +} + +func TestAMD64GFNIRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT gfnisemantics(SB),$0-48 + MOVQ out+0(FP), AX + MOVQ matrix+8(FP), BX + MOVQ data+16(FP), CX + MOVQ other+24(FP), DX + MOVQ old+32(FP), R8 + MOVQ mask+40(FP), R9 + KMOVQ R9, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VMOVDQU64 0(DX), Z2 + VMOVDQU64 0(R8), Z3 + STC + VGF2P8MULB Z2, Z1, Z4 + VMOVDQU64 Z4, 0(AX) + VGF2P8AFFINEQB $0x63, Z0, Z1, Z4 + VMOVDQU64 Z4, 64(AX) + VGF2P8AFFINEINVQB $0xa5, Z0, Z1, Z4 + VMOVDQU64 Z4, 128(AX) + VGF2P8MULB Z2, Z1, K1, Z3 + VMOVDQU64 Z3, 192(AX) + VGF2P8AFFINEQB.Z $0x63, Z0, Z1, K1, Z4 + VMOVDQU64 Z4, 256(AX) + VGF2P8AFFINEQB.BCST $0x63, 0(BX), Z1, Z4 + VMOVDQU64 Z4, 320(AX) + VMOVDQU64 0(R8), Z3 + VGF2P8AFFINEINVQB.BCST.Z $0xa5, 0(BX), Z1, K1, Z3 + VMOVDQU64 Z3, 384(AX) + VGF2P8MULB Z2, Z1, Z2 + VMOVDQU64 Z2, 448(AX) + SETCS 512(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"gfnisemantics": { + Name: "gfnisemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void gfnisemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint8_t gf_mul(uint8_t a, uint8_t b) { + uint8_t r = 0; + for (int i = 0; i < 8; i++) { + if (b & 1) r ^= a; + uint8_t hi = a & 0x80; + a <<= 1; + if (hi) a ^= 0x1b; + b >>= 1; + } + return r; +} +static uint8_t gf_inv(uint8_t x) { + if (!x) return 0; + uint8_t r = 1, a = x; + for (unsigned e = 254; e; e >>= 1) { + if (e & 1) r = gf_mul(r, a); + a = gf_mul(a, a); + } + return r; +} +static unsigned parity8(uint8_t x) { + x ^= x >> 4; x &= 0xf; + return (0x6996u >> x) & 1u; +} +static uint8_t affine(const uint8_t *matrix, uint8_t x, uint8_t imm, int inverse) { + if (inverse) x = gf_inv(x); + uint8_t r = 0; + for (int bit = 0; bit < 8; bit++) r |= (uint8_t)(parity8(matrix[7-bit] & x) << bit); + return r ^ imm; +} +int main(void) { + uint8_t matrix[64], data[64], other[64], old[64], out[513] = {0}; + const uint64_t mask = UINT64_C(0xa55aa55af00f0ff0); + for (int i = 0; i < 64; i++) { + matrix[i] = (uint8_t)(0x81u + 29u * (unsigned)i); + data[i] = (uint8_t)(3u + 17u * (unsigned)i); + other[i] = (uint8_t)(7u + 11u * (unsigned)i); + old[i] = (uint8_t)(0xe0u - (unsigned)i); + } + gfnisemantics(out, matrix, data, other, old, mask); + for (int i = 0; i < 64; i++) { + const uint8_t mul = gf_mul(other[i], data[i]); + const uint8_t aq = affine(matrix + (i/8)*8, data[i], 0x63, 0); + const uint8_t ai = affine(matrix + (i/8)*8, data[i], 0xa5, 1); + if (out[i] != mul) return 10; + if (out[64+i] != aq) return 11; + if (out[128+i] != ai) return 12; + if (out[192+i] != (((mask>>i)&1) ? mul : old[i])) return 13; + if (out[256+i] != (((mask>>i)&1) ? aq : 0)) return 14; + if (out[320+i] != affine(matrix, data[i], 0x63, 0)) return 15; + if (out[384+i] != (((mask>>i)&1) ? affine(matrix, data[i], 0xa5, 1) : 0)) return 16; + if (out[448+i] != mul) return 17; + } + if (out[512] != 1) return 18; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "gfni_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_halt_test.go b/amd64_lower_halt_test.go new file mode 100644 index 000000000..baff7b503 --- /dev/null +++ b/amd64_lower_halt_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86HLTCompleteGoAssemblerForms(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT haltforms(SB),NOSPLIT,$0-0\n\tHLT\n") + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"haltforms": {Name: "haltforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, `asm sideeffect "hlt"`) || !strings.Contains(ll, "unreachable") { + t.Fatalf("HLT must retain the hardware instruction and terminate control flow:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "hlt-"+target.name+".ll", "hlt-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86HLTRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{"HLT AX", "HLT $1", "HLT.P"} { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n") + if err != nil { + continue + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's HLT ynone table", instruction) + } + } +} diff --git a/amd64_lower_horizontal_float.go b/amd64_lower_horizontal_float.go new file mode 100644 index 000000000..1f4abcbae --- /dev/null +++ b/amd64_lower_horizontal_float.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64HorizontalFloatingSpec struct { + laneBits int + subtract bool + vector bool +} + +// amd64HorizontalFloatingSpecs is the complete Go 1.27 horizontal +// floating-point add/subtract family. Legacy forms use yxm; V-prefixed forms +// use _yvaddsubpd and are deliberately limited to VEX X/Y registers. +var amd64HorizontalFloatingSpecs = map[Op]amd64HorizontalFloatingSpec{ + "HADDPS": {laneBits: 32}, + "HADDPD": {laneBits: 64}, + "HSUBPS": {laneBits: 32, subtract: true}, + "HSUBPD": {laneBits: 64, subtract: true}, + "VHADDPS": {laneBits: 32, vector: true}, + "VHADDPD": {laneBits: 64, vector: true}, + "VHSUBPS": {laneBits: 32, subtract: true, vector: true}, + "VHSUBPD": {laneBits: 64, subtract: true, vector: true}, +} + +func (c *amd64Ctx) lowerHorizontalFloating(op Op, ins Instr) (ok bool, terminated bool, err error) { + baseOp, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + _, recognized := amd64HorizontalFloatingSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("horizontal floating dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86HorizontalFloatProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + byteWidth, spec := form.byteWidth, form.spec + if ins.Args[form.first].Kind == OpFP { + if err := c.validateFPVectorSpan(ins.Args[form.first].FPOffset, byteWidth); err != nil { + return true, false, err + } + } + first, err := c.loadHorizontalFloatingOperand(ins.Args[form.first], byteWidth, spec.laneBits) + if err != nil { + return true, false, err + } + second, err := c.loadHorizontalFloatingOperand(ins.Args[form.second], byteWidth, spec.laneBits) + if err != nil { + return true, false, err + } + result := c.emitHorizontalFloating(spec, byteWidth, first, second) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, amd64FMA3LLVMType(byteWidth*8/spec.laneBits, spec.laneBits), result, byteWidth) + if !spec.vector { + return true, false, c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+out) + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadHorizontalFloatingOperand(operand Operand, byteWidth, laneBits int) (string, error) { + bytesValue, err := c.loadPackedCompareBytes(operand, byteWidth) + if err != nil { + return "", err + } + lanes := byteWidth * 8 / laneBits + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", result, byteWidth, bytesValue, amd64FMA3LLVMType(lanes, laneBits)) + return "%" + result, nil +} + +func (c *amd64Ctx) emitHorizontalFloating(spec amd64HorizontalFloatingSpec, byteWidth int, first, second string) string { + lanes := byteWidth * 8 / spec.laneBits + lanesPer128 := 128 / spec.laneBits + typeName := amd64FMA3LLVMType(lanes, spec.laneBits) + leftIndices, rightIndices := make([]string, lanes), make([]string, lanes) + for lane := 0; lane < lanes; lane++ { + position := lane % lanesPer128 + block := lane - position + pair, sourceBase := position, 0 + if position >= lanesPer128/2 { + sourceBase = lanes + pair -= lanesPer128 / 2 + } + leftIndex := sourceBase + block + 2*pair + leftIndices[lane] = fmt.Sprintf("i32 %d", leftIndex) + rightIndices[lane] = fmt.Sprintf("i32 %d", leftIndex+1) + } + left, right := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector %s %s, %s %s, <%d x i32> <%s>\n", left, typeName, second, typeName, first, lanes, strings.Join(leftIndices, ", ")) + fmt.Fprintf(c.b, " %%%s = shufflevector %s %s, %s %s, <%d x i32> <%s>\n", right, typeName, second, typeName, first, lanes, strings.Join(rightIndices, ", ")) + // Project each independent 128-bit block without additional arithmetic. + // One constrained vector operation retains joint exception semantics; + // the reviewed integer-only binary helper preserves pair NaN priority. + form := x86FloatAddSubForm{ + spec: x86FloatAddSubSpec{bits: spec.laneBits, subtract: spec.subtract}, + x86FloatBinaryForm: x86FloatBinaryForm{bits: spec.laneBits}, + } + return c.emitFloatAddSub(form, lanes, "%"+left, "%"+right, "") +} diff --git a/amd64_lower_horizontal_integer.go b/amd64_lower_horizontal_integer.go new file mode 100644 index 000000000..d64db5a64 --- /dev/null +++ b/amd64_lower_horizontal_integer.go @@ -0,0 +1,154 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerHorizontalInteger(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + _, recognized := amd64HorizontalIntegerSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("horizontal integer dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86HorizontalIntegerProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + spec, byteWidth := form.spec, form.byteWidth + if form.memory >= 0 && ins.Args[form.memory].Kind == OpFP { + if err := c.validateFPVectorSpan(ins.Args[form.memory].FPOffset, byteWidth); err != nil { + return true, false, err + } + } + destination := ins.Args[form.destination].Reg + if form.mmx { + memoryBytes := 0 + if form.memory >= 0 { + memoryBytes = byteWidth + } + firstBits, err := c.loadPackedUnpackMMXSource(ins.Args[form.first], memoryBytes) + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(destination) + if err != nil { + return true, false, err + } + first := c.bitcastI64ToIntegerLanes(2, 32, firstBits) + second := c.bitcastI64ToIntegerLanes(2, 32, secondBits) + result := c.emitHorizontalInteger(spec, 8, first, second) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i32> %s to i64\n", bits, result) + return true, false, c.storeReg(destination, "%"+bits) + } + firstOperand := ins.Args[form.first] + if !spec.vector && form.memory >= 0 && (firstOperand.Kind == OpFP || ins.x86AddressBits != 0 || + firstOperand.Kind == OpMem && !x86OrdinaryNativeMemoryOperand(firstOperand, c.goarch)) { + return true, false, fmt.Errorf("%w: legacy horizontal m128 needs original native backing/alignment", ErrProbeNeedsContext) + } + second, err := c.loadHorizontalIntegerOperand(ins.Args[form.second], byteWidth, spec.laneBits) + if err != nil { + return true, false, err + } + var result string + if !spec.vector && form.memory >= 0 { + result, err = c.emitPackedBinaryNativeMemory(firstOperand, spec.laneBits, second, strings.ToLower(baseOp)) + } else { + var first string + first, err = c.loadHorizontalIntegerOperand(firstOperand, byteWidth, spec.laneBits) + if err == nil { + result = c.emitHorizontalInteger(spec, byteWidth, first, second) + } + } + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + if !spec.vector { + return true, false, c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+out) + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadHorizontalIntegerOperand(operand Operand, byteWidth, laneBits int) (string, error) { + bytesValue, err := c.loadX86PackedShiftBytes(operand, byteWidth, true) + if err != nil { + return "", err + } + lanes := byteWidth * 8 / laneBits + return c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, bytesValue), nil +} + +func (c *amd64Ctx) emitHorizontalInteger(spec amd64HorizontalIntegerSpec, byteWidth int, first, second string) string { + lanes := byteWidth * 8 / spec.laneBits + lanesPer128 := 128 / spec.laneBits + // The legacy MMX PHADDD row is one 64-bit lane group rather than one + // 128-bit group, but the same half-from-each-source construction applies. + if byteWidth == 8 { + lanesPer128 = lanes + } + typeName := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + elementType := fmt.Sprintf("i%d", spec.laneBits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + position := lane % lanesPer128 + block := lane - position + source := second + pair := position + if position >= lanesPer128/2 { + source = first + pair -= lanesPer128 / 2 + } + leftIndex := block + 2*pair + rightIndex := leftIndex + 1 + left := c.newTmp() + right := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", left, typeName, source, leftIndex) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", right, typeName, source, rightIndex) + value := c.emitHorizontalIntegerPair(spec, "%"+left, "%"+right) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %s, i32 %d\n", inserted, typeName, result, elementType, value, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitHorizontalIntegerPair(spec amd64HorizontalIntegerSpec, left, right string) string { + operation := "add" + if spec.subtract { + operation = "sub" + } + if !spec.saturate { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i%d %s, %s\n", value, operation, spec.laneBits, left, right) + return "%" + value + } + leftWide := c.newTmp() + rightWide := c.newTmp() + value := c.newTmp() + below := c.newTmp() + lowerClamped := c.newTmp() + above := c.newTmp() + clamped := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i16 %s to i32\n", leftWide, left) + fmt.Fprintf(c.b, " %%%s = sext i16 %s to i32\n", rightWide, right) + fmt.Fprintf(c.b, " %%%s = %s i32 %%%s, %%%s\n", value, operation, leftWide, rightWide) + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %%%s, -32768\n", below, value) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -32768, i32 %%%s\n", lowerClamped, below, value) + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, 32767\n", above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 32767, i32 %%%s\n", clamped, above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = trunc i32 %%%s to i16\n", result, clamped) + return "%" + result +} diff --git a/amd64_lower_horizontal_integer_test.go b/amd64_lower_horizontal_integer_test.go new file mode 100644 index 000000000..9bb7adddb --- /dev/null +++ b/amd64_lower_horizontal_integer_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86HorizontalIntegerCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT horizontalintegerforms(SB),NOSPLIT,$0-0\n") + // PHADDD uniquely uses Go's ymmxmm0f38 table and therefore has + // both MMX/m64 and XMM/m128 legacy rows. + source.WriteString("\tPHADDD M0, M1\n\tPHADDD 8(AX), M2\n") + for _, op := range []string{"PHADDD", "PHADDSW", "PHADDW", "PHSUBD", "PHSUBSW", "PHSUBW"} { + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, last) + vector := "V" + op + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s%d\n", vector, width, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s1, %s%d\n", vector, width, width, last) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "horizontalintegerforms": {Name: "horizontalintegerforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "horizontal-integer-"+target.name+".ll", "horizontal-integer-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86HorizontalIntegerRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PHADDD.Z X0, X1", + "PHADDD X0", + "PHADDSW M0, M1", + "PHADDW M0, M1", + "PHSUBD M0, M1", + "PHSUBSW M0, M1", + "PHSUBW M0, M1", + "PHADDW X0, M1", + "PHSUBD Y0, Y1", + "VPHADDD.Z X0, X1, X2", + "VPHADDSW X0, X1", + "VPHADDW X0, Y1, Y2", + "VPHSUBD Z0, Z1, Z2", + "VPHSUBSW X0, X1, K1, X2", + "VPHSUBW X0, X1, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86HorizontalIntegerRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PHADDD X8, X0", + "PHADDW X0, X8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86HorizontalIntegerRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86HorizontalIntegerRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's horizontal-integer forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_ifma.go b/amd64_lower_ifma.go new file mode 100644 index 000000000..6c4ee2335 --- /dev/null +++ b/amd64_lower_ifma.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64IFMASpec struct { + high bool +} + +// amd64IFMASpecs is the complete Go 1.27 integer fused multiply-add family. +// Both instructions use _yvblendmpd and operate on the low unsigned 52 bits +// of each qword source. LUQ adds product bits 51:0; HUQ adds bits 103:52. +var amd64IFMASpecs = map[Op]amd64IFMASpec{ + "VPMADD52HUQ": {high: true}, + "VPMADD52LUQ": {}, +} + +func (c *amd64Ctx) lowerIFMA(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64IFMASpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvblendmpd encoding: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects source2, source1, [K mask,] accumulator: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7 as its third operand: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s accumulator must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || + !c.isGoPackedVectorMoveRegister(destination, byteWidth) || + !c.isGoPackedVectorMoveRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s source1 and accumulator must be matching Go vector registers: %q", c.goarch, baseOp, ins.Raw) + } + rmSource := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source2: %q", c.goarch, baseOp, ins.Raw) + } + } else if rmSource.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(rmSource, byteWidth) { + return true, false, fmt.Errorf("%s %s source2 register must match the accumulator width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(rmSource) { + return true, false, fmt.Errorf("%s %s source2 must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth / 8 + first, err := c.loadPackedCompareLanes(rmSource, byteWidth, 64, properties.broadcast) + if err != nil { + return true, false, err + } + second, err := c.loadPackedCompareLanes(ins.Args[1], byteWidth, 64, false) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, 64, oldBytes) + computed := c.emitIFMA(spec, lanes, first, second, old) + if masked { + mask, loadErr := c.loadK(ins.Args[2].Reg) + if loadErr != nil { + return true, false, loadErr + } + computed = amd64ApplyIntegerLaneMask(c, lanes, 64, computed, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", out, lanes, computed, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitIFMA(spec amd64IFMASpec, lanes int, first, second, accumulator string) string { + const low52Mask = "4503599627370495" + result := "poison" + for lane := 0; lane < lanes; lane++ { + firstLane := c.newTmp() + secondLane := c.newTmp() + oldLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", firstLane, lanes, first, lane) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", secondLane, lanes, second, lane) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", oldLane, lanes, accumulator, lane) + maskedFirst := c.newTmp() + maskedSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %s\n", maskedFirst, firstLane, low52Mask) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %s\n", maskedSecond, secondLane, low52Mask) + wideFirst := c.newTmp() + wideSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i64 %%%s to i128\n", wideFirst, maskedFirst) + fmt.Fprintf(c.b, " %%%s = zext i64 %%%s to i128\n", wideSecond, maskedSecond) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i128 %%%s, %%%s\n", product, wideFirst, wideSecond) + selected := "%" + product + if spec.high { + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i128 %%%s, 52\n", high, product) + selected = "%" + high + } else { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i128 %%%s, %s\n", low, product, low52Mask) + selected = "%" + low + } + productPart := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i128 %s to i64\n", productPart, selected) + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", added, oldLane, productPart) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, lanes, result, added, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_ifma_test.go b/amd64_lower_ifma_test.go new file mode 100644 index 000000000..b13daf9fe --- /dev/null +++ b/amd64_lower_ifma_test.go @@ -0,0 +1,223 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86IFMACompleteGo127Forms(t *testing.T) { + // VPMADD52HUQ/LUQ share Go 1.27's six-row _yvblendmpd table: X/Y/Z, + // each with an unmasked and K1-K7 masked row. Their EVEX attributes add + // scalar qword broadcast and zeroing. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT ifma_forms(SB),$0-0\n") + for _, op := range []string{"VPMADD52HUQ", "VPMADD52LUQ"} { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s 8(BX), %s3, %s4\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST 16(BX), %s5, %s6\n", op, width, width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", op, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 24(BX), %s23, K7, %s24\n", op, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(BX), %s25, K2, %s26\n", op, width, width) + } + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "ifma_forms": {Name: "ifma_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "ifma-"+target.name+".ll", "ifma-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86IFMARejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VPMADD52HUQ X0, X1", + "VPMADD52LUQ X0, X1, X2, X3, X4", + "VPMADD52HUQ Y0, X1, X2", + "VPMADD52LUQ X0, Y1, X2", + "VPMADD52HUQ X0, X1, Y2", + "VPMADD52LUQ X0, 8(BX), X2", + "VPMADD52HUQ X0, X1, 8(BX)", + "VPMADD52LUQ X0, X1, K0, X2", + "VPMADD52HUQ X0, K1, X1, X2", + "VPMADD52LUQ.Z X0, X1, X2", + "VPMADD52HUQ.BCST X0, X1, X2", + "VPMADD52LUQ.BCST.Z 8(BX), X1, X2", + "VPMADD52HUQ.Z.BCST 8(BX), X1, K1, X2", + "VPMADD52LUQ.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's IFMA table", instruction) + } + }) + } + for _, instruction := range []string{ + "VPMADD52HUQ X0, X1, K1, X2", + "VPMADD52LUQ.Z 8(BX), Y1, K7, Y2", + "VPMADD52HUQ.BCST.Z 16(BX), Z1, K2, Z2", + "VPMADD52LUQ Z8, Z0, Z1", + "VPMADD52HUQ Z0, Z8, Z1", + "VPMADD52LUQ Z0, Z1, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted 386 %q outside Go 1.27's IFMA frontend boundary", instruction) + } + }) + } +} + +func TestAMD64IFMARuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT ifmaSemantics(SB),$0-48 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + MOVQ accumulator+24(FP), DX + MOVQ scalar+32(FP), SI + VMOVDQU64 (BX), X0 + VMOVDQU64 (CX), X1 + VMOVDQU64 (DX), X2 + VPMADD52LUQ X0, X1, X2 + VMOVDQU64 X2, 0(AX) + VMOVDQU64 (DX), X3 + VPMADD52HUQ X0, X1, X3 + VMOVDQU64 X3, 16(AX) + KMOVQ mask+40(FP), K1 + VMOVDQU64 (DX), X4 + VPMADD52LUQ X0, X1, K1, X4 + VMOVDQU64 X4, 32(AX) + VMOVDQU64 (DX), X5 + VPMADD52LUQ.Z X0, X1, K1, X5 + VMOVDQU64 X5, 48(AX) + VMOVDQU64 (DX), X6 + VPMADD52LUQ.BCST (SI), X1, X6 + VMOVDQU64 X6, 64(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "ifmaSemantics": { + Name: "ifmaSemantics", + Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void ifmaSemantics(uint64_t *, const uint64_t *, const uint64_t *, const uint64_t *, const uint64_t *, uint64_t); +static const uint64_t mask52 = (((uint64_t)1) << 52) - 1; +static uint64_t lo(uint64_t a, uint64_t b) { + return (uint64_t)((((unsigned __int128)(a & mask52)) * (b & mask52)) & mask52); +} +static uint64_t hi(uint64_t a, uint64_t b) { + return (uint64_t)((((unsigned __int128)(a & mask52)) * (b & mask52)) >> 52); +} +int main(void) { + const uint64_t first[2] = {(UINT64_C(1) << 60) | mask52, (UINT64_C(1) << 55) | (UINT64_C(1) << 51) | 3}; + const uint64_t second[2] = {(UINT64_C(1) << 59) | mask52, (UINT64_C(1) << 58) | (UINT64_C(1) << 51) | 5}; + const uint64_t accumulator[2] = {UINT64_MAX - 15, 100}; + const uint64_t scalar = (UINT64_C(1) << 63) | 7; + uint64_t out[10] = {0}; + const uint64_t want[10] = { + accumulator[0] + lo(first[0], second[0]), accumulator[1] + lo(first[1], second[1]), + accumulator[0] + hi(first[0], second[0]), accumulator[1] + hi(first[1], second[1]), + accumulator[0] + lo(first[0], second[0]), accumulator[1], + accumulator[0] + lo(first[0], second[0]), 0, + accumulator[0] + lo(scalar, second[0]), accumulator[1] + lo(scalar, second[1]), + }; + ifmaSemantics(out, first, second, accumulator, &scalar, 1); + for (int i = 0; i < 10; i++) if (out[i] != want[i]) return i + 1; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "ifma", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_implicit_legacy_test.go b/amd64_lower_implicit_legacy_test.go new file mode 100644 index 000000000..bd6db908b --- /dev/null +++ b/amd64_lower_implicit_legacy_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "strings" + "testing" +) + +func TestAMD64LeaveGrammarIsComplete(t *testing.T) { + want := map[Op]amd64LeaveSpec{ + "LEAVEW": {bits: 16}, + "LEAVEL": {bits: 32}, + "LEAVEQ": {bits: 64}, + } + if !reflect.DeepEqual(amd64LeaveSpecs, want) { + t.Fatalf("LEAVE grammar = %+v, want %+v", amd64LeaveSpecs, want) + } +} + +func TestTranslateX86LeaveAndXLATCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + secondLeave := "LEAVEQ" + if target.goarch == "386" { + secondLeave = "LEAVEL" + } + source := "TEXT implicitlegacyforms(SB),$0-0\n" + + "\tLEAVEW\n" + + "\t" + secondLeave + "\n" + + "\tXLAT\n" + + "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "implicitlegacyforms": {Name: "implicitlegacyforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"load i16", "load i8", "getelementptr i8", "-256"} { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if target.goarch == "386" { + if !strings.Contains(ir, "load i32") { + t.Errorf("386 IR is missing LEAVEL load:\n%s", ir) + } + } else if !strings.Contains(ir, "load i64") { + t.Errorf("amd64 IR is missing LEAVEQ load:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "implicit-legacy-"+target.name+".ll", "implicit-legacy-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LeaveAndXLATRejectFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LEAVEW AX"}, + {goarch: "amd64", instruction: "LEAVEL"}, + {goarch: "amd64", instruction: "LEAVEQ.P"}, + {goarch: "amd64", instruction: "XLAT AX"}, + {goarch: "amd64", instruction: "XLAT.P"}, + {goarch: "386", instruction: "LEAVEQ"}, + {goarch: "386", instruction: "LEAVEL AX"}, + {goarch: "386", instruction: "XLAT AX"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's LEAVE/XLAT tables", test.instruction) + } + }) + } +} + +func TestAMD64LeaveAndXLATRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT implicitlegacysemantics(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), R8 + MOVQ data+8(FP), BP + LEAVEQ + MOVQ BP, 0(R8) + MOVQ SP, 8(R8) + MOVQ data+8(FP), BX + ADDQ $16, BX + MOVQ $0x1122334455667703, AX + XLAT + MOVQ AX, 16(R8) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{ + "implicitlegacysemantics": { + Name: "implicitlegacysemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void implicitlegacysemantics(uint8_t *, uint8_t *); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint8_t data[256] = {0}, out[24] = {0}; + const uint64_t saved = UINT64_C(0xa1b2c3d4e5f60718); memcpy(data,&saved,8); + data[19] = 0xaa; + implicitlegacysemantics(out,data); + if (load64(out) != saved) return 10; + if (load64(out+8) != (uintptr_t)(data+8)) return 11; + if (load64(out+16) != UINT64_C(0x11223344556677aa)) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "implicit_legacy_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_implicit_mask_move.go b/amd64_lower_implicit_mask_move.go new file mode 100644 index 000000000..5baedd5ea --- /dev/null +++ b/amd64_lower_implicit_mask_move.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ImplicitMaskMoveSpec struct { + byteWidth int + vector bool +} + +// Go's frontend exposes the legacy SSE spelling both as MASKMOVOU and via +// the Intel-name alias MASKMOVDQU. MASKMOVQ is the corresponding MMX form; +// VMASKMOVDQU is the VEX spelling. All use the first operand as a byte mask, +// the second as data, and DI as the implicit memory destination. +var amd64ImplicitMaskMoveSpecs = map[Op]amd64ImplicitMaskMoveSpec{ + "MASKMOVQ": {byteWidth: 8}, + "MASKMOVOU": {byteWidth: 16}, + "MASKMOVDQU": {byteWidth: 16}, + "VMASKMOVDQU": {byteWidth: 16, vector: true}, +} + +func (c *amd64Ctx) lowerImplicitMaskMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64ImplicitMaskMoveSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed forms in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects mask and data registers: %q", c.goarch, baseOp, ins.Raw) + } + if spec.byteWidth == 8 { + for _, arg := range ins.Args { + if arg.Kind != OpReg { + return true, false, fmt.Errorf("%s %s uses Go's MMX-register-only ymr table: %q", c.goarch, baseOp, ins.Raw) + } + if _, ok := amd64ParseMReg(arg.Reg); !ok { + return true, false, fmt.Errorf("%s %s uses Go's MMX-register-only ymr table: %q", c.goarch, baseOp, ins.Raw) + } + } + maskBits, err := c.loadReg(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + dataBits, err := c.loadReg(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + mask := c.bitcastI64ToIntegerLanes(8, 8, maskBits) + data := c.bitcastI64ToIntegerLanes(8, 8, dataBits) + return true, false, c.emitImplicitByteMaskStore(data, mask, 8) + } + + valid := func(arg Operand) bool { + if spec.vector { + return c.isGoVEXVectorRegister(arg, 16, false) + } + return arg.Kind == OpReg && c.isGoLegacyXReg(arg.Reg) + } + if !valid(ins.Args[0]) || !valid(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s uses Go's X-register-only table: %q", c.goarch, baseOp, ins.Raw) + } + mask, err := c.loadX(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + data, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + return true, false, c.emitImplicitByteMaskStore(data, mask, 16) +} + +func (c *amd64Ctx) emitImplicitByteMaskStore(data, mask string, byteWidth int) error { + predicate := c.newTmp() + addressInteger, err := c.loadReg(DI) + if err != nil { + return err + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <%d x i8> %s, zeroinitializer\n", predicate, byteWidth, mask) + fmt.Fprintf(c.b, " %%%s = inttoptr i64 %s to ptr\n", address, addressInteger) + fmt.Fprintf(c.b, " call void @llvm.masked.store.v%di8.p0(<%d x i8> %s, ptr align 1 %%%s, <%d x i1> %%%s)\n", byteWidth, byteWidth, data, address, byteWidth, predicate) + return nil +} diff --git a/amd64_lower_implicit_mask_move_test.go b/amd64_lower_implicit_mask_move_test.go new file mode 100644 index 000000000..bec2fa1f0 --- /dev/null +++ b/amd64_lower_implicit_mask_move_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ImplicitMaskMoveCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + source := fmt.Sprintf("TEXT implicitmaskmoveforms(SB),NOSPLIT,$0-0\n"+ + "\tMASKMOVQ M0, M7\n"+ + "\tMASKMOVOU X0, X%d\n"+ + "\tMASKMOVDQU X0, X%d\n"+ + "\tVMASKMOVDQU X0, X%d\n"+ + "\tRET\n", last, last, last) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "implicitmaskmoveforms": {Name: "implicitmaskmoveforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "implicit-mask-move-"+target.name+".ll", "implicit-mask-move-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ImplicitMaskMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "MASKMOVQ.Z M0, M1", + "MASKMOVQ M0", + "MASKMOVQ M0, X1", + "MASKMOVQ 8(AX), M1", + "MASKMOVOU.Z X0, X1", + "MASKMOVOU X0", + "MASKMOVOU M0, M1", + "MASKMOVOU X0, Y1", + "MASKMOVDQU.Z X0, X1", + "MASKMOVDQU X0, X1, X2", + "MASKMOVDQU 8(AX), X1", + "VMASKMOVDQU.Z X0, X1", + "VMASKMOVDQU X0", + "VMASKMOVDQU Y0, Y1", + "VMASKMOVDQU X0, K1, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ImplicitMaskMoveRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "MASKMOVOU X8, X0", + "MASKMOVDQU X0, X8", + "VMASKMOVDQU X8, X0", + "VMASKMOVDQU X0, X8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86ImplicitMaskMoveRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86ImplicitMaskMoveRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's implicit mask-move forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_implicit_system.go b/amd64_lower_implicit_system.go new file mode 100644 index 000000000..b8b80c28f --- /dev/null +++ b/amd64_lower_implicit_system.go @@ -0,0 +1,200 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ImplicitSystemOperand uint8 + +const ( + amd64ImplicitSystemNoOperand amd64ImplicitSystemOperand = iota + amd64ImplicitSystemGP + amd64ImplicitSystemAddressGP +) + +type amd64ImplicitRegisterSet uint8 + +const ( + amd64ImplicitAX amd64ImplicitRegisterSet = 1 << iota + amd64ImplicitCX + amd64ImplicitDX +) + +type amd64ImplicitSystemSpec struct { + operand amd64ImplicitSystemOperand + inputs amd64ImplicitRegisterSet + outputs amd64ImplicitRegisterSet + writesCarry bool + rawPrefix byte +} + +// amd64ImplicitSystemSpecs is the complete Go 1.27 grammar for the system +// instructions whose architectural state is carried by implicit AX/CX/DX +// registers, plus the WAITPKG instructions' one explicit Yrl register. The +// table separates operand syntax from data flow so parsing, register liveness, +// and emission all consume the same typed description. +var amd64ImplicitSystemSpecs = map[Op]amd64ImplicitSystemSpec{ + "MONITOR": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "MWAIT": {inputs: amd64ImplicitAX | amd64ImplicitCX}, + "RDPMC": {inputs: amd64ImplicitCX, outputs: amd64ImplicitAX | amd64ImplicitDX}, + "RDPKRU": {inputs: amd64ImplicitCX, outputs: amd64ImplicitAX | amd64ImplicitDX}, + "WRPKRU": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "XSETBV": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "UMONITOR": {operand: amd64ImplicitSystemAddressGP, rawPrefix: 0xf3}, + "UMWAIT": {operand: amd64ImplicitSystemGP, inputs: amd64ImplicitAX | amd64ImplicitDX, writesCarry: true, rawPrefix: 0xf2}, + "TPAUSE": {operand: amd64ImplicitSystemGP, inputs: amd64ImplicitAX | amd64ImplicitDX, writesCarry: true, rawPrefix: 0x66}, +} + +type amd64ImplicitSystemForm struct { + register Reg + addressBits int + segmentPrefix byte +} + +func (c *amd64Ctx) lowerImplicitSystem(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64ImplicitSystemSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + form, err := c.parseImplicitSystemForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + return c.emitImplicitSystem(baseOp, spec, form) +} + +func (c *amd64Ctx) parseImplicitSystemForm(op string, spec amd64ImplicitSystemSpec, ins Instr) (amd64ImplicitSystemForm, error) { + var form amd64ImplicitSystemForm + switch spec.operand { + case amd64ImplicitSystemNoOperand: + if len(ins.Args) != 0 { + return form, fmt.Errorf("%s %s takes no operands: %q", c.goarch, op, ins.Raw) + } + case amd64ImplicitSystemGP, amd64ImplicitSystemAddressGP: + if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[0].Reg, c.goarch) { + return form, fmt.Errorf("%s %s expects one Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + form.register = ins.Args[0].Reg + if spec.operand == amd64ImplicitSystemAddressGP { + form.addressBits = 64 + if c.goarch == "386" { + form.addressBits = 32 + } + if ins.x86Encoded { + if ins.x86AddressBits != 0 { + if ins.x86AddressBits != form.addressBits && ins.x86AddressBits != form.addressBits/2 { + return form, fmt.Errorf("invalid %s address size %d", c.goarch, ins.x86AddressBits) + } + form.addressBits = ins.x86AddressBits + } + form.segmentPrefix = ins.x86SegmentPrefix + } + } + default: + panic("unknown amd64 implicit-system operand kind") + } + return form, nil +} + +func (c *amd64Ctx) emitImplicitSystem(op string, spec amd64ImplicitSystemSpec, form amd64ImplicitSystemForm) (bool, bool, error) { + wordType := I64 + if c.goarch == "386" { + wordType = I32 + } + constraints := make([]string, 0, 8) + arguments := make([]string, 0, 4) + if spec.outputs == amd64ImplicitAX|amd64ImplicitDX { + constraints = append(constraints, "={ax}", "={dx}") + } + if spec.writesCarry { + constraints = append(constraints, "=q") + } + if spec.operand != amd64ImplicitSystemNoOperand { + operandType := I32 // UMWAIT/TPAUSE read control[31:0], even with REX.W. + if spec.operand == amd64ImplicitSystemAddressGP { + operandType = LLVMType(fmt.Sprintf("i%d", form.addressBits)) + } + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.register}, operandType) + if err != nil { + return true, false, err + } + if spec.operand == amd64ImplicitSystemAddressGP && operandType != wordType { + // Monitor the same zero-extended offset using the native address + // width. The segment base is still applied by the actual instruction. + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", extended, operandType, value, wordType) + value, operandType = "%"+extended, wordType + } + constraints = append(constraints, "r") + arguments = append(arguments, fmt.Sprintf("%s %s", operandType, value)) + } + for _, implicit := range []struct { + bit amd64ImplicitRegisterSet + reg Reg + constraint string + }{ + {bit: amd64ImplicitAX, reg: AX, constraint: "{ax}"}, + {bit: amd64ImplicitCX, reg: CX, constraint: "{cx}"}, + {bit: amd64ImplicitDX, reg: DX, constraint: "{dx}"}, + } { + if spec.inputs&implicit.bit == 0 { + continue + } + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: implicit.reg}, I32) + if err != nil { + return true, false, err + } + constraints = append(constraints, implicit.constraint) + arguments = append(arguments, "i32 "+value) + } + constraints = append(constraints, "~{memory}", "~{dirflag}", "~{fpsr}", "~{flags}") + + assembly := strings.ToLower(op) + if form.segmentPrefix != 0 { + assembly = fmt.Sprintf(".byte 0x%02x; %s", form.segmentPrefix, assembly) + } + if spec.operand != amd64ImplicitSystemNoOperand { + operandIndex := 0 + if spec.writesCarry { + operandIndex = 1 + } + assembly += fmt.Sprintf(" $%d", operandIndex) + } + if spec.writesCarry { + assembly += "; setc $0" + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i8 asm sideeffect %q, %q(%s)\n", + call, assembly, strings.Join(constraints, ","), strings.Join(arguments, ", ")) + carry := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i8 %%%s to i1\n", carry, call) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + // WAITPKG defines CF and clears the other arithmetic flags (Intel SDM). + c.clearX86StatusFlags(x86StatusAll &^ x86StatusCF) + return true, false, nil + } + if spec.outputs == amd64ImplicitAX|amd64ImplicitDX { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call { i32, i32 } asm sideeffect %q, %q(%s)\n", + call, assembly, strings.Join(constraints, ","), strings.Join(arguments, ", ")) + for index, reg := range []Reg{AX, DX} { + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { i32, i32 } %%%s, %d\n", part, call, index) + if err := c.storeRegSized(reg, I32, "%"+part); err != nil { + return true, false, err + } + } + return true, false, nil + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", + assembly, strings.Join(constraints, ","), strings.Join(arguments, ", ")) + return true, false, nil +} diff --git a/amd64_lower_implicit_system_test.go b/amd64_lower_implicit_system_test.go new file mode 100644 index 000000000..5db6e097f --- /dev/null +++ b/amd64_lower_implicit_system_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64ImplicitSystemGrammarIsComplete(t *testing.T) { + want := map[Op]amd64ImplicitSystemSpec{ + "MONITOR": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "MWAIT": {inputs: amd64ImplicitAX | amd64ImplicitCX}, + "RDPMC": {inputs: amd64ImplicitCX, outputs: amd64ImplicitAX | amd64ImplicitDX}, + "RDPKRU": {inputs: amd64ImplicitCX, outputs: amd64ImplicitAX | amd64ImplicitDX}, + "WRPKRU": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "XSETBV": {inputs: amd64ImplicitAX | amd64ImplicitCX | amd64ImplicitDX}, + "UMONITOR": {operand: amd64ImplicitSystemAddressGP, rawPrefix: 0xf3}, + "UMWAIT": {operand: amd64ImplicitSystemGP, inputs: amd64ImplicitAX | amd64ImplicitDX, writesCarry: true, rawPrefix: 0xf2}, + "TPAUSE": {operand: amd64ImplicitSystemGP, inputs: amd64ImplicitAX | amd64ImplicitDX, writesCarry: true, rawPrefix: 0x66}, + } + if !reflect.DeepEqual(amd64ImplicitSystemSpecs, want) { + t.Fatalf("implicit-system grammar = %+v, want %+v", amd64ImplicitSystemSpecs, want) + } +} + +func TestTranslateX86ImplicitSystemCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + register := "R11" + if target.goarch == "386" { + register = "BX" + } + source := "TEXT implicitsystemforms(SB),$0-0\n" + + "\tMONITOR\n" + + "\tMWAIT\n" + + "\tRDPMC\n" + + "\tRDPKRU\n" + + "\tWRPKRU\n" + + "\tXSETBV\n" + + "\tUMONITOR " + register + "\n" + + "\tUMWAIT " + register + "\n" + + "\tTPAUSE " + register + "\n" + + "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "implicitsystemforms": {Name: "implicitsystemforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"target-features"="+pku,+waitpkg,+xsave"`, + `asm sideeffect "monitor"`, + `asm sideeffect "mwait"`, + `asm sideeffect "rdpmc"`, + `asm sideeffect "rdpkru"`, + `asm sideeffect "wrpkru"`, + `asm sideeffect "xsetbv"`, + `asm sideeffect "umonitor $0"`, + `asm sideeffect "umwait $1; setc $0"`, + `asm sideeffect "tpause $1; setc $0"`, + `store i1 %`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "implicit-system-"+target.name+".ll", "implicit-system-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ImplicitSystemRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "MONITOR AX"}, + {goarch: "amd64", instruction: "MWAIT.P"}, + {goarch: "amd64", instruction: "RDPMC AX"}, + {goarch: "amd64", instruction: "RDPKRU.P"}, + {goarch: "amd64", instruction: "WRPKRU AX"}, + {goarch: "amd64", instruction: "XSETBV AX"}, + {goarch: "amd64", instruction: "UMONITOR (BX)"}, + {goarch: "amd64", instruction: "UMWAIT X1"}, + {goarch: "amd64", instruction: "TPAUSE"}, + {goarch: "386", instruction: "UMONITOR R11"}, + {goarch: "386", instruction: "UMWAIT R11"}, + {goarch: "386", instruction: "TPAUSE R11"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's implicit-system tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_incdec.go b/amd64_lower_incdec.go new file mode 100644 index 000000000..fcf85c0a4 --- /dev/null +++ b/amd64_lower_incdec.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarIncDec implements the complete Go 1.27 INCB/W/L/Q and +// DECB/W/L/Q tables. INC and DEC preserve CF while updating the modeled +// ZF/SF/AF/PF/OF flags. +func (c *amd64Ctx) lowerScalarIncDec(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + bits := 0 + switch baseOp { + case "INCB", "DECB": + bits = 8 + case "INCW", "DECW": + bits = 16 + case "INCL", "DECL": + bits = 32 + case "INCQ", "DECQ": + bits = 64 + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one destination: %q", c.goarch, baseOp, ins.Raw) + } + dst := ins.Args[0] + if dst.Kind == OpReg { + if bits == 8 { + if !isGoYmbRegisterForArch(dst.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Ymb class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isX86YrlRegisterForArch(dst.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yml class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(dst) { + return true, false, fmt.Errorf("%s %s expects a GP register or memory destination: %q", c.goarch, baseOp, ins.Raw) + } + + ty := amd64IntegerTypeForBits(bits) + old, err := c.evalIntSized(dst, ty) + if err != nil { + return true, false, err + } + result := c.newTmp() + llvmOp := "add" + if baseOp[:3] == "DEC" { + llvmOp = "sub" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, 1\n", result, llvmOp, ty, old) + value := "%" + result + if err := c.storeScalarIntegerOperand(dst, ty, value); err != nil { + return true, false, err + } + + zero := c.newTmp() + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, ty, value) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", sign, ty, value) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", sign, c.flagsSltSlot) + c.setParityFlagSized(ty, value) + c.setAuxiliaryCarryFlag(ty, old, "1", value) + + overflowInput := int64(1) << (bits - 1) + if baseOp[:3] == "INC" { + overflowInput-- + } else { + overflowInput = -overflowInput + } + overflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, %d\n", overflow, ty, old, overflowInput) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", overflow, c.flagsOFSlot) + return true, false, nil +} diff --git a/amd64_lower_insert.go b/amd64_lower_insert.go new file mode 100644 index 000000000..d54ecfc16 --- /dev/null +++ b/amd64_lower_insert.go @@ -0,0 +1,9 @@ +package plan9asm + +// lowerPackedVectorInsert implements every Go 1.27 form in _yvinsertf128, +// _yvinsertf32x4, and _yvinsertf32x8. The floating and integer spellings have +// identical bit-level insert behavior; their lane type only changes EVEX mask +// granularity. +func (c *amd64Ctx) lowerPackedVectorInsert(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerInsertExtractFamily(op, ins) +} diff --git a/amd64_lower_insert_extract.go b/amd64_lower_insert_extract.go new file mode 100644 index 000000000..3e083014f --- /dev/null +++ b/amd64_lower_insert_extract.go @@ -0,0 +1,140 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerInsertExtractFamily(op Op, ins Instr) (bool, bool, error) { + ins, f, known, err := c.bindInsertExtractForm(op, ins) + if !known || err != nil { + return known, false, err + } + if f.mask >= 0 && f.memoryWrite { + return true, false, c.lowerNativeMaskedExtract(ins, f) + } + if f.spec.scalar { + if f.spec.insert { + return c.lowerInsertPackedSingle(op, ins) + } + return c.lowerPackedScalarExtract(op, ins) + } + src, err := c.loadInsertExtractSource(ins.Args[f.source], f) + if err != nil { + return true, false, err + } + result := src + if f.spec.insert { + base, err := c.loadPackedCompareBytes(ins.Args[f.base], f.width) + if err != nil { + return true, false, err + } + // Widen the insertion source only with defined zero bytes; then one + // shuffle reads the two old inputs before an aliased destination write. + wide := c.newTmp() + indices := make([]int, f.width) + for i := range indices { + indices[i] = i + if i >= f.spec.chunk { + indices[i] = f.spec.chunk + } + } + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> %s\n", wide, f.spec.chunk, src, f.spec.chunk, f.width, llvmI32Mask(indices)) + for i := range indices { + indices[i] = i + if i >= f.offset && i < f.offset+f.spec.chunk { + indices[i] = f.width + i - f.offset + } + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %%%s, <%d x i32> %s\n", out, f.width, base, f.width, wide, f.width, llvmI32Mask(indices)) + result = "%" + out + } else { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> poison, <%d x i32> %s\n", out, f.width, src, f.width, f.output, llvmI32RangeMask(f.offset, f.output)) + result = "%" + out + } + if f.mask >= 0 { + mask, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return true, false, err + } + lanes := f.output * 8 / f.spec.laneBits + computed := c.bitcastVectorBytesToIntegerLanes(f.output, lanes, f.spec.laneBits, result) + old := "zeroinitializer" + if !f.zeroing { + bytes, err := c.loadPackedCompareBytes(ins.Args[f.destination], f.output) + if err != nil { + return true, false, err + } + old = c.bitcastVectorBytesToIntegerLanes(f.output, lanes, f.spec.laneBits, bytes) + } + masked := amd64ApplyIntegerLaneMask(c, lanes, f.spec.laneBits, computed, old, mask, f.zeroing) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, f.spec.laneBits, masked, f.output) + result = "%" + out + } + return true, false, c.storePackedMoveOperand(ins.Args[f.destination], f.output, result) +} + +func (c *amd64Ctx) lowerNativeMaskedExtract(ins Instr, f x86InsertExtractForm) error { + destination := ins.Args[f.destination] + if destination.Kind == OpFP && c.classicFrame == "" { + return fmt.Errorf("%w: %s E6NF masked extraction needs contiguous original FP storage, not separately allocated virtual slots", ErrProbeNeedsContext, ins.Op) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(destination) + if err != nil { + return err + } + mask, err := c.loadK(ins.Args[f.mask].Reg) + if err != nil { + return err + } + index, _ := amd64VectorRegisterIndex(ins.Args[f.source].Reg, f.width) + buffer := c.yRegSlot[index] + if f.width == 64 { + buffer = c.zRegSlot[index] + } + if buffer == "" { + return fmt.Errorf("%w: missing private virtual source buffer for %s", ErrProbeNeedsContext, ins.Args[f.source].Reg) + } + bank := "ymm" + if f.width == 64 { + bank = "zmm" + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } + if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + // This write-only native instruction supplies E6NF's nonsuppressing + // address/fault behavior without inventing old destination read access + // or writing inactive lanes. Only private physical scratch is clobbered; + // virtual SIMD/K state and CPU flags are unchanged. No target feature is + // enabled globally. Native execution still requires the source's ISA. + assembly := fmt.Sprintf("vmovdqu64 ($0), %%%s0; kmovq $1, %%k1; %s $$%d, %%%s0, %s($2) {%%k1}", bank, strings.ToLower(string(ins.Op)), f.immediate, bank, segment) + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(ptr %s, i64 %s, %s %s)\n", assembly, "r,r,r,~{zmm0},~{k1},~{memory}", buffer, mask, pointerType, pointer) + return nil +} + +func (c *amd64Ctx) loadInsertExtractSource(arg Operand, f x86InsertExtractForm) (string, error) { + if f.spec.insert && f.memory >= 0 && arg.Kind != OpFP { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(arg) + if err != nil { + return "", err + } + value := c.newTmp() + // Intel E6NF retains the entire native access even if every result + // lane is inactive or subsequently overwritten. Volatile prevents + // LLVM O2 from deleting that original observable faulting load. + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, f.spec.chunk, pointerType, pointer) + return "%" + value, nil + } + width := f.width + if f.spec.insert { + width = f.spec.chunk + } + return c.loadPackedCompareBytes(arg, width) +} diff --git a/amd64_lower_insert_integer.go b/amd64_lower_insert_integer.go new file mode 100644 index 000000000..79568f8aa --- /dev/null +++ b/amd64_lower_insert_integer.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedIntegerInsertSpec struct { + laneBits int + legacy bool +} + +var amd64PackedIntegerInsertSpecs = map[string]amd64PackedIntegerInsertSpec{ + "PINSRB": {laneBits: 8, legacy: true}, + "PINSRW": {laneBits: 16, legacy: true}, + "PINSRD": {laneBits: 32, legacy: true}, + "PINSRQ": {laneBits: 64, legacy: true}, + "VPINSRB": {laneBits: 8}, + "VPINSRW": {laneBits: 16}, + "VPINSRD": {laneBits: 32}, + "VPINSRQ": {laneBits: 64}, +} + +func (c *amd64Ctx) lowerPackedIntegerInsert(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64PackedIntegerInsertSpecs[base]; !known { + return false, false, nil + } + ins, f, _, err := c.bindScalarLaneForm(op, ins) + if err != nil { + return true, false, err + } + source := ins.Args[f.source] + if source.Kind == OpReg { + source.Reg = f.transferReg + } + // Exact native m8/m16/m32/m64 reads remain volatile; FP uses the + // independent complete-byte-span helper, not the first scalar slot. + var scalar string + if source.Kind == OpFP { + scalar, err = c.loadScalarLaneFPBits(source.FPOffset, f.bits) + } else { + scalar, err = c.loadGPSIMDTransferSource(source, f.bits) + } + if err != nil { + return true, false, err + } + baseBytes, err := c.loadX(ins.Args[f.base].Reg) + if err != nil { + return true, false, err + } + lanes := 128 / f.bits + baseLanes, inserted, out := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x i%d>\n", baseLanes, baseBytes, lanes, f.bits) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %%%s, i%d %s, i32 %d\n", inserted, lanes, f.bits, baseLanes, f.bits, scalar, f.offset/(f.bits/8)) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", out, lanes, f.bits, inserted) + if f.legacy { + return true, false, c.storeX(ins.Args[f.destination].Reg, "%"+out) + } + return true, false, c.storePackedMoveOperand(ins.Args[f.destination], 16, "%"+out) +} + +func amd64IsExtendedGPRegister(reg Reg) bool { + switch strings.ToUpper(string(reg)) { + case "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return true + default: + return false + } +} diff --git a/amd64_lower_insert_integer_test.go b/amd64_lower_insert_integer_test.go new file mode 100644 index 000000000..736f2e40c --- /dev/null +++ b/amd64_lower_insert_integer_test.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedIntegerInsertCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packedintegerinsertforms(SB),$0-0\n") + legacyOps := []string{"PINSRB", "PINSRW", "PINSRD", "PINSRQ"} + vectorOps := []string{"VPINSRB", "VPINSRW", "VPINSRD", "VPINSRQ"} + if target.goarch == "386" { + legacyOps = legacyOps[:3] + vectorOps = nil + } + for _, op := range legacyOps { + fmt.Fprintf(&source, "\t%s $0, AX, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $255, 8(AX), X%d\n", op, legacyLast) + } + for _, op := range vectorOps { + fmt.Fprintf(&source, "\t%s $0, AX, X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $255, 8(AX), X0, X%d\n", op, legacyLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $82, R9, X22, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedintegerinsertforms": {Name: "packedintegerinsertforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-integer-insert.ll", "packed-integer-insert.o", ll) + }) + } +} + +func TestTranslateX86PackedIntegerInsertRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "PINSRB $-1, AX, X0", + "PINSRW $256, AX, X0", + "PINSRD $1, AX, Y0", + "PINSRQ.Z $1, AX, X0", + "PINSRB $1, AL, X0", + "PINSRD $1, X0, X1", + "PINSRQ $1, AX, X16", + "VPINSRB $1, AX, X0, Y1", + "VPINSRW $1, AX, Y0, X1", + "VPINSRD $1, X0, X1, X2", + "VPINSRQ $1, AX, X0, K1, X1", + "VPINSRQ.Z $1, AX, X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PINSR/VPINSR tables", instruction) + } + }) + } + + for _, instruction := range []string{ + "PINSRQ $1, AX, X0", + "VPINSRB $1, AX, X0, X1", + "VPINSRQ $1, AX, X0, X1", + "PINSRD $1, R8, X0", + "PINSRD $1, AX, X8", + } { + t.Run("386_"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's 386 PINSR/VPINSR tables", instruction) + } + }) + } +} diff --git a/amd64_lower_insert_test.go b/amd64_lower_insert_test.go new file mode 100644 index 000000000..deac8e303 --- /dev/null +++ b/amd64_lower_insert_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedVectorInsertCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 defines three complete insert tables: + // _yvinsertf128: imm8, X/m, Y, Y + // _yvinsertf32x4: imm8, X/m, Y|Z, [K1-K7,] Y|Z + // _yvinsertf32x8: imm8, Y/m, Z, [K1-K7,] Z + // Exercise register and memory members of every source class, every + // destination width, and both merge and zero masking for all ten opcodes. + for _, target := range []struct { + name string + goarch string + triple string + zreg int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", zreg: 21}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", zreg: 21}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", zreg: 21}, + } { + t.Run(target.name, func(t *testing.T) { + var src strings.Builder + src.WriteString("TEXT packedvectorinsertforms(SB),NOSPLIT,$0-0\n") + if target.goarch == "amd64" { + for _, op := range []string{"VINSERTF128", "VINSERTI128"} { + fmt.Fprintf(&src, "\t%s $0, X0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y3, Y4\n", op) + } + } + for _, op := range []string{"VINSERTF32X4", "VINSERTF64X2", "VINSERTI32X4", "VINSERTI64X2"} { + fmt.Fprintf(&src, "\t%s $0, X0, Y1, Y2\n", op) + fmt.Fprintf(&src, "\t%s $255, (AX), Y3, Y4\n", op) + fmt.Fprintf(&src, "\t%s $1, X20, Y20, K1, Y21\n", op) + fmt.Fprintf(&src, "\t%s.Z $2, (AX), Y20, K2, Y21\n", op) + fmt.Fprintf(&src, "\t%s $0, X0, Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $255, (AX), Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $3, X20, Z0, K3, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s.Z $1, (AX), Z0, K4, Z%d\n", op, target.zreg) + } + for _, op := range []string{"VINSERTF32X8", "VINSERTF64X4", "VINSERTI32X8", "VINSERTI64X4"} { + fmt.Fprintf(&src, "\t%s $0, Y0, Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $255, (AX), Z0, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s $1, Y20, Z0, K5, Z%d\n", op, target.zreg) + fmt.Fprintf(&src, "\t%s.Z $1, (AX), Z0, K6, Z%d\n", op, target.zreg) + } + src.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, strings.Replace(src.String(), ",NOSPLIT", "", 1), true) + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedvectorinsertforms": {Name: "packedvectorinsertforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-vector-insert-"+target.name+".ll", "packed-vector-insert-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedVectorInsertRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VINSERTI128 $0, Y0, Y1, Y2", + "VINSERTF128 $0, X0, Z1, Z2", + "VINSERTI128 $256, X0, Y1, Y2", + "VINSERTF128 $-1, X0, Y1, Y2", + "VINSERTI128.Z $0, X0, Y1, Y2", + "VINSERTF128 $0, X0, Y1, K1, Y2", + "VINSERTI32X4 $0, Y0, Y1, Y2", + "VINSERTF64X2 $0, X0, X1, X2", + "VINSERTI64X2 $0, X0, Y1, Z2", + "VINSERTF32X4 $0, X0, Y1, K0, Y2", + "VINSERTI32X4.Z $0, X0, Y1, Y2", + "VINSERTF64X2.BCST $0, (AX), Z1, Z2", + "VINSERTI32X8 $0, X0, Z1, Z2", + "VINSERTF64X4 $0, Y0, Y1, Y2", + "VINSERTI64X4 $0, Y0, Z1, K0, Z2", + "VINSERTF32X8.Z $0, Y0, Z1, Z2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + src := "TEXT bad(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", strings.Replace(src, ",NOSPLIT", "", 1), false) + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's packed insert tables", instruction) + } + }) + } + + for _, instruction := range []string{ + "VINSERTI128 $0, X0, Y0, Y1", + "VINSERTF128 $0, X0, Y0, Y1", + "VINSERTI128 $0, X16, Y0, Y1", + "VINSERTF128 $0, X0, Y16, Y1", + "VINSERTF32X4 $0, X0, Y0, Y1", + "VINSERTF64X2 $0, X0, Y0, Y1", + "VINSERTI32X4 $0, X0, Y0, Y1", + "VINSERTI64X2 $0, X0, Y0, Y1", + "VINSERTF32X8 $0, Y0, Z0, Z1", + "VINSERTF64X4 $0, Y0, Z0, Z1", + "VINSERTI32X8 $0, Y0, Z0, Z1", + "VINSERTI64X4 $0, Y0, Z0, Z1", + "VINSERTI32X4 $0, X0, Z8, Z0", + "VINSERTF64X2 $0, X0, Z0, K1, Z8", + "VINSERTI32X8 $0, Y0, Z8, Z0", + "VINSERTI64X4.Z $0, Y0, Z0, K1, Z8", + } { + src := "TEXT bad386(SB),NOSPLIT,$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", strings.Replace(src, ",NOSPLIT", "", 1), false) + file, err := Parse(ArchAMD64, src) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad386": {Name: "bad386", Ret: Void}}, + }); err == nil { + t.Fatalf("386 Translate accepted architecture-restricted %q", instruction) + } + } +} diff --git a/amd64_lower_insertps.go b/amd64_lower_insertps.go new file mode 100644 index 000000000..a1e0a2c29 --- /dev/null +++ b/amd64_lower_insertps.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerInsertPackedSingle implements all forms in Go 1.27's yxshuf and +// _yvinsertps tables. The imm8 selects the register-source lane in bits 7:6, +// the destination lane in bits 5:4, and lanes to clear in bits 3:0. +func (c *amd64Ctx) lowerInsertPackedSingle(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if baseOp != "INSERTPS" && baseOp != "VINSERTPS" { + return false, false, nil + } + var bindingErr error + ins, _, _, bindingErr = c.bindInsertExtractForm(op, ins) + if bindingErr != nil { + return true, false, bindingErr + } + if rawOp != baseOp { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", baseOp, ins.Raw) + } + wantArgs := 3 + if baseOp == "VINSERTPS" { + wantArgs = 4 + } + if len(ins.Args) != wantArgs || ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("amd64 %s expects unsigned imm8, X/m32 source, %s: %q", baseOp, map[bool]string{false: "X destination", true: "X base, X destination"}[baseOp == "VINSERTPS"], ins.Raw) + } + + source := ins.Args[1] + base := ins.Args[2] + destination := ins.Args[2] + if baseOp == "VINSERTPS" { + destination = ins.Args[3] + } + if base.Kind != OpReg || destination.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects X base/destination registers: %q", baseOp, ins.Raw) + } + if baseOp == "INSERTPS" { + if !c.isGoLegacyXReg(base.Reg) { + return true, false, fmt.Errorf("%s INSERTPS destination is outside Go 1.27's X register class: %q", c.goarch, ins.Raw) + } + } else if !amd64EVEXVectorRegister(base, 16) || !amd64EVEXVectorRegister(destination, 16) { + return true, false, fmt.Errorf("%s VINSERTPS base/destination is outside Go 1.27's VEX/EVEX X register class: %q", c.goarch, ins.Raw) + } + + var inserted string + if source.Kind == OpReg { + valid := c.isGoLegacyXReg(source.Reg) + if baseOp == "VINSERTPS" { + valid = amd64EVEXVectorRegister(source, 16) + } + if !valid { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's X register class: %q", c.goarch, baseOp, ins.Raw) + } + sourceBytes, err := c.loadX(source.Reg) + if err != nil { + return true, false, err + } + sourceLanes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", sourceLanes, sourceBytes) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 %d\n", selected, sourceLanes, (ins.Args[0].Imm>>6)&3) + inserted = "%" + selected + } else if source.Kind == OpFP { + // An original m32 FP span can comprise four independent ABI0 + // shadow fields. It is not a contiguous load from the first alloca. + value, err := c.loadFPVectorBytes(source.FPOffset, 4) + if err != nil { + return true, false, err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i8> %s to i32\n", bits, value) + inserted = "%" + bits + } else { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("amd64 %s expects an X register or memory source: %q", baseOp, ins.Raw) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return true, false, err + } + bits := c.newTmp() + // INSERTPS always reads m32, even when imm8 subsequently clears it. + fmt.Fprintf(c.b, " %%%s = load volatile i32, %s %s, align 1\n", bits, pointerType, pointer) + inserted = "%" + bits + } + + baseBytes, err := c.loadX(base.Reg) + if err != nil { + return true, false, err + } + baseLanes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", baseLanes, baseBytes) + placed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> %%%s, i32 %s, i32 %d\n", placed, baseLanes, inserted, (ins.Args[0].Imm>>4)&3) + result := "%" + placed + for lane := 0; lane < 4; lane++ { + if ins.Args[0].Imm&(1< %s, i32 0, i32 %d\n", cleared, result, lane) + result = "%" + cleared + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %s to <16 x i8>\n", out, result) + if baseOp == "VINSERTPS" { + return true, false, c.storePackedMoveOperand(destination, 16, "%"+out) + } + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) +} diff --git a/amd64_lower_insertps_test.go b/amd64_lower_insertps_test.go new file mode 100644 index 000000000..56c11cc8a --- /dev/null +++ b/amd64_lower_insertps_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86InsertPSCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's yxshuf and _yvinsertps tables contain exactly: + // INSERTPS imm8, X/m32, X + // VINSERTPS imm8, X/m32, X, X + // VINSERTPS can select either its VEX (<16) or EVEX (<32) entry. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + src := fmt.Sprintf(`TEXT insertpsforms(SB),NOSPLIT,$0-0 + INSERTPS $0, X0, X1 + INSERTPS $255, 4(AX), X%d +`, legacyLast) + vex := ` + VINSERTPS $0, X0, X1, X2 + VINSERTPS $255, 8(AX), X14, X15 + VINSERTPS $16, X20, X21, X22 + VINSERTPS $32, 12(AX), X30, X31 +` + if target.goarch == "386" { + // Preserve the previous mixed positive verbatim as an actual + // Go rejection: the 386 frontend cannot take four operands. + goAssemblerOracleOutput(t, "386", strings.Replace(src+vex+"RET\n", "NOSPLIT", "4", 1), "", false) + original, err := Parse(ArchAMD64, src+vex+"RET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(original, Options{Goarch: "386", TargetTriple: target.triple, Sigs: map[string]FuncSig{"insertpsforms": {Ret: Void}}}); err == nil { + t.Fatal("original Go-rejected mixed 386 fixture translated") + } + } else { + src += vex + } + src += "RET\n" + goAssemblerOracleOutput(t, target.goarch, strings.Replace(src, "NOSPLIT", "4", 1), "", true) + file, err := Parse(ArchAMD64, src) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "insertpsforms": {Name: "insertpsforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "insertps-"+target.name+".ll", "insertps-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86InsertPSRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "INSERTPS X0, X1", + "INSERTPS $-1, X0, X1", + "INSERTPS $256, X0, X1", + "INSERTPS $0, Y0, X1", + "INSERTPS $0, X0, Y1", + "INSERTPS.Z $0, X0, X1", + "VINSERTPS $0, X0, X1", + "VINSERTPS $0, X0, X1, X2, X3", + "VINSERTPS $0, Y0, X1, X2", + "VINSERTPS $0, X0, Y1, X2", + "VINSERTPS $0, X0, X1, Y2", + "VINSERTPS $0, X0, K1, X1", + "VINSERTPS.Z $0, X0, X1, X2", + "VINSERTPS.BCST $0, (AX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86InsertPSRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "INSERTPS $0, X8, X0", + "INSERTPS $0, X0, X8", + } { + assertX86InsertPSRejected(t, "386", "i386-unknown-linux-gnu", instruction) + } +} + +func assertX86InsertPSRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's INSERTPS tables", instruction) + } +} diff --git a/amd64_lower_lddqu.go b/amd64_lower_lddqu.go new file mode 100644 index 000000000..e15cb6498 --- /dev/null +++ b/amd64_lower_lddqu.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64UnalignedLoadSpec struct { + maxBytes int + vector bool +} + +// amd64UnalignedLoadSpecs is the complete Go 1.27 ylddqu/_yvlddqu grammar. +// Both instructions are load-only; legacy LDDQU is X-width, while VLDDQU +// adds the VEX Y-width row. +var amd64UnalignedLoadSpecs = map[Op]amd64UnalignedLoadSpec{ + "LDDQU": {maxBytes: 16}, + "VLDDQU": {maxBytes: 32, vector: true}, +} + +func (c *amd64Ctx) lowerUnalignedVectorLoad(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64UnalignedLoadSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 || !isAMD64MemoryOperand(ins.Args[0]) || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects memory, X%s: %q", c.goarch, baseOp, map[bool]string{true: "/Y"}[spec.maxBytes == 32], ins.Raw) + } + byteWidth := amd64VectorByteWidth(ins.Args[1].Reg) + validDestination := c.isGoVEXVectorRegister(ins.Args[1], byteWidth, false) + if spec.vector { + validDestination = amd64VEXVectorRegister(ins.Args[1], byteWidth) + } + if (byteWidth != 16 && byteWidth != 32) || byteWidth > spec.maxBytes || !validDestination { + return true, false, fmt.Errorf("%s %s destination width or register is outside its Go 1.27 table: %q", c.goarch, baseOp, ins.Raw) + } + if !spec.vector && ins.Args[0].Kind == OpMem && !x86MemoryRegistersValidForArch(ins.Args[0].Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s source uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.loadPackedCompareBytes(ins.Args[0], byteWidth) + if err != nil { + return true, false, err + } + return true, false, c.storeVectorBytes(ins.Args[1].Reg, byteWidth, value) +} diff --git a/amd64_lower_lddqu_test.go b/amd64_lower_lddqu_test.go new file mode 100644 index 000000000..7d24affee --- /dev/null +++ b/amd64_lower_lddqu_test.go @@ -0,0 +1,163 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64UnalignedLoadGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64UnalignedLoadSpec{ + "LDDQU": {maxBytes: 16}, + "VLDDQU": {maxBytes: 32, vector: true}, + } + if len(amd64UnalignedLoadSpecs) != len(expected) { + t.Fatalf("unaligned-load grammar has %d entries, want %d", len(amd64UnalignedLoadSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64UnalignedLoadSpecs[op]; !ok { + t.Errorf("unaligned-load grammar omitted %s", op) + } else if got != want { + t.Errorf("unaligned-load grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86LDDQUCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + source := fmt.Sprintf(`TEXT lddquforms(SB),$0-0 + LDDQU 1(AX), X%d + VLDDQU 2(AX), X%d + VLDDQU 3(AX), Y%d +`, last, last, last) + if target.goarch == "386" { + source += "\tVLDDQU 4(R9), Y8\n" + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"lddquforms": {Name: "lddquforms", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"load <16 x i8>", "load <32 x i8>"} { + if !strings.Contains(ir, want) { + t.Fatalf("LDDQU/VLDDQU lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "lddqu-"+target.name+".ll", "lddqu-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LDDQURejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LDDQU X0, X1"}, + {goarch: "amd64", instruction: "LDDQU 0(AX), Y1"}, + {goarch: "amd64", instruction: "VLDDQU X0, X1"}, + {goarch: "amd64", instruction: "VLDDQU 0(AX), Z1"}, + {goarch: "amd64", instruction: "VLDDQU.Z 0(AX), X1"}, + {goarch: "386", instruction: "LDDQU 0(AX), X8"}, + {goarch: "386", instruction: "LDDQU 0(R9), X0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: test.goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside the Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64LDDQURuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT lddqusemantics(SB),$0-16 + MOVQ out+0(FP), AX + MOVQ in+8(FP), CX + STC + LDDQU 1(CX), X0 + MOVUPS X0, 0(AX) + SETCS 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"lddqusemantics": { + Name: "lddqusemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void lddqusemantics(uint8_t *, const uint8_t *); +int main(void) { + uint8_t in[17], out[17] = {0}; + for (int i = 0; i < 17; i++) in[i] = (uint8_t)(i * 7 + 3); + lddqusemantics(out, in); + for (int i = 0; i < 16; i++) if (out[i] != in[i + 1]) return 10 + i; + if (out[16] != 1) return 30; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lddqu_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_lea_complete_test.go b/amd64_lower_lea_complete_test.go new file mode 100644 index 000000000..25d2f95e6 --- /dev/null +++ b/amd64_lower_lea_complete_test.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func x86LEACompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA leadata+0(SB)/8, $1\n") + source.WriteString("GLOBL leadata(SB), $8\n") + source.WriteString("TEXT leaforms(SB),$0-0\n") + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + op := "LEA" + width + source.WriteString("\t" + op + " 8(BX)(CX*2), AX\n") + source.WriteString("\t" + op + " leadata(SB), DX\n") + source.WriteString("\t" + op + " 16(BX), SP\n") + if goarch == "amd64" { + source.WriteString("\t" + op + " 24(R11)(R12*4), R13\n") + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86LEACompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86LEACompleteFormsSource(target.goarch) + requireLEANativeGoObject(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"leaforms": {Name: "leaforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"trunc i64", "to i16", "to i32"} { + if !strings.Contains(ir, want) { + t.Fatalf("LEA lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "lea-complete-"+target.name+".ll", "lea-complete-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LEARejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LEAW AX, BX"}, + {goarch: "amd64", instruction: "LEAL 8(AX), X0"}, + {goarch: "amd64", instruction: "LEAQ 8(AX)"}, + {goarch: "amd64", instruction: "LEAW.Z 8(AX), BX"}, + {goarch: "amd64", instruction: "LEAL $ret+0(FP), AX"}, + {goarch: "386", instruction: "LEAQ 8(AX), BX"}, + {goarch: "386", instruction: "LEAL $ret+0(FP), AX"}, + {goarch: "386", instruction: "LEAW 8(R9), AX"}, + {goarch: "386", instruction: "LEAL 8(AX), R8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireLEANativeGoObject(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ym_rl table", test.instruction) + } + }) + } +} + +func TestAMD64LEARuntimeWidthsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT leasementics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVQ $0x1122334455667788, AX + LEAW 0x1234(DI), AX + MOVQ AX, 0(DI) + LEAL 0x1234(DI), BX + MOVQ BX, 8(DI) + LEAQ 0x1234(DI), CX + MOVQ CX, 16(DI) + SETCS 24(DI) + SETOS 25(DI) + SETEQ 26(DI) + SETMI 27(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "leasementics": { + Name: "leasementics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void leasementics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[32] = {0}; + uintptr_t address = (uintptr_t)(out + 0x1234); + leasementics(out); + uint64_t want16 = UINT64_C(0x1122334455660000) | (address & UINT64_C(0xffff)); + if (load64(out+0) != want16) return 10; + if (load64(out+8) != (uint32_t)address) return 11; + if (load64(out+16) != address) return 12; + if (out[24] != 1 || out[25] != 1 || out[26] != 0 || out[27] != 1) return 13; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lea_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_leave.go b/amd64_lower_leave.go new file mode 100644 index 000000000..95696d254 --- /dev/null +++ b/amd64_lower_leave.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64LeaveSpec struct { + bits int +} + +// amd64LeaveSpecs is the complete Go 1.27 LEAVE grammar. W is shared, +// L is 386-only, and Q is amd64-only. The lowering models the architectural +// sequence SP = BP; POP BP against the translated register/address state. +var amd64LeaveSpecs = map[Op]amd64LeaveSpec{ + "LEAVEW": {bits: 16}, + "LEAVEL": {bits: 32}, + "LEAVEQ": {bits: 64}, +} + +func (c *amd64Ctx) lowerLeave(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64LeaveSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + if c.goarch != "386" && spec.bits == 32 { + return true, false, fmt.Errorf("amd64 %s is absent from Go 1.27's 64-bit encoding table: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + + base, err := c.loadReg(BP) + if err != nil { + return true, false, err + } + address := base + if c.goarch == "386" { + low := c.truncI64(base, I32) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, low) + address = "%" + wide + } + pointer := c.ptrFromAddrI64(address) + typ := amd64StackLLVMType(spec.bits) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", value, typ, pointer) + if err := c.storeRegSized(BP, typ, "%"+value); err != nil { + return true, false, err + } + if c.goarch == "386" { + address32 := c.truncI64(address, I32) + next32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i32 %s, %d\n", next32, address32, spec.bits/8) + next64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", next64, next32) + return true, false, c.storeRegUnchecked(SP, "%"+next64) + } + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", next, address, spec.bits/8) + return true, false, c.storeRegUnchecked(SP, "%"+next) +} diff --git a/amd64_lower_legacy_float_memory.go b/amd64_lower_legacy_float_memory.go new file mode 100644 index 000000000..38fcb5891 --- /dev/null +++ b/amd64_lower_legacy_float_memory.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86LegacyPackedFloatingNativeMemoryBits(ins Instr, goarch string) int { + var form x86FloatBinaryForm + addsub, known, err := x86FloatAddSubProperties(ins, goarch) + if known { + form = addsub.x86FloatBinaryForm + } else { + muldiv, recognized, parseErr := x86FloatMulDivProperties(ins, goarch) + known, err, form = recognized, parseErr, muldiv.x86FloatBinaryForm + } + if !known || err != nil || !form.legacy || form.scalar || form.memory < 0 || ins.Args[form.memory].Kind == OpFP { + return 0 + } + return form.bits +} + +// Legacy packed SSE arithmetic consumes one complete native m128 operand. +// An align-1 LLVM load followed by floating arithmetic can select a different +// load/instruction sequence, losing the original alignment and fault contract. +// Execute the exact validated opcode, including its source-order NaNs, MXCSR +// exceptions/rounding/DAZ/FTZ, and observable read even when the result is dead. +func (c *amd64Ctx) lowerLegacyPackedFloatingMemory(ins Instr, form x86FloatBinaryForm) error { + source := ins.Args[form.first] + if source.Kind == OpFP { + // ABI0 virtual fields are independently allocated. Even a logical + // contiguous field span is not original native stack alignment. + return fmt.Errorf("%w: %s m128 needs contiguous original FP storage and native alignment context, not virtual ABI0 fields", ErrProbeNeedsContext, ins.Op) + } + if ins.x86AddressBits != 0 { + return fmt.Errorf("%w: %s m128 lacks its original native address-size binding", ErrProbeNeedsContext, ins.Op) + } + if source.Kind == OpMem && !x86CallerMemoryCertified(ins, form.first) { + if !x86OrdinaryNativeMemoryOperand(source, c.goarch) { + return fmt.Errorf("%w: %s m128 needs original native stack backing, not virtual SP/BP", ErrProbeNeedsContext, ins.Op) + } + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + left, err := c.loadX(ins.Args[form.second].Reg) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + carrierType := amd64FMA3LLVMType(128/form.bits, form.bits) + carrier, result, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", carrier, left, carrierType) + assembly := strings.ToLower(string(ins.Op)) + " " + segment + "($2), $0" + // The tied private XMM value is read before its output is written. No + // original GP, flags, upper SIMD bytes, or additional pointer permission + // is created. Segment-aware pointers retain the original FS/GS prefix. + // SSE1 admits a <4 x float> XMM carrier on physical i386; a byte-vector + // constraint there requires SSE2 and fails LLVM legalization. These are + // only bitcasts, not extra arithmetic or an x87 transport ABI. + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %%%s, %s %s)\n", result, carrierType, assembly, "=x,0,r,~{memory}", carrierType, carrier, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", bytes, carrierType, result) + return c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+bytes) +} diff --git a/amd64_lower_legacy_mmx.go b/amd64_lower_legacy_mmx.go new file mode 100644 index 000000000..bb8ffc544 --- /dev/null +++ b/amd64_lower_legacy_mmx.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64LegacyMMXOperandClass uint8 + +const ( + amd64LegacyMMXMMX amd64LegacyMMXOperandClass = 1 << iota + amd64LegacyMMXXMM + amd64LegacyMMXMemory +) + +type amd64LegacyMMXMoveSpec struct { + source amd64LegacyMMXOperandClass + destination amd64LegacyMMXOperandClass + transferBits int + zeroExtend bool + nonTemporal bool +} + +// amd64LegacyMMXMoveSpecs describes the complete Go 1.27 legacy cross-domain +// and non-temporal MMX move grammar. Operand classes and transfer semantics are +// data; one parser validates all four opcode spellings. +var amd64LegacyMMXMoveSpecs = map[Op]amd64LegacyMMXMoveSpec{ + "MOVQOZX": {source: amd64LegacyMMXMMX | amd64LegacyMMXXMM | amd64LegacyMMXMemory, destination: amd64LegacyMMXXMM, transferBits: 64, zeroExtend: true}, + "MOVDQ2Q": {source: amd64LegacyMMXXMM, destination: amd64LegacyMMXMMX, transferBits: 64}, + "MOVNTQ": {source: amd64LegacyMMXMMX, destination: amd64LegacyMMXMemory, transferBits: 64, nonTemporal: true}, + "MOVNTDQ": {source: amd64LegacyMMXXMM, destination: amd64LegacyMMXMemory, transferBits: 128, nonTemporal: true}, +} + +type amd64LegacyMMXMoveForm struct { + source Operand + destination Operand +} + +func (c *amd64Ctx) lowerLegacyMMXMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64LegacyMMXMoveSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + form, err := c.parseLegacyMMXMoveForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + return c.emitLegacyMMXMove(spec, form) +} + +func (c *amd64Ctx) parseLegacyMMXMoveForm(op string, spec amd64LegacyMMXMoveSpec, ins Instr) (amd64LegacyMMXMoveForm, error) { + var form amd64LegacyMMXMoveForm + if len(ins.Args) != 2 { + return form, fmt.Errorf("%s %s expects exactly two operands: %q", c.goarch, op, ins.Raw) + } + sourceClass := c.legacyMMXOperandClass(ins.Args[0]) + destinationClass := c.legacyMMXOperandClass(ins.Args[1]) + if sourceClass&spec.source == 0 || destinationClass&spec.destination == 0 { + return form, fmt.Errorf("%s %s operands are outside its Go 1.27 legacy MMX table: %q", c.goarch, op, ins.Raw) + } + for _, operand := range ins.Args { + if operand.Kind == OpMem && !x86MemoryRegistersValidForArch(operand.Mem, c.goarch) { + return form, fmt.Errorf("%s %s memory address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + } + return amd64LegacyMMXMoveForm{source: ins.Args[0], destination: ins.Args[1]}, nil +} + +func (c *amd64Ctx) legacyMMXOperandClass(operand Operand) amd64LegacyMMXOperandClass { + if operand.Kind == OpReg { + if _, ok := amd64ParseMReg(operand.Reg); ok { + return amd64LegacyMMXMMX + } + if c.isGoLegacyXReg(operand.Reg) { + return amd64LegacyMMXXMM + } + return 0 + } + if isAMD64MemoryOperand(operand) { + return amd64LegacyMMXMemory + } + return 0 +} + +func (c *amd64Ctx) emitLegacyMMXMove(spec amd64LegacyMMXMoveSpec, form amd64LegacyMMXMoveForm) (bool, bool, error) { + if spec.destination == amd64LegacyMMXXMM { + bits, err := c.loadLegacyMMXMoveLow64(form.source) + if err != nil { + return true, false, err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> zeroinitializer, i64 %s, i32 0\n", words, bits) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bytes, words) + return true, false, c.storeX(form.destination.Reg, "%"+bytes) + } + + if spec.destination == amd64LegacyMMXMMX { + bits, err := c.loadLegacyMMXMoveLow64(form.source) + if err != nil { + return true, false, err + } + return true, false, c.storeReg(form.destination.Reg, bits) + } + + if spec.transferBits == 64 { + bits, err := c.evalIntSized(form.source, I64) + if err != nil { + return true, false, err + } + return true, false, c.storeMOVNTIMemory(form.destination, I64, bits) + } + bytes, err := c.loadX(form.source.Reg) + if err != nil { + return true, false, err + } + return true, false, c.storeVectorBytesOperandWithMetadata(form.destination, 16, bytes, x86NonTemporalMetadata) +} + +func (c *amd64Ctx) loadLegacyMMXMoveLow64(source Operand) (string, error) { + if source.Kind == OpReg { + if _, ok := amd64ParseMReg(source.Reg); ok { + return c.loadReg(source.Reg) + } + if c.isGoLegacyXReg(source.Reg) { + value, err := c.loadX(source.Reg) + if err != nil { + return "", err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", words, value) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, words) + return "%" + low, nil + } + } + return c.evalIntSized(source, I64) +} diff --git a/amd64_lower_legacy_mmx_test.go b/amd64_lower_legacy_mmx_test.go new file mode 100644 index 000000000..c9840e14c --- /dev/null +++ b/amd64_lower_legacy_mmx_test.go @@ -0,0 +1,223 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "strings" + "testing" +) + +func TestAMD64LegacyMMXMoveGrammarIsComplete(t *testing.T) { + want := map[Op]amd64LegacyMMXMoveSpec{ + "MOVQOZX": {source: amd64LegacyMMXMMX | amd64LegacyMMXXMM | amd64LegacyMMXMemory, destination: amd64LegacyMMXXMM, transferBits: 64, zeroExtend: true}, + "MOVDQ2Q": {source: amd64LegacyMMXXMM, destination: amd64LegacyMMXMMX, transferBits: 64}, + "MOVNTQ": {source: amd64LegacyMMXMMX, destination: amd64LegacyMMXMemory, transferBits: 64, nonTemporal: true}, + "MOVNTDQ": {source: amd64LegacyMMXXMM, destination: amd64LegacyMMXMemory, transferBits: 128, nonTemporal: true}, + } + if !reflect.DeepEqual(amd64LegacyMMXMoveSpecs, want) { + t.Fatalf("legacy MMX move grammar = %+v, want %+v", amd64LegacyMMXMoveSpecs, want) + } +} + +func TestTranslateX86LegacyMMXCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + base := "BX" + xreg := "X7" + if target.goarch == "amd64" { + base = "R11" + xreg = "X15" + } + source := `TEXT legacymmxforms(SB),$0-0 + PSHUFW $-128, M1, M2 + PSHUFW $127, 8(BX), M3 + MOVQOZX M2, X2 + MOVQOZX X1, X3 + MOVQOZX 16(BX), X4 + MOVNTQ M2, 24(BX) +` + "\tMOVQOZX M3, " + xreg + "\n" + + "\tMOVNTQ M3, 32(" + base + ")\n\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"legacymmxforms": {Name: "legacymmxforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"!nontemporal !0", "!0 = !{i32 1}"} { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "legacy-mmx-"+target.name+".ll", "legacy-mmx-"+target.name+".o", ir) + }) + } +} + +func TestAMD64LegacyMMXRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT legacymmxsemantics(SB),NOSPLIT,$0-16 + MOVQ value+0(FP), AX + MOVQ AX, M1 + MOVQ out+8(FP), DI + PSHUFW $0x1b, M1, M2 + MOVNTQ M2, 0(DI) + MOVQOZX M2, X1 + MOVDQ2Q X1, M3 + MOVNTQ M3, 8(DI) + MOVNTDQ X1, 16(DI) + SFENCE + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"legacymmxsemantics": { + Name: "legacymmxsemantics", Args: []LLVMType{I64, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void legacymmxsemantics(uint64_t, uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + __attribute__((aligned(16))) uint8_t out[32] = {0xff}; + legacymmxsemantics(UINT64_C(0x1122334455667788), out); + const uint64_t shuffled = UINT64_C(0x7788556633441122); + if (load64(out) != shuffled) return 10; + if (load64(out + 8) != shuffled) return 11; + if (load64(out + 16) != shuffled) return 12; + if (load64(out + 24) != 0) return 13; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_mmx_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86LegacyMMXEncoderAliases(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + const source = `TEXT encoderaliases(SB),$0-0 + MOVDQ2Q X1, M1 + MOVNTDQ X2, 8(BX) + RET +` + // Go accepts these Intel decoder names in addition to the MOVQ and + // MOVNTO spellings used by its primary optab entries. + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"encoderaliases": {Name: "encoderaliases", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "legacy-mmx-alias-"+target.name+".ll", "legacy-mmx-alias-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LegacyMMXRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PSHUFW $-129, M1, M2"}, + {goarch: "amd64", instruction: "PSHUFW $128, M1, M2"}, + {goarch: "amd64", instruction: "PSHUFW $1, X1, M2"}, + {goarch: "amd64", instruction: "PSHUFW $1, M1, X2"}, + {goarch: "amd64", instruction: "PSHUFW.Z $1, M1, M2"}, + {goarch: "amd64", instruction: "MOVQOZX M1, M2"}, + {goarch: "amd64", instruction: "MOVQOZX $1, X2"}, + {goarch: "amd64", instruction: "MOVNTQ X1, 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTQ M1, M2"}, + {goarch: "amd64", instruction: "MOVDQ2Q M1, X1"}, + {goarch: "amd64", instruction: "MOVNTDQ 8(BX), X1"}, + {goarch: "386", instruction: "MOVQOZX M1, X8"}, + {goarch: "386", instruction: "MOVNTQ M1, 8(R11)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the legacy MMX grammar", test.instruction) + } + }) + } +} diff --git a/amd64_lower_legacy_packed_logical_test.go b/amd64_lower_legacy_packed_logical_test.go new file mode 100644 index 000000000..21f1a9e79 --- /dev/null +++ b/amd64_lower_legacy_packed_logical_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86LegacyPackedLogicalCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastX := 15 + if target.goarch == "386" { + lastX = 7 + } + var source strings.Builder + source.WriteString("TEXT legacypackedlogicalforms(SB),$0-0\n") + for _, op := range []string{"PAND", "PANDN", "POR", "PXOR"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M7\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M7\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, lastX) + fmt.Fprintf(&source, "\t%s 16(AX), X%d\n", op, lastX) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"legacypackedlogicalforms": {Name: "legacypackedlogicalforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + wantIR := []string{"and <16 x i8>", "or <16 x i8>", "xor <16 x i8>"} + if target.goarch == "amd64" { + wantIR = append(wantIR, "and i64", "or i64", "xor i64") + } + for _, want := range wantIR { + if !strings.Contains(ir, want) { + t.Fatalf("legacy packed logical lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "legacy-packed-logical-"+target.name+".ll", "legacy-packed-logical-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LegacyPackedLogicalRejectsFormsOutsideGo127Table(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PAND M0, X1"}, + {goarch: "amd64", instruction: "POR X0, M1"}, + {goarch: "amd64", instruction: "PXOR Y0, Y1"}, + {goarch: "amd64", instruction: "PAND.Z X0, X1"}, + {goarch: "amd64", instruction: "PAND M0"}, + {goarch: "amd64", instruction: "POR M0, M1, M2"}, + {goarch: "386", instruction: "PXOR X0, X8"}, + {goarch: "386", instruction: "PAND 8(R9), X0"}, + {goarch: "386", instruction: "PAND M0, M1"}, + {goarch: "386", instruction: "PANDN M0, M1"}, + {goarch: "386", instruction: "POR M0, M1"}, + {goarch: "386", instruction: "PXOR M0, M1"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed logical table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64LegacyPackedLogicalMMXRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT legacypackedlogicalsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), M0 + MOVQ b+16(FP), M1 + STC + PAND M0, M1 + MOVQ M1, 0(AX) + MOVQ a+8(FP), M0 + MOVQ b+16(FP), M1 + PANDN M0, M1 + MOVQ M1, 8(AX) + MOVQ a+8(FP), M0 + MOVQ b+16(FP), M1 + POR M0, M1 + MOVQ M1, 16(AX) + MOVQ a+8(FP), M0 + MOVQ b+16(FP), M1 + PXOR M0, M1 + MOVQ M1, 24(AX) + SETCS 32(AX) + EMMS + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "legacypackedlogicalsemantics": { + Name: "legacypackedlogicalsemantics", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void legacypackedlogicalsemantics(uint8_t *, uint64_t, uint64_t); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v; } +int main(void) { + uint8_t out[33] = {0}; + uint64_t a = UINT64_C(0x0ff00ff0a55aa55a); + uint64_t b = UINT64_C(0x33cc55aa5aa5f00f); + legacypackedlogicalsemantics(out, a, b); + if (load64(out+0) != (a & b)) return 10; + if (load64(out+8) != ((~b) & a)) return 11; + if (load64(out+16) != (a | b)) return 12; + if (load64(out+24) != (a ^ b)) return 13; + if (out[32] != 1) return 14; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_packed_logical_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_legacy_packed_move.go b/amd64_lower_legacy_packed_move.go new file mode 100644 index 000000000..d8dabe8c8 --- /dev/null +++ b/amd64_lower_legacy_packed_move.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Both floating and integer dispatch use the same whole-128-bit yxmov grammar. +// Native operands keep the original opcode: align-1 loads lose aligned faults, +// and ordinary LLVM loads disappear when the source result is dead. +func (c *amd64Ctx) lowerLegacyPackedMove(op Op, ins Instr) (bool, bool, error) { + known, err := c.validateLegacyPackedMove(op, ins) + if !known || err != nil { + return known, false, err + } + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, _, err := x86LegacyPackedMoveProperties(op, ins, arch) + if err != nil { + return true, false, err + } + if form.memory >= 0 { + memory := ins.Args[form.memory] + if memory.Kind == OpFP && form.aligned { + return true, false, fmt.Errorf("%w: %s FP operand needs its original native alignment", ErrProbeNeedsContext, op) + } + // Existing checked virtual FP/AUTO views are not native addresses. + // They keep their independent layout handling; this rule gives them + // neither native alignment nor permission to read adjacent storage. + virtual := !x86OrdinaryNativeMemoryOperand(memory, arch) + if memory.Kind == OpMem && (x86CallerMemoryCertified(ins, form.memory) || c.fixedLocalMemoryCertified(ins, form.memory)) { + virtual = false + } + if !virtual { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: legacy packed move needs its original native address size", ErrProbeNeedsContext) + } + return true, false, c.lowerLegacyPackedMoveNative(op, form) + } + } + value, err := c.loadXVecOperand(form.source) + if err != nil { + return true, false, err + } + if form.destination.Kind == OpReg { + return true, false, c.storeX(form.destination.Reg, value) + } + return true, false, c.storeXVecOperand(form.destination, value) +} + +func (c *amd64Ctx) lowerLegacyPackedMoveNative(op Op, form x86LegacyPackedMoveForm) error { + memory := form.source + if form.memoryWrite { + memory = form.destination + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(memory) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + mnemonic := strings.ToLower(string(op)) + switch strings.ToUpper(string(op)) { + case "MOVOU": + mnemonic = "movdqu" + case "MOVO", "MOVOA": + mnemonic = "movdqa" + } + if form.memoryWrite { + value, err := c.loadX(form.source.Reg) + if err != nil { + return err + } + carrier := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x float>\n", carrier, value) + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(<4 x float> %%%s, %s %s)\n", + mnemonic+" $0, "+segment+"($1)", "x,r,~{memory}", carrier, pointerType, pointer) + return nil + } + // A float carrier preserves SSE1's XMM class even on 386. It is a bit + // transport only, with no floating arithmetic or change to MXCSR/FLAGS. + loaded, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s %s)\n", + loaded, mnemonic+" "+segment+"($1), $0", "=x,r,~{memory}", pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, loaded) + return c.storeX(form.destination.Reg, "%"+bytes) +} diff --git a/amd64_lower_lods_test.go b/amd64_lower_lods_test.go new file mode 100644 index 000000000..4e26c2806 --- /dev/null +++ b/amd64_lower_lods_test.go @@ -0,0 +1,225 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86LODSCompleteFormsSource(goarch string) string { + widths := []string{"B", "W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + var source strings.Builder + source.WriteString("TEXT lodsforms(SB),$0-0\n") + for _, width := range widths { + op := "LODS" + width + fmt.Fprintf(&source, "\t%s\n", op) + fmt.Fprintf(&source, "\tCLD; REP; %s\n", op) + fmt.Fprintf(&source, "\tSTD; REPN; %s\n", op) + source.WriteString("\tCLD\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86LODSCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86LODSCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"lodsforms": {Name: "lodsforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, width := range []string{"b", "w", "l"} { + if want := "@__plan9asm_rep_lods" + width; !strings.Contains(ir, want) { + t.Fatalf("LODS lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@__plan9asm_rep_lodsq") { + t.Fatalf("LODSQ lowering omitted qword helper:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "lods-"+target.name+".ll", "lods-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86LODSRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LODSB AX"}, + {goarch: "amd64", instruction: "LODSW AX, BX"}, + {goarch: "amd64", instruction: "LODSL.Z"}, + {goarch: "386", instruction: "LODSQ"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's LODS ynone table", test.instruction) + } + }) + } +} + +func TestAMD64LODSRuntimeSemanticsWidthsPrefixesAndDirection(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT lodssemantics(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), R8 + MOVQ data+8(FP), R9 + CLD + + MOVQ R9, SI + MOVQ $0x1122334455667788, AX + LODSB + MOVQ AX, 0(R8) + MOVQ SI, 8(R8) + + MOVQ R9, SI + ADDQ $8, SI + MOVQ $0x1122334455667788, AX + LODSW + MOVQ AX, 16(R8) + + MOVQ R9, SI + ADDQ $16, SI + MOVQ $-1, AX + LODSL + MOVQ AX, 24(R8) + + MOVQ R9, SI + ADDQ $24, SI + LODSQ + MOVQ AX, 32(R8) + + MOVQ R9, SI + ADDQ $32, SI + MOVQ $3, CX + REP; LODSB + MOVQ AX, 40(R8) + MOVQ SI, 48(R8) + MOVQ CX, 56(R8) + + MOVQ R9, SI + ADDQ $40, SI + MOVQ $3, CX + REPN; LODSW + MOVQ AX, 64(R8) + MOVQ SI, 72(R8) + MOVQ CX, 80(R8) + + MOVQ R9, SI + ADDQ $52, SI + MOVQ $2, CX + STD + REP; LODSL + CLD + MOVQ AX, 88(R8) + MOVQ SI, 96(R8) + MOVQ CX, 104(R8) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "lodssemantics": { + Name: "lodssemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void lodssemantics(uint8_t *, uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t data[64] = {0}; + data[0] = 0xaa; + const uint16_t w = UINT16_C(0xbbcc); memcpy(data+8, &w, sizeof(w)); + const uint32_t l = UINT32_C(0x89abcdef); memcpy(data+16, &l, sizeof(l)); + const uint64_t q = UINT64_C(0x0123456789abcdef); memcpy(data+24, &q, sizeof(q)); + data[32] = 1; data[33] = 2; data[34] = 3; + const uint16_t ws[3] = {0x1111,0x2222,0x3333}; memcpy(data+40, ws, sizeof(ws)); + const uint32_t ls[2] = {0x44444444,0x55555555}; memcpy(data+48, ls, sizeof(ls)); + uint8_t out[112] = {0}; + lodssemantics(out, data); + if (load64(out+0) != UINT64_C(0x11223344556677aa) || load64(out+8) != (uintptr_t)(data+1)) return 10; + if (load64(out+16) != UINT64_C(0x112233445566bbcc)) return 11; + if (load64(out+24) != UINT64_C(0x89abcdef)) return 12; + if (load64(out+32) != q) return 13; + if ((load64(out+40) & 0xff) != 3 || load64(out+48) != (uintptr_t)(data+35) || load64(out+56) != 0) return 14; + if ((load64(out+64) & 0xffff) != 0x3333 || load64(out+72) != (uintptr_t)(data+46) || load64(out+80) != 0) return 15; + if (load64(out+88) != UINT64_C(0x44444444) || load64(out+96) != (uintptr_t)(data+44) || load64(out+104) != 0) return 16; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lods_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_logical_family.go b/amd64_lower_logical_family.go new file mode 100644 index 000000000..6c90f4bc6 --- /dev/null +++ b/amd64_lower_logical_family.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedLogicalFamily(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := x86PackedLogicalSpecs[Op(base)]; !recognized { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed logical dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86PackedLogicalProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: legacy packed logical m128 lacks original native address-size binding", ErrProbeNeedsContext) + } + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed logical m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.mmx { + a, err := c.evalIntSized(first, I64) + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + if form.spec.mode == amd64PackedLogicalAndNot { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, -1\n", inverted, b) + b = "%" + inverted + } + operation := "and" + if form.spec.mode == amd64PackedLogicalOr { + operation = "or" + } + if form.spec.mode == amd64PackedLogicalXor { + operation = "xor" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %s\n", result, operation, b, a) + return true, false, c.storeReg(destination.Reg, "%"+result) + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + lanes := form.byteWidth * 8 / form.spec.laneBits + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, form.spec.laneBits, false) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 { + // Retain the single original memory opcode: an align-1 load and + // portable logical arithmetic lose its alignment/dead-read faults. + result, err = c.emitPackedLogicalNativeMemory(first, form, b, strings.ToLower(string(op))) + } else { + // EVEX inactive lanes suppress faults, including scalar broadcasts. + // Keep the reviewed masked reader and complete virtual FP spans. + var a string + a, err = c.loadMaskedPackedCompareLanes(first, form.byteWidth, form.spec.laneBits, form.broadcast, mask) + if err == nil { + result = c.emitPackedLogical(lanes, amd64PackedLogicalSpec{form.spec.laneBits, form.spec.mode}, a, b) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old, err := c.loadPackedCompareLanes(destination, form.byteWidth, form.spec.laneBits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.spec.laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.spec.laneBits, result, form.byteWidth) + if form.legacy { + // Legacy writes update low X bits in every wider view but preserve the + // upper bytes. storeX alone is a separate shadow slot, not an alias. + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedLogicalNativeMemory(source Operand, form x86PackedLogicalForm, data, mnemonic string) (string, error) { + if form.spec.encoding != x86PackedLogicalLegacyFP || form.spec.laneBits != 32 { + return c.emitPackedBinaryNativeMemory(source, form.spec.laneBits, data, mnemonic) + } + // PS is an SSE1 source instruction. A floating bit carrier retains its + // XMM class on i386 without silently introducing SSE2 integer transport. + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + carrier, result, value := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %s to <4 x float>\n", carrier, data) + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(<4 x float> %%%s, %s %s)\n", result, mnemonic+" "+segment+"($2), $0", "=x,0,r,~{memory}", carrier, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <4 x i32>\n", value, result) + return "%" + value, nil +} + +func (c *amd64Ctx) storeLegacyPackedLogical(dst Operand, value string) error { + index, _ := amd64ParseXReg(dst.Reg) + for _, view := range []struct { + width int + slots map[int]string + }{{32, c.yRegSlot}, {64, c.zRegSlot}} { + slot, live := view.slots[index] + if !live { + continue + } + old, merged := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = load <%d x i8>, ptr %s\n", old, view.width, slot) + extended := c.newTmp() + extendIndices := make([]string, view.width) + for i := range extendIndices { + lane := i + if i >= 16 { + lane = 16 + } + extendIndices[i] = fmt.Sprintf("i32 %d", lane) + } + fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <%d x i32> <%s>\n", extended, value, view.width, strings.Join(extendIndices, ", ")) + indices := make([]string, view.width) + for i := range indices { + lane := i + if i < 16 { + lane += view.width + } + indices[i] = fmt.Sprintf("i32 %d", lane) + } + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %%%s, <%d x i8> %%%s, <%d x i32> <%s>\n", merged, view.width, old, view.width, extended, view.width, strings.Join(indices, ", ")) + fmt.Fprintf(c.b, " store <%d x i8> %%%s, ptr %s\n", view.width, merged, slot) + } + return c.storeX(dst.Reg, value) +} diff --git a/amd64_lower_logical_shift.go b/amd64_lower_logical_shift.go new file mode 100644 index 000000000..88bf80f1e --- /dev/null +++ b/amd64_lower_logical_shift.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64PackedLogicalShiftSpec struct { + laneBits int + left bool + vector bool +} + +// amd64PackedLogicalShiftSpecs covers all Go 1.27 uniform packed logical +// shifts. Legacy operations use yps (MMX/XMM, immediate or packed count), and +// V-prefixed operations use _yvpslld (X/Y/Z data and an immediate or X/m128 +// count, with the table's EVEX mask and D/Q broadcast forms). +var amd64PackedLogicalShiftSpecs = map[Op]amd64PackedLogicalShiftSpec{ + "PSLLW": {laneBits: 16, left: true}, + "PSLLL": {laneBits: 32, left: true}, + "PSLLQ": {laneBits: 64, left: true}, + "PSRLW": {laneBits: 16}, + "PSRLL": {laneBits: 32}, + "PSRLQ": {laneBits: 64}, + "VPSLLW": {laneBits: 16, left: true, vector: true}, + "VPSLLD": {laneBits: 32, left: true, vector: true}, + "VPSLLQ": {laneBits: 64, left: true, vector: true}, + "VPSRLW": {laneBits: 16, vector: true}, + "VPSRLD": {laneBits: 32, vector: true}, + "VPSRLQ": {laneBits: 64, vector: true}, +} + +func (c *amd64Ctx) lowerPackedLogicalShift(op Op, ins Instr) (ok bool, terminated bool, err error) { + return c.lowerX86PackedShift(op, ins) +} + +func (c *amd64Ctx) loadPackedUniformLogicalShiftCount(count Operand, mmx bool) (string, error) { + if count.Kind == OpImm { + return fmt.Sprintf("%d", uint8(count.Imm)), nil + } + if mmx { + if count.Kind == OpMem || count.Kind == OpSym { + bytes, err := c.loadX86PackedShiftBytes(count, 8, true) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i8> %s to i64\n", value, bytes) + return "%" + value, nil + } + return c.loadLegacyMMXPackedSource(count) + } + // The ignored upper count qword still belongs to the complete native + // m128 read, even when a zero mask or saturated result is optimized. + countBytes, err := c.loadX86PackedShiftBytes(count, 16, true) + if err != nil { + return "", err + } + countWords := c.newTmp() + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", countWords, countBytes) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", low, countWords) + return "%" + low, nil +} + +func (c *amd64Ctx) emitPackedLogicalShift(spec amd64PackedLogicalShiftSpec, lanes int, data, count64 string) string { + inRange := c.newTmp() + safe64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %d\n", inRange, count64, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 0\n", safe64, inRange, count64) + count := "%" + safe64 + if spec.laneBits < 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", narrow, count, spec.laneBits) + count = "%" + narrow + } + counts := amd64SplatInteger(c, lanes, spec.laneBits, count) + shifted := c.newTmp() + operation := "lshr" + if spec.left { + operation = "shl" + } + typeName := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", shifted, operation, typeName, data, counts) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s zeroinitializer\n", result, inRange, typeName, shifted, typeName) + return "%" + result +} diff --git a/amd64_lower_logical_shift_test.go b/amd64_lower_logical_shift_test.go new file mode 100644 index 000000000..74ebd5394 --- /dev/null +++ b/amd64_lower_logical_shift_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedLogicalShiftCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast, dataReg, dstReg := 15, 22, 20, 21 + if target.goarch == "386" { + legacyLast, zLast, dataReg, dstReg = 7, 7, 6, 7 + } + var source strings.Builder + source.WriteString("TEXT packedlogicalshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $-1, M1\n", op) + fmt.Fprintf(&source, "\t%s M0, M1\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M1\n", op) + } + fmt.Fprintf(&source, "\t%s $3, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSLLW", "VPSLLD", "VPSLLQ", "VPSRLW", "VPSRLD", "VPSRLQ"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s $-1, %s0, %s1\n", op, width, width) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s%d\n", op, width, dstReg) + fmt.Fprintf(&source, "\t%s X0, %s%d, %s%d\n", op, width, dataReg, width, dstReg) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d, %s%d\n", op, width, dataReg, width, dstReg) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, %s20, K1, %s22\n", op, width, width) + fmt.Fprintf(&source, "\t%s.Z X0, %s20, K7, %s22\n", op, width, width) + } + } + fmt.Fprintf(&source, "\t%s $3, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s $3, 8(AX), Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s X0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, Z20, K2, Z22\n", op) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z20, K6, Z22\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), X%d\n", op, dstReg) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Y%d\n", op, dstReg) + fmt.Fprintf(&source, "\t%s.BCST $3, 8(AX), Z%d\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z $3, 8(AX), K5, Z22\n", op) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4"), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedlogicalshiftforms": {Name: "packedlogicalshiftforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-logical-shift-"+target.name+".ll", "packed-logical-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedLogicalShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSLLW.Z $3, X0", + "PSLLL $3", + "PSLLQ $-129, X0", + "PSRLW $128, X0", + "PSRLL M0, X1", + "PSRLQ X0, M1", + "VPSLLW $-129, X0, X1", + "VPSLLD $256, X0, X1", + "VPSLLQ Y0, X1, X2", + "VPSRLW X0, 8(AX), X2", + "VPSRLD Y0, X1, X2", + "VPSRLQ.BCST X0, X1, X2", + "VPSLLW.BCST $3, 8(AX), X2", + "VPSLLD.BCST $3, X1, X2", + "VPSRLQ.Z $3, X1, X2", + "VPSLLQ $3, X1, K0, X2", + "VPSRLD $3, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedLogicalShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSLLQ X8, X0", + "PSRLQ X0, X8", + "VPSLLQ $3, Z8, Z1", + "VPSRLQ X0, Z1, Z8", + "VPSLLD $3, X1, K1, X2", + "VPSRLD.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedLogicalShiftRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedLogicalShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed logical-shift forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_low_multiply.go b/amd64_lower_low_multiply.go new file mode 100644 index 000000000..b80576d44 --- /dev/null +++ b/amd64_lower_low_multiply.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedLowMultiply(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86PackedLowMultiplyProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed low multiply m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.spec.laneBits + lanes := form.byteWidth * 8 / bits + b, err := c.loadPackedCompareLanes(second, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + var mask, value string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.legacy && form.memory >= 0 { + // Keep the original SSE4.1 opcode and its alignment/fault behavior, + // including a dead result. An align-1 portable load is not equivalent. + value, err = c.emitPackedBinaryNativeMemory(first, bits, b, "pmulld") + } else { + // Intel's E4 class suppresses inactive source lanes. The caller still + // proves the entire possible read envelope independently of the mask. + a, loadErr := c.loadX86PackedShiftLanes(first, form.byteWidth, bits, form.broadcast, mask) + err = loadErr + if err == nil { + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul <%d x i%d> %s, %s\n", product, lanes, bits, b, a) + value = "%" + product + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + } + value = amd64ApplyIntegerLaneMask(c, lanes, bits, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, bits, value, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func isAMD64MemoryOperand(op Operand) bool { + switch op.Kind { + case OpFP, OpMem: + return true + case OpSym: + // Go TYPE_ADDR is an address, not a TYPE_MEM read. + return !strings.HasPrefix(strings.TrimSpace(op.Sym), "$") + default: + return false + } +} diff --git a/amd64_lower_lzcnt_test.go b/amd64_lower_lzcnt_test.go new file mode 100644 index 000000000..6cd9add73 --- /dev/null +++ b/amd64_lower_lzcnt_test.go @@ -0,0 +1,192 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86LZCNTCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "lzcntdata" + size := 8 + if goarch == "386" { + symbol = "lzcntdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT lzcntforms(SB),$0-0\n") + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + fmt.Fprintf(&source, "\tLZCNT%s AX, BX\n", width) + fmt.Fprintf(&source, "\tLZCNT%s 8(BX), CX\n", width) + fmt.Fprintf(&source, "\tLZCNT%s %s(SB), DX\n", width, symbol) + if goarch == "amd64" { + fmt.Fprintf(&source, "\tLZCNT%s R11, R12\n", width) + } else { + fmt.Fprintf(&source, "\tLZCNT%s AX, SP\n", width) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86LZCNTCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86LZCNTCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"lzcntforms": {Name: "lzcntforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.ctlz.i16", "@llvm.ctlz.i32", `"target-features"="+lzcnt"`} { + if !strings.Contains(ll, want) { + t.Fatalf("LZCNT lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "@llvm.ctlz.i64") { + t.Fatalf("LZCNTQ lowering omitted llvm.ctlz.i64:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "lzcnt-"+target.name+".ll", "lzcnt-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86LZCNTRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LZCNTW AL, BX"}, + {goarch: "amd64", instruction: "LZCNTL AX, AH"}, + {goarch: "amd64", instruction: "LZCNTQ X0, AX"}, + {goarch: "amd64", instruction: "LZCNTW AX, 0(BX)"}, + {goarch: "amd64", instruction: "LZCNTL $1, AX"}, + {goarch: "amd64", instruction: "LZCNTQ AX"}, + {goarch: "amd64", instruction: "LZCNTW.Z AX, BX"}, + {goarch: "386", instruction: "LZCNTL R8, AX"}, + {goarch: "386", instruction: "LZCNTQ AX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's LZCNT yml_rl table", test.instruction) + } + }) + } +} + +func TestAMD64LZCNTRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT lzcntsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x1122334455660000, AX + MOVQ $0xaabbccddeeff1234, BX + LZCNTW AX, BX + MOVQ BX, 0(DI) + SETCS 8(DI) + SETEQ 9(DI) + MOVQ $0x80000000, AX + MOVQ $0x1122334455667788, CX + LZCNTL AX, CX + MOVQ CX, 16(DI) + SETCS 24(DI) + SETEQ 25(DI) + MOVQ $1, 32(DI) + LZCNTQ 32(DI), DX + MOVQ DX, 40(DI) + SETCS 48(DI) + SETEQ 49(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "lzcntsemantics": { + Name: "lzcntsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void lzcntsemantics(uint8_t *); +int main(void) { + uint8_t out[56] = {0}; + lzcntsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0xaabbccddeeff0010)) return 10; + if (out[8] != 1 || out[9] != 0) return 11; + if (*(uint64_t *)(out+16) != 0) return 12; + if (out[24] != 0 || out[25] != 1) return 13; + if (*(uint64_t *)(out+40) != 63) return 14; + if (out[48] != 0 || out[49] != 0) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "lzcnt_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_lzcnt_vector_test.go b/amd64_lower_lzcnt_vector_test.go new file mode 100644 index 000000000..911062170 --- /dev/null +++ b/amd64_lower_lzcnt_vector_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedBitCountGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64PackedBitCountSpec{ + "VPOPCNTB": {laneBits: 8, kind: amd64PackedPopulationCount}, + "VPOPCNTW": {laneBits: 16, kind: amd64PackedPopulationCount}, + "VPOPCNTD": {laneBits: 32, kind: amd64PackedPopulationCount, broadcast: true}, + "VPOPCNTQ": {laneBits: 64, kind: amd64PackedPopulationCount, broadcast: true}, + "VPLZCNTD": {laneBits: 32, kind: amd64PackedLeadingZeroCount, broadcast: true}, + "VPLZCNTQ": {laneBits: 64, kind: amd64PackedLeadingZeroCount, broadcast: true}, + } + if len(amd64PackedBitCountSpecs) != len(expected) { + t.Fatalf("packed-bit-count grammar has %d entries, want %d", len(amd64PackedBitCountSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedBitCountSpecs[op]; !ok { + t.Errorf("packed-bit-count grammar omitted %s", op) + } else if got != want { + t.Errorf("packed-bit-count grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PackedLeadingZeroCountCompleteGoFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedlzcntforms(SB),$0-0\n") + for _, op := range []string{"VPLZCNTD", "VPLZCNTQ"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s0, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(R9), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 16(R9), K7, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST 24(R9), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(R9), K2, %s%d\n", op, width, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"packedlzcntforms": {Name: "packedlzcntforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.ctlz.v4i32", "@llvm.ctlz.v8i64", "select i1"} { + if !strings.Contains(ir, want) { + t.Fatalf("packed-LZCNT lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "packed-lzcnt-"+target.name+".ll", "packed-lzcnt-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedLeadingZeroCountRejectsFormsOutsideGoTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPLZCNTD X0, Y1"}, + {goarch: "amd64", instruction: "VPLZCNTQ Y0, Z1"}, + {goarch: "amd64", instruction: "VPLZCNTD X0, K0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTQ.Z X0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTD.BCST X0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTQ.SAE X0, X1"}, + {goarch: "amd64", instruction: "VPLZCNTD X0, 0(AX)"}, + {goarch: "amd64", instruction: "VPLZCNTQ AX, X1"}, + {goarch: "amd64", instruction: "VPLZCNTD X0, K1, K2, X1"}, + {goarch: "386", instruction: "VPLZCNTD Z8, Z0"}, + {goarch: "386", instruction: "VPLZCNTQ Z0, K1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPLZCNT _yvexpandpd grammar", test.instruction) + } + }) + } +} + +func TestAMD64PackedLeadingZeroCountRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT packedlzcntsemantics(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VPLZCNTD Z0, Z2 + VMOVDQU64 Z2, 0(AX) + VPLZCNTQ 0(BX), Z2 + VMOVDQU64 Z2, 64(AX) + VPLZCNTD Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VMOVDQU64 0(CX), Z1 + VPLZCNTQ.Z 0(BX), K1, Z1 + VMOVDQU64 Z1, 192(AX) + VPLZCNTD.BCST 0(BX), Z2 + VMOVDQU64 Z2, 256(AX) + VPLZCNTQ.BCST.Z 0(BX), K1, Z1 + VMOVDQU64 Z1, 320(AX) + STC + VPLZCNTD Z0, Z2 + SETCS 384(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"packedlzcntsemantics": { + Name: "packedlzcntsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void packedlzcntsemantics(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static unsigned lz32(uint32_t v) { unsigned n = 0; if (!v) return 32; while (!(v & UINT32_C(0x80000000))) { n++; v <<= 1; } return n; } +static unsigned lz64(uint64_t v) { unsigned n = 0; if (!v) return 64; while (!(v & UINT64_C(0x8000000000000000))) { n++; v <<= 1; } return n; } +int main(void) { + uint32_t source32[16], old32[16]; uint8_t out[385] = {0}; + const uint64_t mask = UINT64_C(0xa55a); + for (int i = 0; i < 16; i++) { source32[i] = i ? (UINT32_C(1) << ((i*7)%32)) : 0; old32[i] = UINT32_C(0x90000000) + i; } + packedlzcntsemantics(out, (const uint8_t *)source32, (const uint8_t *)old32, mask); + uint32_t *plain32=(uint32_t *)(out+0), *merge32=(uint32_t *)(out+128), *broadcast32=(uint32_t *)(out+256); + for (int i=0;i<16;i++) { + if (plain32[i] != lz32(source32[i])) return 10+i; + uint32_t want = (mask>>i)&1 ? lz32(source32[i]) : old32[i]; + if (merge32[i] != want) return 30+i; + if (broadcast32[i] != lz32(source32[0])) return 50+i; + } + uint64_t source64[8], old64[8], plain64[8], zero64[8], broadcast64[8]; + memcpy(source64, source32, 64); memcpy(old64, old32, 64); + memcpy(plain64,out+64,64); memcpy(zero64,out+192,64); memcpy(broadcast64,out+320,64); + for (int i=0;i<8;i++) { + if (plain64[i] != lz64(source64[i])) return 70+i; + uint64_t want = (mask>>i)&1 ? lz64(source64[i]) : 0; + if (zero64[i] != want) return 80+i; + want = (mask>>i)&1 ? lz64(source64[0]) : 0; + if (broadcast64[i] != want) return 90+i; + } + if (out[384] != 1) return 100; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_lzcnt_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_machine_register.go b/amd64_lower_machine_register.go new file mode 100644 index 000000000..8fef3a577 --- /dev/null +++ b/amd64_lower_machine_register.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" +) + +// lowerMachineRegisterMove covers the complete machine-state-register portion +// of Go 1.27's x86 ymovtab: CR, DR, and TR registers plus the descriptor-state +// aliases GDTR, IDTR, LDTR, MSW, and TASK. Segment and TLS moves are a separate +// family. +func (c *amd64Ctx) lowerMachineRegisterMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if baseOp != "MOVW" && baseOp != "MOVL" && baseOp != "MOVQ" { + return false, false, nil + } + if len(ins.Args) != 2 { + return false, false, nil + } + sourceClass, sourceIndex, sourceSpecial := x86MachineRegisterOperand(ins.Args[0]) + destinationClass, destinationIndex, destinationSpecial := x86MachineRegisterOperand(ins.Args[1]) + if !sourceSpecial && !destinationSpecial { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s machine-register %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if sourceSpecial == destinationSpecial { + return true, false, fmt.Errorf("%s %s machine-register form expects exactly one special register: %q", c.goarch, baseOp, ins.Raw) + } + + specialClass, specialIndex := sourceClass, sourceIndex + other := ins.Args[1] + readSpecial := sourceSpecial + if destinationSpecial { + specialClass, specialIndex = destinationClass, destinationIndex + other = ins.Args[0] + } + + switch specialClass { + case "cr": + return true, false, c.lowerControlDebugRegisterMove(baseOp, "cr", specialIndex, other, readSpecial, ins) + case "dr": + return true, false, c.lowerControlDebugRegisterMove(baseOp, "dr", specialIndex, other, readSpecial, ins) + case "tr": + return true, false, c.lowerTestRegisterMove(baseOp, specialIndex, other, readSpecial, ins) + case "gdtr", "idtr", "ldtr", "msw", "task": + return true, false, c.lowerDescriptorRegisterAlias(baseOp, specialClass, other, readSpecial, ins) + default: + panic("machine-register classifier drift") + } +} + +func x86MachineRegisterOperand(operand Operand) (class string, index int, ok bool) { + if operand.Kind != OpReg { + return "", 0, false + } + return x86MachineRegister(operand.Reg) +} + +func x86MachineRegister(reg Reg) (class string, index int, ok bool) { + name := strings.ToUpper(string(reg)) + switch name { + case "GDTR", "IDTR", "LDTR", "MSW", "TASK": + return strings.ToLower(name), 0, true + } + if len(name) < 3 { + return "", 0, false + } + prefix := strings.ToLower(name[:2]) + if prefix != "cr" && prefix != "dr" && prefix != "tr" { + return "", 0, false + } + index, err := strconv.Atoi(name[2:]) + if err != nil { + return "", 0, false + } + return prefix, index, true +} + +func (c *amd64Ctx) lowerControlDebugRegisterMove(op, class string, index int, other Operand, readSpecial bool, ins Instr) error { + if op != "MOVL" && op != "MOVQ" { + return fmt.Errorf("%s %s %s register form is absent from Go 1.27's ymovtab: %q", c.goarch, op, strings.ToUpper(class), ins.Raw) + } + allowed := false + switch class { + case "cr": + allowed = index == 0 || index == 2 || index == 3 || index == 4 || index == 8 + if index == 8 && c.goarch == "386" { + allowed = false + } + case "dr": + if op == "MOVL" { + allowed = index == 0 || index == 6 || index == 7 + } else { + allowed = index == 0 || index == 2 || index == 3 || index == 6 || index == 7 + } + } + if !allowed { + return fmt.Errorf("%s %s %s%d form is absent from Go 1.27's ymovtab: %q", c.goarch, op, strings.ToUpper(class), index, ins.Raw) + } + if other.Kind != OpReg || !isX86YrlRegisterForArch(other.Reg, c.goarch) { + return fmt.Errorf("%s %s %s%d expects a Go 1.27 Yrl GP register: %q", c.goarch, op, strings.ToUpper(class), index, ins.Raw) + } + + special := fmt.Sprintf("%%%s%d", class, index) + hardwareType := I64 + if c.goarch == "386" { + hardwareType = I32 + } + constraints := "~{memory},~{dirflag},~{fpsr},~{flags}" + if readSpecial { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", + call, hardwareType, "mov "+special+", $0", "=r,"+constraints) + value := "%" + call + resultType := hardwareType + if op == "MOVL" && hardwareType == I64 { + value = c.truncI64(value, I32) + resultType = I32 + } + return c.storeRegSized(other.Reg, resultType, value) + } + + value, err := c.loadReg(other.Reg) + if err != nil { + return err + } + if hardwareType == I32 { + value = c.truncI64(value, I32) + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", + "mov $0, "+special, "r,"+constraints, hardwareType, value) + return nil +} + +func (c *amd64Ctx) lowerTestRegisterMove(op string, index int, other Operand, readSpecial bool, ins Instr) error { + if op != "MOVL" || (index != 6 && index != 7) { + return fmt.Errorf("%s %s TR%d form is absent from Go 1.27's ymovtab: %q", c.goarch, op, index, ins.Raw) + } + if other.Kind == OpReg { + if !isX86YrlRegisterForArch(other.Reg, c.goarch) { + return fmt.Errorf("%s MOVL TR%d operand is outside Go 1.27's Yml class: %q", c.goarch, index, ins.Raw) + } + } else if !isAMD64MemoryOperand(other) { + return fmt.Errorf("%s MOVL TR%d expects a Go 1.27 Yml register or memory: %q", c.goarch, index, ins.Raw) + } + modRM := 0xc0 | byte(index<<3) + constraints := "~{memory},~{dirflag},~{fpsr},~{flags}" + if readSpecial { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 asm sideeffect %q, %q()\n", + call, fmt.Sprintf(".byte 0x0f, 0x24, 0x%02x", modRM), "={ax},"+constraints) + return c.storeScalarIntegerOperand(other, I32, "%"+call) + } + value, err := c.evalIntSized(other, I32) + if err != nil { + return err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i32 %s)\n", + fmt.Sprintf(".byte 0x0f, 0x26, 0x%02x", modRM), "{ax},"+constraints, value) + return nil +} + +func (c *amd64Ctx) lowerDescriptorRegisterAlias(op, class string, other Operand, readSpecial bool, ins Instr) error { + directOp := "" + switch class { + case "gdtr": + if op != "MOVL" && op != "MOVQ" { + break + } + if readSpecial { + directOp = "SGDT" + } else { + directOp = "LGDT" + } + case "idtr": + if op != "MOVL" && op != "MOVQ" { + break + } + if readSpecial { + directOp = "SIDT" + } else { + directOp = "LIDT" + } + case "ldtr": + if op == "MOVW" { + if readSpecial { + directOp = "SLDTW" + } else { + directOp = "LLDT" + } + } + case "msw": + if op == "MOVW" { + if readSpecial { + directOp = "SMSWW" + } else { + directOp = "LMSW" + } + } + case "task": + if op == "MOVW" { + if readSpecial { + directOp = "STRW" + } else { + directOp = "LTR" + } + } + } + if directOp == "" { + return fmt.Errorf("%s %s %s form is absent from Go 1.27's ymovtab: %q", c.goarch, op, strings.ToUpper(class), ins.Raw) + } + ok, _, err := c.lowerDescriptorTable(Op(directOp), Instr{Op: Op(directOp), Args: []Operand{other}, Raw: ins.Raw}) + if !ok { + panic("descriptor MOV alias lowering drift") + } + return err +} diff --git a/amd64_lower_maddubsw_test.go b/amd64_lower_maddubsw_test.go new file mode 100644 index 000000000..a035d381e --- /dev/null +++ b/amd64_lower_maddubsw_test.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86PackedUnsignedSignedByteMultiplyAddCompleteGoAssemblerForms(t *testing.T) { + amd64Source := `TEXT packedmaddubswforms(SB),NOSPLIT,$0-0 + PMADDUBSW X0, X1 + PMADDUBSW (AX), X15 + VPMADDUBSW X0, X1, X2 + VPMADDUBSW (AX), X1, X2 + VPMADDUBSW Y0, Y1, Y2 + VPMADDUBSW (AX), Y1, Y2 + VPMADDUBSW Z0, Z1, Z2 + VPMADDUBSW (AX), Z1, Z2 + VPMADDUBSW X20, X21, X22 + VPMADDUBSW Y20, Y21, Y22 + VPMADDUBSW Z20, Z21, Z22 + VPMADDUBSW X20, X21, K1, X22 + VPMADDUBSW.Z (AX), X21, K2, X22 + VPMADDUBSW Y20, Y21, K3, Y22 + VPMADDUBSW.Z (AX), Y21, K4, Y22 + VPMADDUBSW Z20, Z21, K5, Z22 + VPMADDUBSW.Z (AX), Z21, K6, Z22 + RET +` + ll := translateAMD64EcosystemCase(t, amd64Source, "packedmaddubswforms") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "packed-maddubsw-amd64.ll", "packed-maddubsw-amd64.o", ll) + + file, err := Parse(ArchAMD64, `TEXT packedmaddubswforms386(SB),NOSPLIT,$0-0 + PMADDUBSW X0, X1 + PMADDUBSW (AX), X7 + VPMADDUBSW X0, X1, X2 + VPMADDUBSW (AX), X1, X2 + VPMADDUBSW Y0, Y1, Y2 + VPMADDUBSW (AX), Y1, Y2 + VPMADDUBSW Z0, Z1, Z7 + VPMADDUBSW (AX), Z6, Z7 + RET +`) + if err != nil { + t.Fatal(err) + } + ll386, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{ + "packedmaddubswforms386": {Name: "packedmaddubswforms386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, "i386-unknown-linux-gnu", "packed-maddubsw-386.ll", "packed-maddubsw-386.o", ll386) +} + +func TestTranslateX86PackedUnsignedSignedByteMultiplyAddRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMADDUBSW.Z X0, X1", + "PMADDUBSW X0", + "PMADDUBSW M0, M1", + "PMADDUBSW Y0, Y1", + "PMADDUBSW X0, (AX)", + "VPMADDUBSW X0, X1", + "VPMADDUBSW X0, Y1, Y2", + "VPMADDUBSW X0, X1, (AX)", + "VPMADDUBSW.BCST (AX), X1, X2", + "VPMADDUBSW.Z X0, X1, X2", + "VPMADDUBSW X0, X1, K0, X2", + "VPMADDUBSW X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q, which is absent from Go 1.27's PMADDUBSW/VPMADDUBSW tables", instruction) + } + }) + } + + for _, instruction := range []string{ + "PMADDUBSW X8, X0", + "PMADDUBSW X0, X8", + "VPMADDUBSW Z8, Z1, Z2", + "VPMADDUBSW Z0, Z1, Z8", + "VPMADDUBSW X0, X1, K1, X2", + "VPMADDUBSW.Z X0, X1, K1, X2", + } { + file, err := Parse(ArchAMD64, "TEXT bad386(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "i386-unknown-linux-gnu", + Goarch: "386", + Sigs: map[string]FuncSig{"bad386": {Name: "bad386", Ret: Void}}, + }); err == nil { + t.Fatalf("386 Translate accepted architecture-restricted %q", instruction) + } + } +} diff --git a/amd64_lower_maddword.go b/amd64_lower_maddword.go new file mode 100644 index 000000000..c72a5ac1b --- /dev/null +++ b/amd64_lower_maddword.go @@ -0,0 +1,69 @@ +package plan9asm + +import "fmt" + +// Both Go packed multiply-add families share the same ordered operand, +// memory and write-generation contract. +func (c *amd64Ctx) lowerPackedWordMultiplyAdd(op Op, ins Instr) (bool, bool, error) { + return c.lowerWordMultiplyAddFamily(op, ins) +} + +func (c *amd64Ctx) emitPackedWordMultiplyAdd(wordLanes int, first, second string) string { + wordType := fmt.Sprintf("<%d x i16>", wordLanes) + wideType := fmt.Sprintf("<%d x i32>", wordLanes) + firstWide := c.newTmp() + secondWide := c.newTmp() + products := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", firstWide, wordType, first, wideType) + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", secondWide, wordType, second, wideType) + fmt.Fprintf(c.b, " %%%s = mul %s %%%s, %%%s\n", products, wideType, firstWide, secondWide) + resultLanes := wordLanes / 2 + result := "zeroinitializer" + for lane := 0; lane < resultLanes; lane++ { + low := c.newTmp() + high := c.newTmp() + sum := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", low, wideType, products, lane*2) + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", high, wideType, products, lane*2+1) + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %%%s\n", sum, low, high) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i32> %s, i32 %%%s, i32 %d\n", inserted, resultLanes, result, sum, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitPackedUnsignedSignedByteMultiplyAdd(byteLanes int, signedBytes, unsignedBytes string) string { + byteType := fmt.Sprintf("<%d x i8>", byteLanes) + wideType := fmt.Sprintf("<%d x i32>", byteLanes) + signedWide := c.newTmp() + unsignedWide := c.newTmp() + products := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext %s %s to %s\n", signedWide, byteType, signedBytes, wideType) + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", unsignedWide, byteType, unsignedBytes, wideType) + fmt.Fprintf(c.b, " %%%s = mul %s %%%s, %%%s\n", products, wideType, signedWide, unsignedWide) + resultLanes := byteLanes / 2 + result := "zeroinitializer" + for lane := 0; lane < resultLanes; lane++ { + low := c.newTmp() + high := c.newTmp() + sum := c.newTmp() + below := c.newTmp() + lowerClamped := c.newTmp() + above := c.newTmp() + clamped := c.newTmp() + narrow := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", low, wideType, products, lane*2) + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", high, wideType, products, lane*2+1) + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %%%s\n", sum, low, high) + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %%%s, -32768\n", below, sum) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -32768, i32 %%%s\n", lowerClamped, below, sum) + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, 32767\n", above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 32767, i32 %%%s\n", clamped, above, lowerClamped) + fmt.Fprintf(c.b, " %%%s = trunc i32 %%%s to i16\n", narrow, clamped) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i16> %s, i16 %%%s, i32 %d\n", inserted, resultLanes, result, narrow, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_mask_arithmetic.go b/amd64_lower_mask_arithmetic.go new file mode 100644 index 000000000..681fb5828 --- /dev/null +++ b/amd64_lower_mask_arithmetic.go @@ -0,0 +1,63 @@ +package plan9asm + +import "fmt" + +type amd64MaskArithmeticSpec struct { + bits int + shift string +} + +// amd64MaskArithmeticSpecs covers Go 1.27's complete KADD and KSHIFT +// families. KADD wraps at the named mask width; KSHIFT produces zero when the +// unsigned byte count is greater than or equal to that width. +var amd64MaskArithmeticSpecs = map[Op]amd64MaskArithmeticSpec{ + "KADDB": {bits: 8}, "KADDW": {bits: 16}, "KADDD": {bits: 32}, "KADDQ": {bits: 64}, + "KSHIFTLB": {bits: 8, shift: "shl"}, "KSHIFTLW": {bits: 16, shift: "shl"}, + "KSHIFTLD": {bits: 32, shift: "shl"}, "KSHIFTLQ": {bits: 64, shift: "shl"}, + "KSHIFTRB": {bits: 8, shift: "lshr"}, "KSHIFTRW": {bits: 16, shift: "lshr"}, + "KSHIFTRD": {bits: 32, shift: "lshr"}, "KSHIFTRQ": {bits: 64, shift: "lshr"}, +} + +func (c *amd64Ctx) lowerMaskArithmetic(op Op, ins Instr) error { + spec := amd64MaskArithmeticSpecs[op] + if spec.shift == "" { + if len(ins.Args) != 3 { + return fmt.Errorf("%s %s expects three K-register operands: %q", c.goarch, op, ins.Raw) + } + for _, arg := range ins.Args { + if !amd64IsKOperand(arg) { + return fmt.Errorf("%s %s expects only K0..K7 operands: %q", c.goarch, op, ins.Raw) + } + } + first, err := c.loadK(ins.Args[0].Reg) + if err != nil { + return err + } + second, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return err + } + first = c.maskMoveI64(first, spec.bits) + second = c.maskMoveI64(second, spec.bits) + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", added, first, second) + return c.storeK(ins.Args[2].Reg, c.maskMoveI64("%"+added, spec.bits)) + } + + if len(ins.Args) != 3 || !amd64UnsignedImmediate(ins.Args[0], 8) || !amd64IsKOperand(ins.Args[1]) || !amd64IsKOperand(ins.Args[2]) { + return fmt.Errorf("%s %s expects $u8, K0..K7, K0..K7: %q", c.goarch, op, ins.Raw) + } + source, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return err + } + source = c.maskMoveI64(source, spec.bits) + result := "0" + count := int(uint8(ins.Args[0].Imm)) + if count < spec.bits { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %d\n", shifted, spec.shift, source, count) + result = c.maskMoveI64("%"+shifted, spec.bits) + } + return c.storeK(ins.Args[2].Reg, result) +} diff --git a/amd64_lower_mask_arithmetic_test.go b/amd64_lower_mask_arithmetic_test.go new file mode 100644 index 000000000..3d3258719 --- /dev/null +++ b/amd64_lower_mask_arithmetic_test.go @@ -0,0 +1,166 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86MaskArithmeticCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT maskArithmeticForms(SB),$0-0\n") + for _, width := range []string{"B", "W", "D", "Q"} { + fmt.Fprintf(&source, "\tKADD%s K0, K7, K3\n", width) + fmt.Fprintf(&source, "\tKSHIFTL%s $0, K3, K4\n", width) + fmt.Fprintf(&source, "\tKSHIFTR%s $255, K4, K7\n", width) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "maskArithmeticForms": {Name: "maskArithmeticForms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "mask-arithmetic-"+target.name+".ll", "mask-arithmetic-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86MaskArithmeticRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "KADDB K1, K2", + "KADDW K1, AX, K2", + "KADDD K1, K2, K8", + "KADDQ.Z K1, K2, K3", + "KSHIFTLB K1, K2", + "KSHIFTLW $-1, K1, K2", + "KSHIFTLD $1, AX, K2", + "KSHIFTRQ $1, K1, K8", + "KSHIFTRB.Z $1, K1, K2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + for _, goarch := range []string{"amd64", "386"} { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + assertX86MaskArithmeticRejected(t, goarch, instruction) + } + }) + } +} + +func assertX86MaskArithmeticRejected(t *testing.T, goarch, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %s %q outside Go 1.27's KADD/KSHIFT tables", goarch, instruction) + } +} + +func TestAMD64MaskArithmeticRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT maskArithmetic(SB),$0-24 + MOVQ out+0(FP), AX + KMOVQ a+8(FP), K1 + KMOVQ b+16(FP), K2 + KADDB K1, K2, K3 + KADDW K1, K2, K4 + KSHIFTLW $3, K1, K5 + KSHIFTRW $4, K2, K6 + KSHIFTRB $9, K2, K7 + KMOVQ K3, 0(AX) + KMOVQ K4, 8(AX) + KMOVQ K5, 16(AX) + KMOVQ K6, 24(AX) + KMOVQ K7, 32(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }} + triple := "x86_64-apple-darwin" + if runtime.GOOS != "darwin" { + triple = testTargetTriple(runtime.GOOS, "amd64") + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "maskArithmetic": {Name: "maskArithmetic", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void maskArithmetic(uint64_t *, uint64_t, uint64_t); +int main(void) { + uint64_t out[5] = {0}; + uint64_t a = 0x123456789abcdef5ULL; + uint64_t b = 0xfedcba98765432f0ULL; + maskArithmetic(out, a, b); + if (out[0] != ((a + b) & 0xff)) return 1; + if (out[1] != ((a + b) & 0xffff)) return 2; + if (out[2] != ((a & 0xffff) << 3 & 0xffff)) return 3; + if (out[3] != ((b & 0xffff) >> 4)) return 4; + if (out[4] != 0) return 5; + return 0; +} +` + var runPrefix []string + if crossRosetta { + runPrefix = []string{"arch", "-x86_64"} + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_arithmetic", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_mask_logical.go b/amd64_lower_mask_logical.go new file mode 100644 index 000000000..9de663dd9 --- /dev/null +++ b/amd64_lower_mask_logical.go @@ -0,0 +1,153 @@ +package plan9asm + +import "fmt" + +type amd64MaskLogicalKind uint8 + +const ( + amd64MaskAnd amd64MaskLogicalKind = iota + amd64MaskAndNot + amd64MaskOr + amd64MaskXNOR + amd64MaskXOR + amd64MaskNot + amd64MaskTest + amd64MaskOrTest +) + +type amd64MaskLogicalSpec struct { + kind amd64MaskLogicalKind + bits int +} + +// amd64MaskLogicalSpecs is the complete Go 1.27 x86 mask-register logical +// family. Binary forms use _ykaddb (K,K,K); unary and flag-setting forms use +// _yknotb (K,K). Go accepts every listed form in both amd64 and 386 mode. +var amd64MaskLogicalSpecs = map[Op]amd64MaskLogicalSpec{ + "KANDB": {amd64MaskAnd, 8}, "KANDW": {amd64MaskAnd, 16}, "KANDD": {amd64MaskAnd, 32}, "KANDQ": {amd64MaskAnd, 64}, + "KANDNB": {amd64MaskAndNot, 8}, "KANDNW": {amd64MaskAndNot, 16}, "KANDND": {amd64MaskAndNot, 32}, "KANDNQ": {amd64MaskAndNot, 64}, + "KORB": {amd64MaskOr, 8}, "KORW": {amd64MaskOr, 16}, "KORD": {amd64MaskOr, 32}, "KORQ": {amd64MaskOr, 64}, + "KXNORB": {amd64MaskXNOR, 8}, "KXNORW": {amd64MaskXNOR, 16}, "KXNORD": {amd64MaskXNOR, 32}, "KXNORQ": {amd64MaskXNOR, 64}, + "KXORB": {amd64MaskXOR, 8}, "KXORW": {amd64MaskXOR, 16}, "KXORD": {amd64MaskXOR, 32}, "KXORQ": {amd64MaskXOR, 64}, + "KNOTB": {amd64MaskNot, 8}, "KNOTW": {amd64MaskNot, 16}, "KNOTD": {amd64MaskNot, 32}, "KNOTQ": {amd64MaskNot, 64}, + "KTESTB": {amd64MaskTest, 8}, "KTESTW": {amd64MaskTest, 16}, "KTESTD": {amd64MaskTest, 32}, "KTESTQ": {amd64MaskTest, 64}, + "KORTESTB": {amd64MaskOrTest, 8}, "KORTESTW": {amd64MaskOrTest, 16}, "KORTESTD": {amd64MaskOrTest, 32}, "KORTESTQ": {amd64MaskOrTest, 64}, +} + +func (c *amd64Ctx) lowerMaskLogical(op Op, ins Instr) error { + spec := amd64MaskLogicalSpecs[op] + wantArgs := 3 + if spec.kind == amd64MaskNot || spec.kind == amd64MaskTest || spec.kind == amd64MaskOrTest { + wantArgs = 2 + } + if len(ins.Args) != wantArgs { + return fmt.Errorf("amd64 %s expects %d K-register operands: %q", op, wantArgs, ins.Raw) + } + for _, arg := range ins.Args { + if arg.Kind != OpReg { + return fmt.Errorf("amd64 %s expects only K-register operands: %q", op, ins.Raw) + } + if _, ok := amd64ParseKReg(arg.Reg); !ok { + return fmt.Errorf("amd64 %s operand is not K0..K7: %q", op, ins.Raw) + } + } + + load := func(arg Operand) (string, error) { + value, err := c.loadK(arg.Reg) + if err != nil { + return "", err + } + if spec.bits == 64 { + return value, nil + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, value, amd64MaskWidthLiteral(spec.bits)) + return "%" + masked, nil + } + + first, err := load(ins.Args[0]) + if err != nil { + return err + } + second, err := load(ins.Args[1]) + if err != nil { + return err + } + if spec.kind == amd64MaskTest || spec.kind == amd64MaskOrTest { + return c.lowerMaskLogicalFlags(spec, first, second) + } + + var result string + switch spec.kind { + case amd64MaskNot: + not := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", not, first, amd64MaskWidthLiteral(spec.bits)) + result = "%" + not + case amd64MaskAnd, amd64MaskOr, amd64MaskXOR, amd64MaskXNOR, amd64MaskAndNot: + operation := "and" + lhs, rhs := first, second + switch spec.kind { + case amd64MaskOr: + operation = "or" + case amd64MaskXOR, amd64MaskXNOR: + operation = "xor" + case amd64MaskAndNot: + // Go's first source is the r/m operand and its second source is + // VEX.vvvv, so KANDN computes first & ~second. + notSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", notSecond, second, amd64MaskWidthLiteral(spec.bits)) + rhs = "%" + notSecond + } + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %s\n", combined, operation, lhs, rhs) + result = "%" + combined + if spec.kind == amd64MaskXNOR { + not := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", not, result, amd64MaskWidthLiteral(spec.bits)) + result = "%" + not + } + default: + return fmt.Errorf("amd64 %s has unknown mask logical operation", op) + } + if spec.bits != 64 { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, amd64MaskWidthLiteral(spec.bits)) + result = "%" + masked + } + return c.storeK(ins.Args[wantArgs-1].Reg, result) +} + +func (c *amd64Ctx) lowerMaskLogicalFlags(spec amd64MaskLogicalSpec, first, second string) error { + operation := "and" + if spec.kind == amd64MaskOrTest { + operation = "or" + } + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i64 %s, %s\n", combined, operation, first, second) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 0\n", zero, combined) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + + carry := c.newTmp() + if spec.kind == amd64MaskTest { + notSecond := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", notSecond, second, amd64MaskWidthLiteral(spec.bits)) + andNot := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %%%s\n", andNot, first, notSecond) + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 0\n", carry, andNot) + } else { + // KORTEST observes the same OR result for both flags: zero for ZF, + // all width-selected bits set for CF. KTEST's AND-NOT stays separate. + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, %s\n", carry, combined, amd64MaskWidthLiteral(spec.bits)) + } + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + c.clearX86StatusFlags(x86StatusAll &^ (x86StatusCF | x86StatusZF)) + return nil +} + +func amd64MaskWidthLiteral(bits int) string { + if bits == 64 { + return "-1" + } + return fmt.Sprintf("%d", uint64(1)<memory, K->GP, +// K-or-memory->K, and GP->K rows. Go accepts these rows in amd64 and 386 mode. +var amd64MaskMoveSpecs = map[Op]int{ + "KMOVB": 8, + "KMOVW": 16, + "KMOVD": 32, + "KMOVQ": 64, +} + +func (c *amd64Ctx) lowerMaskMove(op Op, ins Instr) error { + bits, recognized := amd64MaskMoveSpecs[op] + if !recognized { + return fmt.Errorf("amd64 unknown mask-move opcode %q", op) + } + if ins.Op != "" && Op(strings.ToUpper(string(ins.Op))) != op { + return fmt.Errorf("amd64 mask-move dispatch %q disagrees with instruction %q", op, ins.Op) + } + // The internal dispatcher owns the opcode argument, including for helper + // callers without Instr.Op. Bind a local copy before the shared validator; + // source/caller proofs still require their original complete instruction. + ins.Op = op + if len(ins.Args) != 2 { + return fmt.Errorf("amd64 %s expects two operands from Go 1.27's _ykmovb table: %q", op, ins.Raw) + } + src, dst := ins.Args[0], ins.Args[1] + srcK := amd64IsKOperand(src) + dstK := amd64IsKOperand(dst) + dstGP := amd64IsYrlOperand(dst, c.goarch) + dstMem := isAMD64MemoryOperand(dst) + + if !amd64MaskMoveOperandsValid(ins, c.goarch) { + return fmt.Errorf("amd64 %s operands are outside Go 1.27's _ykmovb table: %q", op, ins.Raw) + } + + value, err := c.loadMaskMoveValue(src, bits, srcK) + if err != nil { + return err + } + switch { + case dstK: + return c.storeK(dst.Reg, value) + case dstGP: + // The GP-destination encodings zero-extend B/W/D results. Store the + // already-masked value as the complete modeled register value. + base, _ := amd64FullRegBase(dst.Reg) + return c.storeReg(base, value) + case dstMem: + return c.storeMaskMoveMemory(dst, bits, value) + default: + return fmt.Errorf("amd64 %s has unsupported destination: %q", op, ins.Raw) + } +} + +func amd64MaskMoveOperandsValid(ins Instr, goarch string) bool { + if _, recognized := amd64MaskMoveSpecs[ins.Op]; !recognized || len(ins.Args) != 2 { + return false + } + src, dst := ins.Args[0], ins.Args[1] + sourceK, destinationK := amd64IsKOperand(src), amd64IsKOperand(dst) + return sourceK && (destinationK || amd64IsYrlOperand(dst, goarch) || isAMD64MemoryOperand(dst)) || + (amd64IsYrlOperand(src, goarch) || isAMD64MemoryOperand(src)) && destinationK +} + +func amd64IsKOperand(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + _, ok := amd64ParseKReg(operand.Reg) + return ok +} + +func amd64IsYrlOperand(operand Operand, goarch string) bool { + return operand.Kind == OpReg && isX86YrlRegisterForArch(operand.Reg, goarch) +} + +func (c *amd64Ctx) loadMaskMoveValue(src Operand, bits int, sourceIsK bool) (string, error) { + if sourceIsK { + value, err := c.loadK(src.Reg) + if err != nil { + return "", err + } + return c.maskMoveI64(value, bits), nil + } + + typ := amd64IntegerTypeForBits(bits) + value, err := c.evalIntSized(src, typ) + if err != nil { + return "", err + } + if bits == 64 { + return value, nil + } + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", extended, typ, value) + return "%" + extended, nil +} + +func (c *amd64Ctx) maskMoveI64(value string, bits int) string { + if bits == 64 { + return value + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, value, amd64MaskWidthLiteral(bits)) + return "%" + masked +} + +func (c *amd64Ctx) storeMaskMoveMemory(dst Operand, bits int, value string) error { + typ := amd64IntegerTypeForBits(bits) + stored := value + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", truncated, value, typ) + stored = "%" + truncated + } + switch dst.Kind { + case OpFP: + return c.storeFPResult(dst.FPOffset, typ, stored) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", typ, stored, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", typ, stored, ptr) + return nil + default: + return fmt.Errorf("amd64 mask move expected a memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_mask_to_vector.go b/amd64_lower_mask_to_vector.go new file mode 100644 index 000000000..850c7fec8 --- /dev/null +++ b/amd64_lower_mask_to_vector.go @@ -0,0 +1,64 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerMaskVectorMove(op Op, ins Instr) (bool, bool, error) { + if ins.Op == "" { + ins.Op = op + } + form, recognized, err := x86MaskVectorProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + if !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("mask/vector dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + vectorOperand, maskOperand := ins.Args[form.vector], ins.Args[form.mask] + lanes := form.byteWidth * 8 / form.spec.laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, form.spec.laneBits) + if !form.spec.maskToVector { + bytes, err := c.loadPackedCompareBytes(vectorOperand, form.byteWidth) + if err != nil { + return true, false, err + } + vector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", vector, form.byteWidth, bytes, vectorType) + signBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %%%s, zeroinitializer\n", signBits, vectorType, vector) + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %%%s to %s\n", packed, lanes, signBits, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + return true, false, c.storeK(maskOperand.Reg, result) + } + + mask, err := c.loadK(maskOperand.Reg) + if err != nil { + return true, false, err + } + vector := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %s, %d\n", shifted, mask, lane) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i1\n", bit, shifted) + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d -1, i%d 0\n", element, bit, form.spec.laneBits, form.spec.laneBits) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, vectorType, vector, form.spec.laneBits, element, lane) + vector = "%" + inserted + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, vectorType, vector, form.byteWidth) + // All widths are EVEX writes. Legacy storeX would preserve wider aliases + // and contradict the caller's complete defined-byte effect. + return true, false, c.storePackedMoveOperand(vectorOperand, form.byteWidth, "%"+bytes) +} diff --git a/amd64_lower_mask_to_vector_test.go b/amd64_lower_mask_to_vector_test.go new file mode 100644 index 000000000..8aa9cd0e7 --- /dev/null +++ b/amd64_lower_mask_to_vector_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86MaskToVectorMoveCompleteForms(t *testing.T) { + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT maskvectorforms(SB),$0-0\n") + index := 0 + for _, lane := range []string{"B", "W", "D", "Q"} { + for _, width := range []string{"X", "Y", "Z"} { + vector := index%7 + 1 + mask := index % 8 + fmt.Fprintf(&source, "\tVPMOVM2%s K%d, %s%d\n", lane, mask, width, vector) + fmt.Fprintf(&source, "\tVPMOV%s2M %s%d, K%d\n", lane, width, vector, mask) + index++ + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"maskvectorforms": {Name: "maskvectorforms", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, fmt.Sprintf("mask-to-vector-%s.ll", target.name), fmt.Sprintf("mask-to-vector-%s.o", target.name), ir) + }) + } +} + +func TestTranslateX86MaskVectorMoveRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPMOVM2B X0, X1", + "VPMOVM2W K0, 0(AX)", + "VPMOVM2D.Z K0, Z1", + "VPMOVM2Q K8, Z1", + "VPMOVB2M K0, K1", + "VPMOVW2M 0(AX), K1", + "VPMOVD2M.Z Z0, K1", + "VPMOVQ2M Z0, K8", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + for _, goarch := range []string{"amd64", "386"} { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %s %q outside Go 1.27's mask/vector move table", goarch, instruction) + } + } + }) + } +} + +func TestAMD64MaskVectorMoveRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT maskVectorMoves(SB),NOSPLIT,$0-16 + MOVQ out+0(FP), AX + KMOVQ mask+8(FP), K1 + VPMOVM2B K1, Z0 + VPMOVB2M Z0, K2 + KMOVQ K2, 0(AX) + VPMOVM2W K1, Z1 + VPMOVW2M Z1, K3 + KMOVQ K3, 8(AX) + VPMOVM2D K1, Z2 + VPMOVD2M Z2, K4 + KMOVQ K4, 16(AX) + VPMOVM2Q K1, Z3 + VPMOVQ2M Z3, K5 + KMOVQ K5, 24(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "maskVectorMoves": {Name: "maskVectorMoves", Args: []LLVMType{Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void maskVectorMoves(uint64_t *, uint64_t); +int main(void) { + uint64_t out[4] = {0}; + uint64_t mask = 0xd6a539c7f04b821dULL; + maskVectorMoves(out, mask); + if (out[0] != mask) return 1; + if (out[1] != (mask & 0xffffffffULL)) return 2; + if (out[2] != (mask & 0xffffULL)) return 3; + if (out[3] != (mask & 0xffULL)) return 4; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_vector_moves", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_mask_unpack.go b/amd64_lower_mask_unpack.go new file mode 100644 index 000000000..a49025534 --- /dev/null +++ b/amd64_lower_mask_unpack.go @@ -0,0 +1,44 @@ +package plan9asm + +import "fmt" + +// amd64MaskUnpackBits covers Go 1.27's complete KUNPCK family. The Plan 9 +// first source is the encoded r/m operand and becomes the low half; the second +// source is VEX.vvvv and becomes the high half of the destination mask. +var amd64MaskUnpackBits = map[Op]int{ + "KUNPCKBW": 8, + "KUNPCKWD": 16, + "KUNPCKDQ": 32, +} + +func (c *amd64Ctx) lowerMaskUnpack(op Op, ins Instr) error { + bits := amd64MaskUnpackBits[op] + if len(ins.Args) != 3 { + return fmt.Errorf("%s %s expects three K-register operands: %q", c.goarch, op, ins.Raw) + } + for _, arg := range ins.Args { + if !amd64IsKOperand(arg) { + return fmt.Errorf("%s %s expects only K0..K7 operands: %q", c.goarch, op, ins.Raw) + } + } + + low, err := c.loadK(ins.Args[0].Reg) + if err != nil { + return err + } + high, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return err + } + low = c.maskMoveI64(low, bits) + high = c.maskMoveI64(high, bits) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i64 %s, %d\n", shifted, high, bits) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %s\n", combined, shifted, low) + result := "%" + combined + if bits < 32 { + result = c.maskMoveI64(result, bits*2) + } + return c.storeK(ins.Args[2].Reg, result) +} diff --git a/amd64_lower_mask_unpack_test.go b/amd64_lower_mask_unpack_test.go new file mode 100644 index 000000000..ffa85ac0c --- /dev/null +++ b/amd64_lower_mask_unpack_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestTranslateX86MaskUnpackCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + const source = `TEXT maskunpackforms(SB),$0-0 + KUNPCKBW K0, K7, K3 + KUNPCKBW K2, K4, K4 + KUNPCKWD K7, K5, K3 + KUNPCKDQ K1, K6, K0 + KUNPCKDQ K5, K5, K5 + RET +` + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"maskunpackforms": {Name: "maskunpackforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"shl i64", "or i64", "255", "65535", "4294967295"} { + if !strings.Contains(ir, want) { + t.Fatalf("mask unpack lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "mask-unpack-"+target.name+".ll", "mask-unpack-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86MaskUnpackRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "KUNPCKBW K1, K2", + "KUNPCKWD K1, K2, K3, K4", + "KUNPCKDQ AX, K2, K3", + "KUNPCKBW K1, K8, K3", + "KUNPCKWD K1, K2, K8", + "KUNPCKDQ.Z K1, K2, K3", + } { + for _, goarch := range []string{"amd64", "386"} { + t.Run(goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's KUNPCK table for %s", instruction, goarch) + } + }) + } + } +} + +func TestAMD64MaskUnpackRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT maskunpacksemantics(SB),$0-24 + MOVQ out+0(FP), AX + KMOVQ a+8(FP), K1 + KMOVQ b+16(FP), K2 + STC + KUNPCKBW K1, K2, K3 + KMOVQ K3, 0(AX) + SETCS 24(AX) + KUNPCKWD K1, K2, K4 + KMOVQ K4, 8(AX) + KUNPCKDQ K1, K2, K2 + KMOVQ K2, 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "maskunpacksemantics": { + Name: "maskunpacksemantics", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void maskunpacksemantics(uint8_t *, uint64_t, uint64_t); +int main(void) { + uint8_t storage[25] = {0}; + uint64_t *out = (uint64_t *)storage; + uint64_t a = UINT64_C(0x0123456789abcdef); + uint64_t b = UINT64_C(0xfedcba9876543210); + maskunpacksemantics(storage, a, b); + if (out[0] != ((b & 0xff) << 8 | (a & 0xff))) return 10; + if (out[1] != ((b & 0xffff) << 16 | (a & 0xffff))) return 11; + if (out[2] != ((b & UINT64_C(0xffffffff)) << 32 | (a & UINT64_C(0xffffffff)))) return 12; + if (storage[24] != 1) return 13; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask_unpack_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_minmax.go b/amd64_lower_minmax.go new file mode 100644 index 000000000..3d774248d --- /dev/null +++ b/amd64_lower_minmax.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedIntegerMinMaxSpec struct { + laneBits int + signed bool + minimum bool + broadcast bool + mapNumber int + opcode int +} + +// lowerPackedIntegerMinMax implements the complete 16-opcode min/max family +// in Go 1.27. B/W/D forms use _yvandnpd, Q forms use _yvblendmpd, and only +// D/Q encodings enable scalar-memory broadcast. +func (c *amd64Ctx) lowerPackedIntegerMinMax(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, found := amd64PackedIntegerMinMaxSpecs[baseOp] + if !found { + return false, false, nil + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.sae || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s suffix is absent from its Go 1.27 encodings: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast && !spec.broadcast { + return true, false, fmt.Errorf("%s %s does not enable EVEX broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects source2, source1, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7 as its third operand: %q", baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || + !c.isGoPackedVectorMoveRegister(destination, byteWidth) || + !c.isGoPackedVectorMoveRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s source1 and destination must be matching Go vector registers: %q", c.goarch, baseOp, ins.Raw) + } + first := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source2: %q", c.goarch, baseOp, ins.Raw) + } + } else if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("%s %s source2 register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s source2 must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth * 8 / spec.laneBits + mask := "" + if masked { + mask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + firstValue, err := c.loadMaskedPackedCompareLanes(first, byteWidth, spec.laneBits, properties.broadcast, mask) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(ins.Args[1], byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + predicate := "ult" + if spec.signed { + predicate = "slt" + } + if !spec.minimum { + predicate = strings.Replace(predicate, "lt", "gt", 1) + } + compared := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s <%d x i%d> %s, %s\n", compared, predicate, lanes, spec.laneBits, secondValue, firstValue) + computed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i%d> %s, <%d x i%d> %s\n", computed, lanes, compared, lanes, spec.laneBits, secondValue, lanes, spec.laneBits, firstValue) + result := "%" + computed + if masked { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +var amd64PackedIntegerMinMaxSpecs = map[string]amd64PackedIntegerMinMaxSpec{ + "VPMINSB": {laneBits: 8, signed: true, minimum: true, mapNumber: 2, opcode: 0x38}, + "VPMINUB": {laneBits: 8, minimum: true, mapNumber: 1, opcode: 0xda}, + "VPMAXSB": {laneBits: 8, signed: true, mapNumber: 2, opcode: 0x3c}, + "VPMAXUB": {laneBits: 8, mapNumber: 1, opcode: 0xde}, + "VPMINSW": {laneBits: 16, signed: true, minimum: true, mapNumber: 1, opcode: 0xea}, + "VPMINUW": {laneBits: 16, minimum: true, mapNumber: 2, opcode: 0x3a}, + "VPMAXSW": {laneBits: 16, signed: true, mapNumber: 1, opcode: 0xee}, + "VPMAXUW": {laneBits: 16, mapNumber: 2, opcode: 0x3e}, + + "VPMINSD": {laneBits: 32, signed: true, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x39}, + "VPMINUD": {laneBits: 32, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x3b}, + "VPMAXSD": {laneBits: 32, signed: true, broadcast: true, mapNumber: 2, opcode: 0x3d}, + "VPMAXUD": {laneBits: 32, broadcast: true, mapNumber: 2, opcode: 0x3f}, + "VPMINSQ": {laneBits: 64, signed: true, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x39}, + "VPMINUQ": {laneBits: 64, minimum: true, broadcast: true, mapNumber: 2, opcode: 0x3b}, + "VPMAXSQ": {laneBits: 64, signed: true, broadcast: true, mapNumber: 2, opcode: 0x3d}, + "VPMAXUQ": {laneBits: 64, broadcast: true, mapNumber: 2, opcode: 0x3f}, +} diff --git a/amd64_lower_minmax_test.go b/amd64_lower_minmax_test.go new file mode 100644 index 000000000..9e3936c8d --- /dev/null +++ b/amd64_lower_minmax_test.go @@ -0,0 +1,138 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedIntegerMinMaxCompleteGoAssemblerForms(t *testing.T) { + type operation struct { + name string + broadcast bool + } + operations := []operation{ + {name: "VPMINSB"}, {name: "VPMINUB"}, {name: "VPMAXSB"}, {name: "VPMAXUB"}, + {name: "VPMINSW"}, {name: "VPMINUW"}, {name: "VPMAXSW"}, {name: "VPMAXUW"}, + {name: "VPMINSD", broadcast: true}, {name: "VPMINUD", broadcast: true}, + {name: "VPMAXSD", broadcast: true}, {name: "VPMAXUD", broadcast: true}, + {name: "VPMINSQ", broadcast: true}, {name: "VPMINUQ", broadcast: true}, + {name: "VPMAXSQ", broadcast: true}, {name: "VPMAXUQ", broadcast: true}, + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 22 + if target.goarch == "386" { + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packedintegerminmaxforms(SB),NOSPLIT,$0-0\n") + for _, operation := range operations { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", operation.name, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20, %s21\n", operation.name, width, width) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s20, %s22\n", operation.name, width, width) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", operation.name, width, width, width) + fmt.Fprintf(&source, "\t%s.Z 8(AX), %s21, K7, %s22\n", operation.name, width, width) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST.Z 16(AX), %s21, K2, %s22\n", operation.name, width, width) + } + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", operation.name, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", operation.name, zLast) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 16(AX), Z6, Z%d\n", operation.name, zLast) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K3, Z22\n", operation.name) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K4, Z22\n", operation.name) + if operation.broadcast { + fmt.Fprintf(&source, "\t%s.BCST.Z 16(AX), Z21, K5, Z22\n", operation.name) + } + } + } + source.WriteString("\tRET\n") + assembleX87ControlBytes(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4")) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedintegerminmaxforms": {Name: "packedintegerminmaxforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-integer-minmax-"+target.name+".ll", "packed-integer-minmax-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedIntegerMinMaxRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPMINUB X0, X1", + "VPMAXUB X0, X1, (AX)", + "VPMINSW X0, Y1, X2", + "VPMAXUW Z0, Z1, Y2", + "VPMINSD X0, X1, K0, X2", + "VPMAXUD X0, K1, X1, X2", + "VPMINSQ.Z X0, X1, X2", + "VPMAXUQ.SAE X0, X1, X2", + "VPMINUB.BCST 8(AX), X1, X2", + "VPMAXSW.BCST 8(AX), X1, X2", + "VPMINUD.BCST X0, X1, X2", + "VPMAXSQ.BCST 8(AX), Y1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedIntegerMinMaxRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPMINUB Z8, Z0, Z1", + "VPMAXUW Z0, Z8, Z1", + "VPMINSD Z0, Z1, Z8", + "VPMAXUQ X0, X1, K1, X2", + "VPMINSQ.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedIntegerMinMaxRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedIntegerMinMaxRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed min/max forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_minpos.go b/amd64_lower_minpos.go new file mode 100644 index 000000000..82661867c --- /dev/null +++ b/amd64_lower_minpos.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64MinimumPositionSpec struct { + vector bool + mapNumber int + opcode int +} + +// Both encodings have one X/m128 input, a 128-bit output and eight unsigned +// word lanes. Named lowering and raw VEX decoding share the family table. +var amd64MinimumPositionSpecs = map[Op]amd64MinimumPositionSpec{ + "PHMINPOSUW": {mapNumber: 2, opcode: 0x41}, + "VPHMINPOSUW": {vector: true, mapNumber: 2, opcode: 0x41}, +} + +// lowerPackedUnsignedWordMinimumPosition implements every Go 1.27 form of +// PHMINPOSUW (yxm_q4) and VPHMINPOSUW (_yvaesimc). Both tables are restricted +// to a single X/m128 source and an X destination; the VEX spelling is also a +// two-operand, destructive-looking Plan 9 form. +func (c *amd64Ctx) lowerPackedUnsignedWordMinimumPosition(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64MinimumPositionSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed forms in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects X/m128, X: %q", c.goarch, baseOp, ins.Raw) + } + validRegister := func(arg Operand, allowMemory bool) bool { + if spec.vector { + if allowMemory && isAMD64MemoryOperand(arg) { + return true + } + // Go's VEX frontend accepts X8-X15 on 386. Physical raw encodings + // remain restricted by the decoder, as for the other VEX families. + return amd64VEXVectorRegister(arg, 16) + } + if allowMemory && isAMD64MemoryOperand(arg) { + return true + } + return arg.Kind == OpReg && c.isGoLegacyXReg(arg.Reg) + } + if !validRegister(ins.Args[0], true) || !validRegister(ins.Args[1], false) { + return true, false, fmt.Errorf("%s %s requires an in-range X/m128 source and X destination: %q", c.goarch, baseOp, ins.Raw) + } + bytesValue, err := c.loadPackedCompareBytes(ins.Args[0], 16) + if err != nil { + return true, false, err + } + words := c.bitcastVectorBytesToIntegerLanes(16, 8, 16, bytesValue) + result := c.emitPackedUnsignedWordMinimumPosition(words) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %s to <16 x i8>\n", out, result) + if spec.vector { + return true, false, c.storePackedMoveOperand(ins.Args[1], 16, "%"+out) + } + return true, false, c.storeLegacyXViews(ins.Args[1].Reg, "%"+out) +} + +// Legacy SSE replaces the low 128 bits in every live view without changing +// the upper bytes. A narrow store into each wider slot expresses that directly. +func (c *amd64Ctx) storeLegacyXViews(dst Reg, value string) error { + index, valid := amd64ParseXReg(dst) + if !valid { + return fmt.Errorf("invalid legacy X register %s", dst) + } + for _, slots := range []map[int]string{c.xRegSlot, c.yRegSlot, c.zRegSlot} { + if slot, live := slots[index]; live { + fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s\n", value, slot) + } + } + return nil +} + +func (c *amd64Ctx) emitPackedUnsignedWordMinimumPosition(words string) string { + minimum := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <8 x i16> %s, i32 0\n", minimum, words) + minimumValue := "%" + minimum + indexValue := "0" + for lane := 1; lane < 8; lane++ { + value := c.newTmp() + less := c.newTmp() + selectedMinimum := c.newTmp() + selectedIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <8 x i16> %s, i32 %d\n", value, words, lane) + fmt.Fprintf(c.b, " %%%s = icmp ult i16 %%%s, %s\n", less, value, minimumValue) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %%%s, i16 %s\n", selectedMinimum, less, value, minimumValue) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %d, i16 %s\n", selectedIndex, less, lane, indexValue) + minimumValue = "%" + selectedMinimum + indexValue = "%" + selectedIndex + } + withMinimum := c.newTmp() + withIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <8 x i16> zeroinitializer, i16 %s, i32 0\n", withMinimum, minimumValue) + fmt.Fprintf(c.b, " %%%s = insertelement <8 x i16> %%%s, i16 %s, i32 1\n", withIndex, withMinimum, indexValue) + return "%" + withIndex +} diff --git a/amd64_lower_minpos_test.go b/amd64_lower_minpos_test.go new file mode 100644 index 000000000..eb2d79305 --- /dev/null +++ b/amd64_lower_minpos_test.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedUnsignedWordMinimumPositionCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + source := fmt.Sprintf("TEXT packedminimumpositionforms(SB),NOSPLIT,$0-0\n"+ + "\tPHMINPOSUW X0, X%d\n"+ + "\tPHMINPOSUW 8(AX), X%d\n"+ + "\tVPHMINPOSUW X0, X%d\n"+ + "\tVPHMINPOSUW 8(AX), X%d\n"+ + "\tRET\n", last, last, last, last) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedminimumpositionforms": {Name: "packedminimumpositionforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-minimum-position-"+target.name+".ll", "packed-minimum-position-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedUnsignedWordMinimumPositionRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PHMINPOSUW.Z X0, X1", + "PHMINPOSUW X0", + "PHMINPOSUW M0, M1", + "PHMINPOSUW Y0, Y1", + "PHMINPOSUW X0, (AX)", + "VPHMINPOSUW.Z X0, X1", + "VPHMINPOSUW X0", + "VPHMINPOSUW X0, X1, X2", + "VPHMINPOSUW Y0, Y1", + "VPHMINPOSUW Z0, Z1", + "VPHMINPOSUW X0, K1, X1", + "VPHMINPOSUW X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedWordMinimumPositionRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PHMINPOSUW X8, X0", + "PHMINPOSUW X0, X8", + "VPHMINPOSUW X16, X0", + "VPHMINPOSUW X0, X16", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedUnsignedWordMinimumPositionRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedUnsignedWordMinimumPositionRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PHMINPOSUW forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_mov.go b/amd64_lower_mov.go index 82de66ebc..d6b3b1cb5 100644 --- a/amd64_lower_mov.go +++ b/amd64_lower_mov.go @@ -6,8 +6,17 @@ import ( ) func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ok, terminated, err := c.lowerMOVNTI(op, ins); ok { + return ok, terminated, err + } + if ok, terminated, err := c.lowerMOVBE(op, ins); ok { + return ok, terminated, err + } switch op { - case "MOVQ", "MOVD", "MOVL", "MOVLQZX", "MOVLQSX", "MOVBQZX", "MOVBLZX", "MOVB", "MOVW", "MOVWLZX", "MOVWQZX", "MOVWQSX", "CMOVQLT": + case "MOVQ", "MOVD", "MOVL", "MOVB", "MOVW", "CMOVQLT", + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", "MOVBQSX", "MOVBQZX", + "MOVWLSX", "MOVWLZX", "MOVWQSX", "MOVWQZX", "MOVLQSX", "MOVLQZX", + "MOVSWW", "MOVZWW": // ok default: return false, false, nil @@ -15,71 +24,19 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err if len(ins.Args) != 2 { return true, false, fmt.Errorf("amd64 %s expects 2 operands: %q", op, ins.Raw) } - src, dst := ins.Args[0], ins.Args[1] - zeroExtendSmallDst := false - if op == "MOVLQZX" { - op = "MOVL" + if _, ok := amd64ScalarExtensionMoveSpecs[op]; ok { + return true, false, c.lowerScalarExtensionMove(op, ins) } + if form, known, err := x86ScalarNativeLoadProperties(op, ins, c.goarch); known { + if err != nil { + return true, false, err + } + return true, false, c.lowerScalarNativeLoad(form, ins) + } + src, dst := ins.Args[0], ins.Args[1] if op == "MOVD" { op = "MOVQ" } - if op == "MOVBQZX" || op == "MOVBLZX" { - op = "MOVB" - zeroExtendSmallDst = true - } - if op == "MOVWQZX" || op == "MOVWLZX" { - op = "MOVW" - zeroExtendSmallDst = true - } - if op == "MOVLQSX" || op == "MOVWQSX" { - // Sign-extend i32/i16 source to i64 destination register. - if dst.Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects dst reg: %q", op, ins.Raw) - } - srcTy := I32 - if op == "MOVWQSX" { - srcTy = I16 - } - var sv string - switch src.Kind { - case OpImm: - switch srcTy { - case I16: - sv = fmt.Sprintf("%d", int16(src.Imm)) - default: - sv = fmt.Sprintf("%d", int32(src.Imm)) - } - case OpReg, OpFP: - v64, err := c.evalI64(src) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", tr, v64, srcTy) - sv = "%" + tr - case OpMem: - p, ptrType, err := c.ptrFromMem(src.Mem) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, %s %s, align 1\n", ld, srcTy, ptrType, p) - sv = "%" + ld - case OpSym: - p, err := c.ptrFromSB(src.Sym) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", ld, srcTy, p) - sv = "%" + ld - default: - return true, false, fmt.Errorf("amd64 %s unsupported src: %q", op, ins.Raw) - } - se := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext %s %s to i64\n", se, srcTy, sv) - return true, false, c.storeReg(dst.Reg, "%"+se) - } // Vector moves are handled in lowerVec. if dst.Kind == OpReg { @@ -123,6 +80,17 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err widthTy := I8 if op == "MOVW" { widthTy = I16 + } else { + // ymovb uses Yrb for register operands on both sides. The + // compatibility closure through Ymb is identical for registers, + // including Go's synthesized BP/SI/DI support on 386, but excludes + // the amd64-only explicit low-byte aliases when translating 386. + if src.Kind == OpReg && !isGoYmbRegisterForArch(src.Reg, c.goarch) { + return true, false, fmt.Errorf("amd64 MOVB source register is outside its Go 1.27 class: %q", ins.Raw) + } + if dst.Kind == OpReg && !isGoYmbRegisterForArch(dst.Reg, c.goarch) { + return true, false, fmt.Errorf("amd64 MOVB destination register is outside its Go 1.27 class: %q", ins.Raw) + } } var small string switch src.Kind { @@ -165,11 +133,6 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err } switch dst.Kind { case OpReg: - if zeroExtendSmallDst { - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", z, widthTy, small) - return true, false, c.storeReg(dst.Reg, "%"+z) - } return true, false, c.storeRegSized(dst.Reg, widthTy, small) case OpFP: return true, false, c.storeFPResult(dst.FPOffset, widthTy, small) @@ -274,16 +237,11 @@ func (c *amd64Ctx) lowerMov(op Op, ins Instr) (ok bool, terminated bool, err err case "MOVL": // MOVL src, dst - if c.goarch != "386" && dst.Kind == OpFP { - // Preserve the established amd64 ABI result-slot lowering. - if src.Kind != OpReg { - return true, false, fmt.Errorf("amd64 MOVL expects reg, fp for stores: %q", ins.Raw) - } - v64, err := c.loadReg(src.Reg) - if err != nil { - return true, false, err - } - return true, false, c.storeFPResult(dst.FPOffset, I64, v64) + // FP result slots are memory operands in Go's ymovl table, so only a + // GP register or a 32-bit immediate can write them. Vector-register + // rows are handled by lowerVec before this scalar lowerer. + if dst.Kind == OpFP && src.Kind != OpReg && src.Kind != OpImm { + return true, false, fmt.Errorf("%s MOVL does not allow memory-to-result-memory: %q", c.goarch, ins.Raw) } i32v, err := c.evalIntSized(src, I32) diff --git a/amd64_lower_mov_extension.go b/amd64_lower_mov_extension.go new file mode 100644 index 000000000..8cc2ebc25 --- /dev/null +++ b/amd64_lower_mov_extension.go @@ -0,0 +1,65 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerScalarExtensionMove(op Op, ins Instr) error { + form, recognized, err := x86ScalarExtensionProperties(op, ins, c.goarch) + if err != nil { + return err + } + if !recognized { + return fmt.Errorf("unrecognized scalar extension opcode %s: %q", op, ins.Raw) + } + spec := form.spec + sourceType := amd64IntegerTypeForBits(spec.sourceBits) + destinationType := amd64IntegerTypeForBits(spec.destinationBits) + var value string + native := x86OrdinaryNativeMemoryOperand(form.source, c.goarch) || + x86CallerMemoryCertified(ins, 0) || c.fixedLocalMemoryCertified(ins, 0) + if native { + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return fmt.Errorf("%w: scalar extension needs its original native address size", ErrProbeNeedsContext) + } + pointer, pointerType, _, pointerErr := c.packedVectorMemoryPointer(form.source) + if pointerErr != nil { + return pointerErr + } + loaded := c.newTmp() + // Even a dead extension must read (and possibly fault on) its exact + // source width. No validity or alignment assumption follows. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", loaded, sourceType, pointerType, pointer) + value = "%" + loaded + } else { + value, err = c.evalIntSized(form.source, sourceType) + } + if err != nil { + return err + } + if spec.sourceBits < spec.destinationBits { + extended := c.newTmp() + extension := "zext" + if spec.signed { + extension = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", extended, extension, sourceType, value, destinationType) + value = "%" + extended + } + return c.storeRegSized(form.result, destinationType, value) +} + +// Only the paired volatile emitter can supply these conditional numerical +// result bytes. This creates no reusable pointee, bounds or caller proof. +func x86LoweredScalarExtensionRead(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + form, known, err := x86ScalarExtensionProperties(ins.Op, ins, arch) + if !known || err != nil || form.source.Kind != OpMem || form.source.Mem.Sym != "" || + !x86OrdinaryNativeMemoryOperand(form.source, arch) || ins.x86NumericalRead != nil { + return false, false + } + if ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86RawNearReturn != nil || + ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 || + !x86ResultIncomingPointerAddressIdentity(form.source, state, arch) { + return false, true + } + return x86ResultStore(Operand{Kind: OpReg, Reg: form.result}, int64(form.spec.destinationBits/8), + state, owned, resultOffset, arch), true +} diff --git a/amd64_lower_movbe.go b/amd64_lower_movbe.go new file mode 100644 index 000000000..f6ad7f3df --- /dev/null +++ b/amd64_lower_movbe.go @@ -0,0 +1,67 @@ +package plan9asm + +import "fmt" + +// lowerMOVBE implements both rows of Go 1.27's ymovbe table: memory to +// general register and general register to memory, for W/L/Q widths. +func (c *amd64Ctx) lowerMOVBE(op Op, ins Instr) (ok bool, terminated bool, err error) { + typ := LLVMType("") + switch op { + case "MOVBEW": + typ = I16 + case "MOVBEL": + typ = I32 + case "MOVBEQ", "MOVBEQQ": + if c.goarch == "386" { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", op, ins.Raw) + } + typ = I64 + default: + return false, false, nil + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects one memory and one general-register operand: %q", c.goarch, op, ins.Raw) + } + source, destination := ins.Args[0], ins.Args[1] + sourceRegister := source.Kind == OpReg && isX86YrlRegisterForArch(source.Reg, c.goarch) + destinationRegister := destination.Kind == OpReg && isX86YrlRegisterForArch(destination.Reg, c.goarch) + sourceMemory := isAMD64MemoryOperand(source) + destinationMemory := isAMD64MemoryOperand(destination) + if !((sourceMemory && destinationRegister) || (sourceRegister && destinationMemory)) { + return true, false, fmt.Errorf("%s %s operands are outside Go 1.27's Ym,Yrl/Yrl,Ym rows: %q", c.goarch, op, ins.Raw) + } + + value, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + swapped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.bswap.%s(%s %s)\n", swapped, typ, typ, typ, value) + if destinationRegister { + return true, false, c.storeRegSized(destination.Reg, typ, "%"+swapped) + } + return true, false, c.storeMOVBEMemory(destination, typ, "%"+swapped) +} + +func (c *amd64Ctx) storeMOVBEMemory(destination Operand, typ LLVMType, value string) error { + switch destination.Kind { + case OpMem: + pointer, pointerType, err := c.ptrFromMem(destination.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", typ, value, pointerType, pointer) + return nil + case OpFP: + return c.storeFPResult(destination.FPOffset, typ, value) + case OpSym: + pointer, err := c.ptrFromSB(destination.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", typ, value, pointer) + return nil + default: + return fmt.Errorf("MOVBE destination is not memory: %s", destination.String()) + } +} diff --git a/amd64_lower_movbe_test.go b/amd64_lower_movbe_test.go new file mode 100644 index 000000000..628e874cc --- /dev/null +++ b/amd64_lower_movbe_test.go @@ -0,0 +1,206 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86MOVBECompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA movedata+0(SB)/8, $1\n") + source.WriteString("GLOBL movedata(SB), $8\n") + source.WriteString("TEXT moveforms(SB),$0-0\n") + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + op := "MOVBE" + width + fmt.Fprintf(&source, "\t%s 8(BX), AX\n", op) + fmt.Fprintf(&source, "\t%s AX, 16(BX)\n", op) + fmt.Fprintf(&source, "\t%s movedata(SB), CX\n", op) + fmt.Fprintf(&source, "\t%s DX, movedata(SB)\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s 24(R11), R12\n", op) + fmt.Fprintf(&source, "\t%s R13, 32(R14)\n", op) + } + } + if goarch == "amd64" { + // Older external packages use the Go assembler's MOVBEQQ alias for + // the 64-bit MOVBE form. Keep this spelling covered alongside MOVBEQ. + source.WriteString("\tMOVBEQQ 40(R11), R12\n") + source.WriteString("\tMOVBEQQ R13, 48(R14)\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86MOVBECompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86MOVBECompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"moveforms": {Name: "moveforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.bswap.i16", "@llvm.bswap.i32", `"target-features"="+movbe"`} { + if !strings.Contains(ir, want) { + t.Fatalf("MOVBE lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "@llvm.bswap.i64") { + t.Fatalf("MOVBEQ lowering omitted i64 byte swap:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "movbe-"+target.name+".ll", "movbe-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86MOVBERejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "MOVBEW AX, BX"}, + {goarch: "amd64", instruction: "MOVBEL 0(AX), 0(BX)"}, + {goarch: "amd64", instruction: "MOVBEQ $1, 0(BX)"}, + {goarch: "amd64", instruction: "MOVBEW AL, 0(BX)"}, + {goarch: "amd64", instruction: "MOVBEL 0(AX), X0"}, + {goarch: "amd64", instruction: "MOVBEQ AX"}, + {goarch: "amd64", instruction: "MOVBEQQ AX"}, + {goarch: "amd64", instruction: "MOVBEL.Z 0(AX), BX"}, + {goarch: "386", instruction: "MOVBEQ 0(AX), BX"}, + {goarch: "386", instruction: "MOVBEL R8, 0(AX)"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's MOVBE ymovbe table", test.instruction) + } + }) + } +} + +func TestAMD64MOVBERuntimeSemanticsWidthsDirectionsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT movesemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + + MOVW $0x3412, 0(DI) + MOVQ $0x1122334455667788, AX + MOVBEW 0(DI), AX + MOVQ AX, 8(DI) + + MOVL $0x78563412, 16(DI) + MOVBEL 16(DI), BX + MOVQ BX, 24(DI) + + MOVQ $0xefcdab8967452301, 32(DI) + MOVBEQ 32(DI), CX + MOVQ CX, 40(DI) + + MOVQ $0x1122334455667788, AX + MOVBEW AX, 48(DI) + MOVBEL AX, 50(DI) + MOVBEQ AX, 54(DI) + + SETCS 62(DI) + SETOS 63(DI) + SETEQ 64(DI) + SETMI 65(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "movesemantics": { + Name: "movesemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void movesemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[72] = {0}; + movesemantics(out); + if (load64(out+8) != UINT64_C(0x1122334455661234)) return 10; + if (load64(out+24) != UINT64_C(0x12345678)) return 11; + if (load64(out+40) != UINT64_C(0x0123456789abcdef)) return 12; + const uint8_t stores[14] = {0x77,0x88, 0x55,0x66,0x77,0x88, 0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88}; + if (memcmp(out+48, stores, sizeof(stores)) != 0) return 13; + if (out[62] != 1 || out[63] != 1 || out[64] != 0 || out[65] != 1) return 14; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "movbe_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_move_half_test.go b/amd64_lower_move_half_test.go new file mode 100644 index 000000000..dc58f25bc --- /dev/null +++ b/amd64_lower_move_half_test.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86MoveHalfCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + last int + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", last: 15}, + {goarch: "386", triple: "i386-unknown-linux-gnu", last: 7}, + } { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT movehalfforms(SB),$0-0\n") + for _, op := range []string{"MOVLPS", "MOVHPS", "MOVLPD", "MOVHPD"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, target.last) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, target.last) + fmt.Fprintf(&source, "\t%s X%d, 16(BX)\n", op, target.last) + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"movehalfforms": {Name: "movehalfforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "extractelement <2 x i64>") || !strings.Contains(ll, "insertelement <2 x i64>") { + t.Fatalf("translation omitted half-vector lane operations:\n%s", ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "move-half-"+target.goarch+".ll", "move-half-"+target.goarch+".o", ll) + }) + } +} + +func TestTranslateX86MoveHalfRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "MOVHPS Y0, X1", + "MOVLPS X0, Y1", + "MOVHPD (AX), (BX)", + "MOVLPD $1, X0", + "MOVHPS.Z X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yxmov table", instruction) + } + }) + } +} diff --git a/amd64_lower_move_packed_float.go b/amd64_lower_move_packed_float.go new file mode 100644 index 000000000..1e8c9f0db --- /dev/null +++ b/amd64_lower_move_packed_float.go @@ -0,0 +1,176 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedVectorMove implements the complete Go 1.27 packed vector-move +// tables. VMOVAPD/APS/UPD/UPS share _yvmovapd, while VMOVDQA32/DQA64 and +// VMOVDQU8/DQU16/DQU32/DQU64 share _yvmovdqa32. All have bit-preserving move +// semantics; the element width only controls the granularity of an EVEX mask. +func (c *amd64Ctx) lowerPackedVectorMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := amd64PackedVectorMoveSpecs[Op(base)]; !recognized { + return false, false, nil + } + // Legacy helper calls may omit ins.Op. Never replace a conflicting source + // identity when binding the dispatch argument for the shared grammar. + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed move dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := amd64PackedVectorMoveProperties(ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + if form.undefinedOpcode() { + // Invalid encoding faults before any mask, vector or address access. + // An ordinary masked store would silently change the native semantics. + return c.lowerSyscall("UD2", Instr{Op: "UD2"}) + } + src, dst := ins.Args[0], ins.Args[form.destination] + byteWidth, elemBits := form.byteWidth, form.spec.elementBits + if form.spec.aligned && form.memory >= 0 { + if !form.ordinaryMemory(ins, c.goarch) && + !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return true, false, fmt.Errorf("%w: %s requires original native memory backing and alignment effects", ErrProbeNeedsContext, op) + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: aligned packed move needs its original native address size", ErrProbeNeedsContext) + } + return true, false, c.lowerAlignedPackedMoveMemory(ins, form) + } + for _, operand := range []Operand{src, dst} { + if operand.Kind != OpFP { + continue + } + if err := c.validateFPVectorSpan(operand.FPOffset, byteWidth); err != nil { + return true, false, fmt.Errorf("%s %s: %w: %q", c.goarch, op, err, ins.Raw) + } + } + if !form.spec.aligned && form.memory >= 0 && + (form.unalignedOrdinaryMemory(ins, c.goarch) || x86CallerMemoryCertified(ins, form.memory) || c.fixedLocalMemoryCertified(ins, form.memory)) { + return true, false, c.lowerUnalignedPackedMoveMemory(ins, form) + } + + if form.mask < 0 { + value, err := c.loadPackedCompareBytes(src, byteWidth) + if err != nil { + return true, false, err + } + return true, false, c.storePackedMoveOperand(dst, byteWidth, value) + } + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + // FP operands describe checked virtual slots, not necessarily contiguous + // native memory: each parameter/result can have its own alloca. Retain the + // per-slot loads, mask selection and stores below for those operands. + if (isAMD64MemoryOperand(src) && src.Kind != OpFP) || (isAMD64MemoryOperand(dst) && dst.Kind != OpFP) { + return true, false, c.lowerMaskedPackedMoveMemory(src, dst, byteWidth, elemBits, mask, form.zeroing) + } + value, err := c.loadPackedCompareBytes(src, byteWidth) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(dst, byteWidth) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / elemBits + computed := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, elemBits, value) + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, elemBits, oldBytes) + // The invalid .Z memory-destination encoding was handled before evaluating + // any operand. Valid zeroing forms always have a register destination. + result := amd64ApplyIntegerLaneMask(c, lanes, elemBits, computed, old, mask, form.zeroing && dst.Kind == OpReg) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, elemBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(dst, byteWidth, "%"+out) +} + +// VEX/EVEX register writes replace every overlapping view and zero all bits +// above the written width. Keep this separate from legacy SSE stores, which +// preserve those upper bits. +func (c *amd64Ctx) storePackedMoveOperand(dst Operand, byteWidth int, value string) error { + if dst.Kind != OpReg { + return c.storeVectorBytesOperand(dst, byteWidth, value) + } + index, valid := amd64VectorRegisterIndex(dst.Reg, byteWidth) + if !valid { + return fmt.Errorf("invalid packed move register %s", dst.Reg) + } + for _, view := range []struct { + bytes int + slots map[int]string + }{{16, c.xRegSlot}, {32, c.yRegSlot}, {64, c.zRegSlot}} { + slot, live := view.slots[index] + if !live { + continue + } + stored := value + if view.bytes != byteWidth { + indices := make([]string, view.bytes) + for i := range indices { + lane := i + if lane >= byteWidth { + lane = byteWidth // First byte of the zero second source. + } + indices[i] = fmt.Sprintf("i32 %d", lane) + } + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> <%s>\n", converted, byteWidth, value, byteWidth, view.bytes, strings.Join(indices, ", ")) + stored = "%" + converted + } + fmt.Fprintf(c.b, " store <%d x i8> %s, ptr %s\n", view.bytes, stored, slot) + } + return nil +} + +// Other EVEX families use this architecture/register axis without acquiring +// packed-move effects or caller-memory permissions. +func (c *amd64Ctx) isGoPackedVectorMoveRegister(arg Operand, byteWidth int) bool { + return amd64GoPackedVectorMoveRegister(arg, byteWidth, c.goarch) +} + +// Share the architectural lane mask across floating and integer moves. A +// select after an eager full-width load/store cannot suppress memory faults +// and must not be used for masked memory destinations (including write-only +// memory). LLVM 22's masked intrinsics access only enabled lanes. +func (c *amd64Ctx) lowerMaskedPackedMoveMemory(src, dst Operand, byteWidth, elemBits int, mask string, zeroing bool) error { + load := isAMD64MemoryOperand(src) + memory := dst + if load { + memory = src + } + pointer, pointerType, suffix, err := c.packedVectorMemoryPointer(memory) + if err != nil { + return err + } + if suffix == "" { + suffix = ".p0" + } + lanes := byteWidth * 8 / elemBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, elemBits) + predicate := amd64IntegerMaskVector(c, lanes, mask) + if load { + fallback := "zeroinitializer" + if !zeroing { + fallback, err = c.loadPackedCompareLanes(dst, byteWidth, elemBits, false) + if err != nil { + return err + } + } + loaded, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.masked.load.v%di%d%s(%s align 1 %s, <%d x i1> %s, %s %s)\n", loaded, vectorType, lanes, elemBits, suffix, pointerType, pointer, lanes, predicate, vectorType, fallback) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, vectorType, loaded, byteWidth) + return c.storePackedMoveOperand(dst, byteWidth, "%"+bytes) + } + value, err := c.loadPackedCompareLanes(src, byteWidth, elemBits, false) + if err != nil { + return err + } + fmt.Fprintf(c.b, " call void @llvm.masked.store.v%di%d%s(%s %s, %s align 1 %s, <%d x i1> %s)\n", lanes, elemBits, suffix, vectorType, value, pointerType, pointer, lanes, predicate) + return nil +} diff --git a/amd64_lower_move_sign_extend.go b/amd64_lower_move_sign_extend.go new file mode 100644 index 000000000..02eaa490a --- /dev/null +++ b/amd64_lower_move_sign_extend.go @@ -0,0 +1,226 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedExtendMove struct { + inputBits int + outputBits int + vector bool + zeroExtend bool +} + +// lowerPackedExtendMove implements all six signed and all six unsigned legacy +// and vector widening instructions exposed by Go 1.27. Ratio-two V forms use +// _yvcvtdq2pd; ratio-four and ratio-eight forms use _yvbroadcastss. +func (c *amd64Ctx) lowerPackedExtendMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + base, _, _ := strings.Cut(rawOp, ".") + if _, recognized := x86PackedZeroExtendSpecs[Op(base)]; recognized { + return c.lowerPackedZeroExtendMove(op, ins) + } + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + properties, ok := amd64PackedExtendMoveProperties(baseOp) + if !ok { + return false, false, nil + } + zeroing := false + switch suffix { + case "": + case "Z": + zeroing = true + default: + return true, false, fmt.Errorf("%s %s %s suffix is absent from Go 1.27's widening-move encodings: %q", c.goarch, baseOp, suffix, ins.Raw) + } + if !properties.vector && suffix != "" { + return true, false, fmt.Errorf("%s %s legacy yxm_q4 form does not accept suffixes: %q", c.goarch, baseOp, ins.Raw) + } + wantArgs := 2 + if len(ins.Args) == 3 { + wantArgs = 3 + } + if len(ins.Args) != wantArgs || (!properties.vector && len(ins.Args) != 2) { + return true, false, fmt.Errorf("%s %s expects source, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7 in the middle: %q", c.goarch, baseOp, ins.Raw) + } + + src := ins.Args[0] + dst := ins.Args[len(ins.Args)-1] + if dst.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + outputBytes := amd64VectorByteWidth(dst.Reg) + if properties.vector { + if !c.isGoPackedExtendDestination(dst, outputBytes) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's EVEX vector range: %q", c.goarch, baseOp, ins.Raw) + } + } else if outputBytes != 16 || !c.isGoLegacyPackedExtendX(dst) { + return true, false, fmt.Errorf("%s %s legacy destination must be an in-range X register: %q", c.goarch, baseOp, ins.Raw) + } + + sourceRegisterBytes := 16 + if properties.vector && outputBytes == 64 && properties.outputBits == properties.inputBits*2 { + sourceRegisterBytes = 32 + } + if src.Kind == OpReg { + if properties.vector { + if !amd64EVEXVectorRegister(src, sourceRegisterBytes) { + return true, false, fmt.Errorf("%s %s source register width does not match its Go 1.27 table: %q", c.goarch, baseOp, ins.Raw) + } + } else if !c.isGoLegacyPackedExtendX(src) { + return true, false, fmt.Errorf("%s %s legacy source must be an in-range X register or memory: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(src) { + return true, false, fmt.Errorf("%s %s source must be the table's vector register class or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := outputBytes * 8 / properties.outputBits + inputs, err := c.loadPackedExtendInputs(src, sourceRegisterBytes, lanes, properties.inputBits) + if err != nil { + return true, false, err + } + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", input, lanes, properties.inputBits, inputs, lane) + extended := c.newTmp() + extension := "sext" + if properties.zeroExtend { + extension = "zext" + } + fmt.Fprintf(c.b, " %%%s = %s i%d %%%s to i%d\n", extended, extension, properties.inputBits, input, properties.outputBits) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, properties.outputBits, result, properties.outputBits, extended, lane) + result = "%" + inserted + } + if masked { + mask, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(dst, outputBytes) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(outputBytes, lanes, properties.outputBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, properties.outputBits, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, properties.outputBits, result, outputBytes) + return true, false, c.storeVectorBytes(dst.Reg, outputBytes, "%"+out) +} + +func amd64PackedExtendMoveProperties(op string) (amd64PackedExtendMove, bool) { + vector := strings.HasPrefix(op, "V") + properties := amd64PackedExtendMove{vector: vector, zeroExtend: strings.Contains(op, "MOVZX")} + switch op { + case "PMOVSXBW", "VPMOVSXBW", "PMOVZXBW", "VPMOVZXBW": + properties.inputBits, properties.outputBits = 8, 16 + case "PMOVSXBD", "VPMOVSXBD", "PMOVZXBD", "VPMOVZXBD": + properties.inputBits, properties.outputBits = 8, 32 + case "PMOVSXBQ", "VPMOVSXBQ", "PMOVZXBQ", "VPMOVZXBQ": + properties.inputBits, properties.outputBits = 8, 64 + case "PMOVSXWD", "VPMOVSXWD", "PMOVZXWD", "VPMOVZXWD": + properties.inputBits, properties.outputBits = 16, 32 + case "PMOVSXWQ", "VPMOVSXWQ", "PMOVZXWQ", "VPMOVZXWQ": + properties.inputBits, properties.outputBits = 16, 64 + case "PMOVSXDQ", "VPMOVSXDQ", "PMOVZXDQ", "VPMOVZXDQ": + properties.inputBits, properties.outputBits = 32, 64 + default: + return amd64PackedExtendMove{}, false + } + return properties, true +} + +func (c *amd64Ctx) isGoLegacyPackedExtendX(arg Operand) bool { + index, ok := amd64VectorRegisterIndex(arg.Reg, 16) + if !ok || index >= 16 { + return false + } + return c.goarch != "386" || index < 8 +} + +func (c *amd64Ctx) isGoPackedExtendDestination(arg Operand, outputBytes int) bool { + if outputBytes != 16 && outputBytes != 32 && outputBytes != 64 { + return false + } + if !amd64EVEXVectorRegister(arg, outputBytes) { + return false + } + if c.goarch == "386" && outputBytes == 64 { + index, _ := amd64VectorRegisterIndex(arg.Reg, outputBytes) + return index < 8 + } + return true +} + +func (c *amd64Ctx) loadPackedExtendInputs(src Operand, registerBytes, lanes, inputBits int) (string, error) { + if src.Kind == OpReg { + bytesValue, err := c.loadPackedCompareBytes(src, registerBytes) + if err != nil { + return "", err + } + fullLanes := registerBytes * 8 / inputBits + full := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", full, registerBytes, bytesValue, fullLanes, inputBits) + if fullLanes == lanes { + return "%" + full, nil + } + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %%%s, i32 %d\n", value, fullLanes, inputBits, full, lane) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, inputBits, result, inputBits, value, lane) + result = "%" + inserted + } + return result, nil + } + vectorType := fmt.Sprintf("<%d x i%d>", lanes, inputBits) + switch src.Kind { + case OpMem: + ptr, ptrType, err := c.ptrFromMem(src.Mem) + if err != nil { + return "", err + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, %s %s, align 1\n", loaded, vectorType, ptrType, ptr) + return "%" + loaded, nil + case OpSym: + ptr, err := c.ptrFromSB(src.Sym) + if err != nil { + return "", err + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", loaded, vectorType, ptr) + return "%" + loaded, nil + case OpFP: + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + laneOperand := src + laneOperand.FPOffset += int64(lane * inputBits / 8) + value, err := c.evalIntSized(laneOperand, amd64IntegerTypeForBits(inputBits)) + if err != nil { + return "", err + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, inputBits, value, lane) + result = "%" + inserted + } + return result, nil + default: + return "", fmt.Errorf("expected packed extension register or memory source, got %s", src.String()) + } +} diff --git a/amd64_lower_move_zero_extend.go b/amd64_lower_move_zero_extend.go new file mode 100644 index 000000000..ba487a8fb --- /dev/null +++ b/amd64_lower_move_zero_extend.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedZeroExtendMove(op Op, ins Instr) (bool, bool, error) { + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("packed zero-extension dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op // Preserve the pre-existing helper's empty-op dispatch contract. + form, recognized, err := x86PackedZeroExtendProperties(ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + source, destination := ins.Args[0], ins.Args[form.destination] + lanes := form.byteWidth * 8 / form.spec.outputBits + if source.Kind == OpFP { + if err := c.validateFPVectorSpan(source.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask, inputs string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + switch { + case source.Kind == OpFP: + // Independent ABI0 fields are not a contiguous native vector. Read + // each declared byte and only then assemble the integer source lanes. + var bytes string + bytes, err = c.loadFPVectorBytes(source.FPOffset, form.memoryBytes) + if err == nil { + inputs = c.bitcastVectorBytesToIntegerLanes(form.memoryBytes, lanes, form.spec.inputBits, bytes) + } + case form.memory >= 0: + // E5 inactive lanes suppress reads; active accesses remain observable + // even if the widened result is dead. Read the shrinking source tuple, + // not its larger physical vector container or widened destination. + inputs, err = c.loadX86PackedShiftLanes(source, form.memoryBytes, form.spec.inputBits, false, mask) + default: + inputs, err = c.loadPackedExtendInputs(source, form.sourceRegisterBytes, lanes, form.spec.inputBits) + } + if err != nil { + return true, false, err + } + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + input, extended, inserted := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", input, lanes, form.spec.inputBits, inputs, lane) + fmt.Fprintf(c.b, " %%%s = zext i%d %%%s to i%d\n", extended, form.spec.inputBits, input, form.spec.outputBits) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, form.spec.outputBits, result, form.spec.outputBits, extended, lane) + result = "%" + inserted + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + var bytes string + bytes, err = c.loadPackedCompareBytes(destination, form.byteWidth) + if err != nil { + return true, false, err + } + old = c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.spec.outputBits, bytes) + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.spec.outputBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.spec.outputBits, result, form.byteWidth) + if !form.spec.vector { + return true, false, c.storeLegacyXViews(destination.Reg, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} diff --git a/amd64_lower_move_zero_extend_test.go b/amd64_lower_move_zero_extend_test.go new file mode 100644 index 000000000..0a7818aa7 --- /dev/null +++ b/amd64_lower_move_zero_extend_test.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestTranslateX86PackedZeroExtendMoveCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedzeroextendmoveforms(SB),4,$0-0\n") + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, row := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, target.goarch) { + source.WriteString(row + "\n") + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedzeroextendmoveforms": {Name: "packedzeroextendmoveforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-zero-extend-move-"+target.name+".ll", "packed-zero-extend-move-"+target.name+".o", ll) + + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"packedzeroextendmoveforms": {Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, target.triple, "zero-context.ll", "zero-context.o", module.String()) + + fset := token.NewFileSet() + declaration, err := parser.ParseFile(fset, "forms.go", "package forms\nfunc packedzeroextendmoveforms()\n", 0) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", target.goarch)}).Check("forms", fset, []*ast.File{declaration}, nil) + if err != nil { + t.Fatal(err) + } + bound, err := translateGoModuleInContext(ctx, GoPackage{Path: "forms", Types: typed, Syntax: []*ast.File{declaration}}, []byte(source.String()), GoModuleOptions{ + GOARCH: target.goarch, TargetTriple: target.triple, FileName: "forms.s", + }) + if err != nil { + t.Fatal(err) + } + defer bound.Module.Dispose() + compileLLVMToObject(t, llc, target.triple, "zero-go-module.ll", "zero-go-module.o", bound.Module.String()) + }) + } +} + +func TestTranslateX86PackedZeroExtendMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PMOVZXBW.Z X0, X1", + "PMOVZXBD X0", + "PMOVZXBQ M0, X1", + "PMOVZXWD Y0, X1", + "PMOVZXWQ X0, Y1", + "PMOVZXDQ X0, K1, X1", + "VPMOVZXBW Y0, Y1", + "VPMOVZXWD X0, Z1", + "VPMOVZXDQ Z0, Z1", + "VPMOVZXBD Y0, Z1", + "VPMOVZXBQ X0, K0, X1", + "VPMOVZXWQ.Z X0, X1", + "VPMOVZXBD.BCST 8(AX), X1", + "VPMOVZXWQ X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedZeroExtendMoveRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PMOVZXBW X8, X0", + "PMOVZXDQ X0, X8", + "VPMOVZXBW Y20, Z8", + "VPMOVZXBQ X20, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedZeroExtendMoveRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedZeroExtendMoveRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed zero-extension forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_movemask.go b/amd64_lower_movemask.go new file mode 100644 index 000000000..5f6a03dea --- /dev/null +++ b/amd64_lower_movemask.go @@ -0,0 +1,172 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64MoveMaskSpec struct { + opcode int + prefix int + vector bool + floatingLaneBits int +} + +// The legacy and VEX spellings share the Go table's register-to-GP shape. +// The raw decoder uses the VEX entries; x/arch handles legacy raw bytes. +var amd64MoveMaskSpecs = map[Op]amd64MoveMaskSpec{ + "PMOVMSKB": {opcode: 0xd7, prefix: 1}, + "VPMOVMSKB": {opcode: 0xd7, prefix: 1, vector: true}, + "MOVMSKPS": {opcode: 0x50, floatingLaneBits: 32}, + "MOVMSKPD": {opcode: 0x50, prefix: 1, floatingLaneBits: 64}, + "VMOVMSKPS": {opcode: 0x50, vector: true, floatingLaneBits: 32}, + "VMOVMSKPD": {opcode: 0x50, prefix: 1, vector: true, floatingLaneBits: 64}, +} + +type x86MoveMaskForm struct { + amd64MoveMaskSpec + byteWidth int +} + +// Shape is shared by lowering and result definedness. Acceptance supplies no +// initialized vector, MMX state, physical mode32 bank or caller permission. +func x86MoveMaskProperties(op Op, ins Instr, arch string) (x86MoveMaskForm, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64MoveMaskSpecs[Op(baseOp)] + form := x86MoveMaskForm{amd64MoveMaskSpec: spec} + if !recognized { + return form, false, nil + } + if arch != "amd64" && arch != "386" || rawOp != baseOp || + ins.Op != "" && !strings.EqualFold(string(ins.Op), baseOp) { + return form, true, fmt.Errorf("%s %s has an unbound mode, dispatch or suffix: %q", arch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { + return form, true, fmt.Errorf("%s %s expects vector-or-MMX source and GP destination: %q", arch, baseOp, ins.Raw) + } + if !isX86YrlRegisterForArch(ins.Args[1].Reg, arch) { + return form, true, fmt.Errorf("%s %s destination is outside Go's Yrl class: %q", arch, baseOp, ins.Raw) + } + limit := 15 + if !spec.vector && arch == "386" { + limit = 7 + } + if index, valid := amd64ParseXReg(ins.Args[0].Reg); valid && index <= limit { + form.byteWidth = 16 + } else if index, valid := amd64ParseYReg(ins.Args[0].Reg); spec.vector && valid && index <= 15 { + form.byteWidth = 32 + } else if _, valid := amd64ParseMReg(ins.Args[0].Reg); !spec.vector && spec.floatingLaneBits == 0 && valid { + form.byteWidth = 8 + } else { + return form, true, fmt.Errorf("%s %s source is outside its Go table: %q", arch, baseOp, ins.Raw) + } + return form, true, nil +} + +// No Go move-mask row accepts EVEX, mask-register, suffix or memory forms. +func (c *amd64Ctx) lowerPackedMoveMask(op Op, ins Instr) (ok bool, terminated bool, err error) { + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86MoveMaskProperties(op, ins, arch) + if !known || err != nil { + return known, false, err + } + if form.floatingLaneBits != 0 { + value, err := c.loadPackedCompareBytes(ins.Args[0], form.byteWidth) + if err != nil { + return true, false, err + } + mask32 := c.floatingMoveMask(value, form.byteWidth, form.floatingLaneBits) + return true, false, c.storeRegSized(ins.Args[1].Reg, I32, mask32) + } + + var mask32 string + if form.byteWidth == 16 { + value, err := c.loadX(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + mask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %s)\n", mask, value) + mask32 = "%" + mask + } else if form.byteWidth == 32 { + value, err := c.loadY(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", low, value, llvmI32RangeMask(0, 16)) + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", high, value, llvmI32RangeMask(16, 16)) + lowMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", lowMask, low) + highMask := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", highMask, high) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, 16\n", shifted, highMask) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i32 %%%s, %%%s\n", combined, shifted, lowMask) + mask32 = "%" + combined + } else { + value, err := c.loadReg(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <8 x i8>\n", bytes, value) + signBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <8 x i8> %%%s, zeroinitializer\n", signBits, bytes) + mask8 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i1> %%%s to i8\n", mask8, signBits) + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i32\n", widened, mask8) + mask32 = "%" + widened + } + + return true, false, c.storeRegSized(ins.Args[1].Reg, I32, mask32) +} + +func (c *amd64Ctx) floatingMoveMask(value string, byteWidth, laneBits int) string { + lanes := byteWidth * 8 / laneBits + integerLanes := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, value) + mask := "0" + for lane := 0; lane < lanes; lane++ { + laneValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", laneValue, lanes, laneBits, integerLanes, lane) + sign := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %%%s, 0\n", sign, laneBits, laneValue) + bit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i32\n", bit, sign) + positioned := "%" + bit + if lane != 0 { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i32 %%%s, %d\n", shifted, bit, lane) + positioned = "%" + shifted + } + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i32 %s, %s\n", combined, mask, positioned) + mask = "%" + combined + } + return mask +} + +func isX86YrlRegisterForArch(r Reg, goarch string) bool { + if !isAMD64YrlRegister(r) { + return false + } + if goarch != "386" { + return true + } + switch r { + case AX, BX, CX, DX, SP, BP, SI, DI: + return true + default: + return false + } +} diff --git a/amd64_lower_movhlps.go b/amd64_lower_movhlps.go new file mode 100644 index 000000000..34a82f4dc --- /dev/null +++ b/amd64_lower_movhlps.go @@ -0,0 +1,74 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerMoveHighLowPackedSingle implements all forms in Go 1.27's legacy yxr +// and AVX _yvmovhlps tables for the HLPS/LHPS register-only move family. +func (c *amd64Ctx) lowerMoveHighLowPackedSingle(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + vector := false + switch baseOp { + case "MOVHLPS", "MOVLHPS": + case "VMOVHLPS", "VMOVLHPS": + vector = true + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("amd64 %s has no instruction suffixes in Go 1.27's optab: %q", baseOp, ins.Raw) + } + + wantOperands := 2 + if vector { + wantOperands = 3 + } + if len(ins.Args) != wantOperands { + return true, false, fmt.Errorf("amd64 %s expects %d X-register operands: %q", baseOp, wantOperands, ins.Raw) + } + for _, operand := range ins.Args { + if operand.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s is register-only in Go 1.27's optab: %q", baseOp, ins.Raw) + } + index, ok := amd64ParseXReg(operand.Reg) + if !ok || (!vector && index > 15) { + return true, false, fmt.Errorf("amd64 %s operand is outside its Go 1.27 X-register class: %q", baseOp, ins.Raw) + } + } + + first, err := c.loadXAsI64x2(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + if !vector { + destination, err := c.loadXAsI64x2(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + mask := "" + if baseOp == "MOVHLPS" { + mask = "" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <2 x i32> %s\n", result, destination, first, mask) + return true, false, c.storeXFromI64x2(ins.Args[1].Reg, "%"+result) + } + + second, err := c.loadXAsI64x2(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + mask := "" + if baseOp == "VMOVHLPS" { + mask = "" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <2 x i32> %s\n", result, first, second, mask) + return true, false, c.storeXFromI64x2(ins.Args[2].Reg, "%"+result) +} diff --git a/amd64_lower_movnti.go b/amd64_lower_movnti.go new file mode 100644 index 000000000..21f0c336c --- /dev/null +++ b/amd64_lower_movnti.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +const x86NonTemporalMetadata = ", !nontemporal !0" + +// lowerMOVNTI implements Go 1.27's complete yrl_ml row for MOVNTIL/Q. Yrl is +// a general-register source and Yml accepts ordinary, symbol, and FP memory. +// Go's compatibility closure also accepts a general-register destination; +// Intel defines only a memory destination for that ModRM encoding, so model +// the Go-accepted register form as an explicit undefined-instruction trap +// rather than silently treating it as an ordinary register move. +func (c *amd64Ctx) lowerMOVNTI(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + typ := LLVMType("") + switch baseOp { + case "MOVNTIL": + typ = I32 + case "MOVNTIQ": + if c.goarch == "386" { + return true, false, fmt.Errorf("386 MOVNTIQ is illegal in 32-bit mode: %q", ins.Raw) + } + typ = I64 + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed forms in Go 1.27's yrl_ml table: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects exactly two operands: %q", c.goarch, baseOp, ins.Raw) + } + source, destination := ins.Args[0], ins.Args[1] + if source.Kind != OpReg || !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source must be an in-range general register: %q", c.goarch, baseOp, ins.Raw) + } + if destination.Kind == OpReg { + if !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yml compatibility closure: %q", c.goarch, baseOp, ins.Raw) + } + c.b.WriteString(" call void asm sideeffect \"ud2\", \"~{memory}\"()\n") + c.b.WriteString(" unreachable\n") + return true, true, nil + } + if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s %s destination must be Yml memory or a Go-compatible general register: %q", c.goarch, baseOp, ins.Raw) + } + if destination.Kind == OpMem && !x86MemoryRegistersValidForArch(destination.Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s destination uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.evalIntSized(source, typ) + if err != nil { + return true, false, err + } + return true, false, c.storeMOVNTIMemory(destination, typ, value) +} + +func x86MemoryRegistersValidForArch(memory MemRef, goarch string) bool { + for _, register := range []Reg{memory.Base, memory.Index} { + if register != "" && !isX86YrlRegisterForArch(register, goarch) { + return false + } + } + return true +} + +func (c *amd64Ctx) storeMOVNTIMemory(destination Operand, typ LLVMType, value string) error { + switch destination.Kind { + case OpMem: + pointer, pointerType, err := c.ptrFromMem(destination.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1%s\n", typ, value, pointerType, pointer, x86NonTemporalMetadata) + return nil + case OpFP: + return c.storeFPResultWithMetadata(destination.FPOffset, typ, value, x86NonTemporalMetadata) + case OpSym: + pointer, err := c.ptrFromSB(destination.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1%s\n", typ, value, pointer, x86NonTemporalMetadata) + return nil + default: + return fmt.Errorf("MOVNTI destination is not memory: %s", destination.String()) + } +} + +func fileUsesX86NonTemporalMetadata(file *File) bool { + if file == nil || file.Arch != ArchAMD64 { + return false + } + for _, function := range file.Funcs { + for _, instruction := range function.Instrs { + op := strings.ToUpper(string(instruction.Op)) + switch op { + case "MOVNTIL", "MOVNTIQ", "MOVNTQ", "MOVNTDQ", "MOVNTO", "MOVNTPD", "MOVNTPS", "VMOVNTDQ", "VMOVNTPD", "VMOVNTPS": + return true + } + } + } + return false +} diff --git a/amd64_lower_movnti_test.go b/amd64_lower_movnti_test.go new file mode 100644 index 000000000..d162f74e9 --- /dev/null +++ b/amd64_lower_movnti_test.go @@ -0,0 +1,217 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "runtime" + "strings" + "testing" +) + +func x86MOVNTICompleteFormsSource(goarch string) (string, map[string]FuncSig) { + var source strings.Builder + source.WriteString("DATA movntidata+0(SB)/8, $1\n") + source.WriteString("GLOBL movntidata(SB), $8\n") + source.WriteString("TEXT movntiforms(SB),$0-0\n") + source.WriteString("\tMOVNTIL AX, 8(BX)\n") + source.WriteString("\tMOVNTIL CX, movntidata(SB)\n") + if goarch == "amd64" { + source.WriteString("\tMOVNTIL R11, 16(R12)\n") + source.WriteString("\tMOVNTIQ AX, 24(BX)\n") + source.WriteString("\tMOVNTIQ CX, movntidata(SB)\n") + source.WriteString("\tMOVNTIQ R13, 32(R14)\n") + } + source.WriteString("\tRET\n") + + source.WriteString("TEXT movntiframe(SB),$0-4\n") + source.WriteString("\tMOVNTIL AX, ret+0(FP)\n") + source.WriteString("\tRET\n") + if goarch == "amd64" { + source.WriteString("TEXT movntiframeq(SB),$0-8\n") + source.WriteString("\tMOVNTIQ AX, ret+0(FP)\n") + source.WriteString("\tRET\n") + } + + // Yml's Go compatibility closure includes registers even though the ISA + // defines only memory destinations for MOVNTI. Keep this Go-accepted source + // form visible and require the translated function to trap explicitly. + source.WriteString("TEXT movntiregtrap(SB),$0-0\n") + source.WriteString("\tMOVNTIL AX, BX\n") + source.WriteString("\tRET\n") + + sigs := map[string]FuncSig{ + "movntiforms": {Name: "movntiforms", Ret: Void}, + "movntiframe": {Name: "movntiframe", Ret: I32, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}}}}, + "movntiregtrap": {Name: "movntiregtrap", Ret: Void}, + } + if goarch == "amd64" { + sigs["movntiframeq"] = FuncSig{Name: "movntiframeq", Ret: I64, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}} + } + return source.String(), sigs +} + +func TestTranslateX86MOVNTICompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source, sigs := x86MOVNTICompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`store i32`, `!nontemporal !0`, `!0 = !{i32 1}`, `"target-features"="+sse2"`, `asm sideeffect "ud2"`, "unreachable"} { + if !strings.Contains(ir, want) { + t.Fatalf("MOVNTI lowering omitted %q:\n%s", want, ir) + } + } + if target.goarch == "amd64" && !strings.Contains(ir, "store i64") { + t.Fatalf("MOVNTIQ lowering omitted i64 store:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "movnti-"+target.name+".ll", "movnti-"+target.name+".o", ir) + command := exec.Command(llc, "-mtriple="+target.triple, "-filetype=asm", "-o=-", "-") + command.Stdin = strings.NewReader(ir) + assembly, err := command.CombinedOutput() + if err != nil { + t.Fatalf("llc MOVNTI assembly failed: %v\n%s", err, assembly) + } + if !strings.Contains(strings.ToLower(string(assembly)), "movnti") { + t.Fatalf("LLVM 22 did not preserve the non-temporal integer store:\n%s", assembly) + } + }) + } +} + +func TestTranslateX86MOVNTIRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "MOVNTIL 0(AX), BX"}, + {goarch: "amd64", instruction: "MOVNTIL 0(AX), 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTIL $1, 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTIL X0, 8(BX)"}, + {goarch: "amd64", instruction: "MOVNTIL AX"}, + {goarch: "amd64", instruction: "MOVNTIL AX, 0(BX), CX"}, + {goarch: "amd64", instruction: "MOVNTIL.Z AX, 8(BX)"}, + {goarch: "386", instruction: "MOVNTIQ AX, 8(BX)"}, + {goarch: "386", instruction: "MOVNTIL R8, 8(BX)"}, + {goarch: "386", instruction: "MOVNTIL AX, 8(R9)"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's MOVNTI yrl_ml table", test.instruction) + } + }) + } +} + +func TestAMD64MOVNTIRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT movntisemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + MOVQ $0x1122334455667788, AX + MOVQ $0x8877665544332211, BX + MOVNTIL AX, 0(DI) + MOVNTIQ BX, 8(DI) + SFENCE + SETCS 16(DI) + SETOS 17(DI) + SETEQ 18(DI) + SETMI 19(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "movntisemantics": { + Name: "movntisemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void movntisemantics(uint8_t *); +static uint64_t load64(const uint8_t *p) { + uint64_t value; + memcpy(&value, p, sizeof(value)); + return value; +} +int main(void) { + uint8_t out[24] = {0}; + movntisemantics(out); + if (load64(out) != UINT64_C(0x0000000055667788)) return 10; + if (load64(out+8) != UINT64_C(0x8877665544332211)) return 11; + if (out[16] != 1 || out[17] != 1 || out[18] != 0 || out[19] != 1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "movnti_semantics", triple, ir, mainC, runPrefix) +} + +func TestX86MOVNTIGoAssemblerCompatibilityClosure(t *testing.T) { + for _, goarch := range []string{"amd64", "386"} { + for _, instruction := range []string{"MOVNTIL AX, 8(BX)", "MOVNTIL AX, BX", "MOVNTIL AX, ret+0(FP)"} { + t.Run(fmt.Sprintf("%s/%s", goarch, strings.NewReplacer(" ", "_", ",", "").Replace(instruction)), func(t *testing.T) { + requireX86GoAssemblerResult(t, goarch, "TEXT compat(SB),$0-8\n\t"+instruction+"\n\tRET\n", true) + }) + } + } +} diff --git a/amd64_lower_mpsadbw_test.go b/amd64_lower_mpsadbw_test.go new file mode 100644 index 000000000..04cd064ee --- /dev/null +++ b/amd64_lower_mpsadbw_test.go @@ -0,0 +1,379 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedSADGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64PackedSADSpec{ + "PSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADLegacyX}, + "VPSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADEVEXXYZ}, + "MPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADLegacyX}, + "VMPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADVEXXY}, + "VDBPSADBW": {kind: amd64PackedSADDoubleBlock, form: amd64PackedSADEVEXXYZ}, + } + if len(amd64PackedSADSpecs) != len(expected) { + t.Fatalf("packed SAD grammar has %d entries, want %d", len(amd64PackedSADSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedSADSpecs[op]; !ok { + t.Errorf("packed SAD grammar omitted %s", op) + } else if got != want { + t.Errorf("packed SAD grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86MultipleSADCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 15 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT mpsadforms(SB),$0-0\n") + fmt.Fprintf(&source, "\tMPSADBW $7, X0, X%d\n", last) + fmt.Fprintf(&source, "\tMPSADBW $7, 0(AX), X%d\n", last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\tVMPSADBW $7, X0, X1, X%d\n", last) + fmt.Fprintf(&source, "\tVMPSADBW $7, 16(AX), X1, X%d\n", last) + fmt.Fprintf(&source, "\tVMPSADBW $7, Y0, Y1, Y%d\n", last) + fmt.Fprintf(&source, "\tVMPSADBW $7, 32(AX), Y1, Y%d\n", last) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"mpsadforms": {Name: "mpsadforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "call <8 x i16> @llvm.x86.sse41.mpsadbw") { + t.Fatalf("MPSADBW lowering omitted the SSE4.1 intrinsic:\n%s", ir) + } + if target.goarch == "amd64" && !strings.Contains(ir, "call <16 x i16> @llvm.x86.avx2.mpsadbw") { + t.Fatalf("VMPSADBW Y lowering omitted the AVX2 intrinsic:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "mpsadbw-"+target.name+".ll", "mpsadbw-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86DoubleSADCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 23 + var source strings.Builder + source.WriteString("TEXT dbpsadforms(SB),$0-0\n") + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&source, "\tVDBPSADBW $0, %s0, %s1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVDBPSADBW $85, 0(AX), %s1, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVDBPSADBW $170, %s2, %s1, K1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVDBPSADBW.Z $255, 64(AX), %s1, K2, %s%d\n", width, width, last) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"dbpsadforms": {Name: "dbpsadforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "dbpsadbw-"+target.name+".ll", "dbpsadbw-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86DoubleSADRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VDBPSADBW $-1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW.Z $1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW.BCST $1, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW $1, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VDBPSADBW $1, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VDBPSADBW $1, X0, X1, K1, X2, X3"}, + {goarch: "386", instruction: "VDBPSADBW $1, X0, X1, X2"}, + {goarch: "386", instruction: "VDBPSADBW.Z $1, X0, X1, K1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VDBPSADBW table", test.instruction) + } + }) + } +} + +func TestTranslateX86MultipleSADRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "MPSADBW $-1, X0, X1"}, + {goarch: "amd64", instruction: "MPSADBW $256, X0, X1"}, + {goarch: "amd64", instruction: "MPSADBW.Z $1, X0, X1"}, + {goarch: "amd64", instruction: "MPSADBW $1, Y0, Y1"}, + {goarch: "amd64", instruction: "VMPSADBW $1, Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VMPSADBW $1, X0, Y1, Y2"}, + {goarch: "386", instruction: "MPSADBW $1, X0, X8"}, + {goarch: "386", instruction: "VMPSADBW $1, X0, X1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: test.goarch, TargetTriple: triple, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside the Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestTranslateX86PackedSADRejects386OutOfRangeForms(t *testing.T) { + for _, instruction := range []string{ + "PSADBW X0, X8", + "PSADBW 8(R9), X0", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the Go 386 SAD tables", instruction) + } + }) + } +} + +func TestAMD64MultipleSADRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT mpsadsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), CX + MOVQ b+16(FP), DX + MOVUPS 0(CX), X1 + MOVUPS 0(DX), X0 + STC + MPSADBW $0, X0, X1 + MOVUPS X1, 0(AX) + SETCS 16(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"mpsadsemantics": { + Name: "mpsadsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void mpsadsemantics(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t out[17] = {0}; + const uint8_t a[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + const uint8_t b[16] = {0}; + const uint16_t want[8] = {6, 10, 14, 18, 22, 26, 30, 34}; + uint16_t got[8]; + mpsadsemantics(out, a, b); + memcpy(got, out, 16); + for (int i = 0; i < 8; i++) if (got[i] != want[i]) return 10 + i; + if (out[16] != 1) return 20; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "multiple_sad_semantics", triple, ir, mainC, runPrefix) +} + +func TestAMD64DoubleSADRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT dbpsadsemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ a+8(FP), CX + MOVQ b+16(FP), DX + MOVQ old+24(FP), R8 + MOVQ mask+32(FP), R9 + KMOVQ R9, K1 + VMOVDQU8 0(CX), Z1 + VMOVDQU8 0(DX), Z0 + VMOVDQU16 0(R8), Z2 + STC + VDBPSADBW $27, Z0, Z1, K1, Z2 + VMOVDQU16 Z2, 0(AX) + VDBPSADBW.Z $228, Z0, Z1, K1, Z3 + VMOVDQU16 Z3, 64(AX) + SETCS 128(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"dbpsadsemantics": { + Name: "dbpsadsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void dbpsadsemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint16_t *, uint64_t); +static void oracle(uint16_t *out, const uint8_t *src1, const uint8_t *src2, const uint16_t *old, unsigned imm, uint64_t mask, int zeroing) { + uint8_t shuffled[64]; + for (int lane = 0; lane < 64; lane += 16) + for (int dword = 0; dword < 4; dword++) + for (int byte = 0; byte < 4; byte++) + shuffled[lane + dword * 4 + byte] = src2[lane + ((imm >> (2 * dword)) & 3) * 4 + byte]; + for (int block = 0; block < 64; block += 8) { + for (int word = 0; word < 4; word++) { + unsigned sum = 0; + int src1_base = block + (word >= 2 ? 4 : 0); + int shuffled_base = block + word; + for (int byte = 0; byte < 4; byte++) { + int difference = (int)src1[src1_base + byte] - (int)shuffled[shuffled_base + byte]; + sum += difference < 0 ? -difference : difference; + } + int index = block / 2 + word; + out[index] = (mask >> index) & 1 ? (uint16_t)sum : (zeroing ? 0 : old[index]); + } + } +} +int main(void) { + uint8_t a[64], b[64], bytes[129] = {0}; uint16_t old[32], got[64], want[64]; + for (int i = 0; i < 64; i++) { a[i] = (uint8_t)(i * 7 + 3); b[i] = (uint8_t)(255 - i * 5); } + for (int i = 0; i < 32; i++) old[i] = (uint16_t)(1000 + i); + uint64_t mask = UINT64_C(0xa55aa55a); + dbpsadsemantics(bytes, a, b, old, mask); + memcpy(got, bytes, 128); + oracle(want, a, b, old, 27, mask, 0); + oracle(want + 32, a, b, old, 228, mask, 1); + for (int i = 0; i < 64; i++) if (got[i] != want[i]) return 10 + i; + if (bytes[128] != 1) return 80; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "double_sad_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_msr.go b/amd64_lower_msr.go new file mode 100644 index 000000000..d19eaa063 --- /dev/null +++ b/amd64_lower_msr.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerMSRAccess implements the complete RDMSR/WRMSR instruction family. +// Go 1.27 uses the operand-free ynone optab entry for both instructions on +// amd64 and 386; the architectural operands are ECX and EDX:EAX. +func (c *amd64Ctx) lowerMSRAccess(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + switch baseOp { + case "RDMSR", "WRMSR": + // handled below + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + + cx, err := c.loadReg(CX) + if err != nil { + return true, false, err + } + cx32 := c.truncI64(cx, I32) + if baseOp == "RDMSR" { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call { i32, i32 } asm sideeffect \"rdmsr\", \"={ax},={dx},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %s)\n", call, cx32) + for index, reg := range []Reg{AX, DX} { + part := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { i32, i32 } %%%s, %d\n", part, call, index) + if err := c.storeRegSized(reg, I32, "%"+part); err != nil { + return true, false, err + } + } + return true, false, nil + } + + ax, err := c.loadReg(AX) + if err != nil { + return true, false, err + } + dx, err := c.loadReg(DX) + if err != nil { + return true, false, err + } + ax32 := c.truncI64(ax, I32) + dx32 := c.truncI64(dx, I32) + fmt.Fprintf(c.b, " call void asm sideeffect \"wrmsr\", \"{ax},{dx},{cx},~{dirflag},~{fpsr},~{flags}\"(i32 %s, i32 %s, i32 %s)\n", ax32, dx32, cx32) + return true, false, nil +} diff --git a/amd64_lower_mul_dword_qword.go b/amd64_lower_mul_dword_qword.go new file mode 100644 index 000000000..ed3e711f8 --- /dev/null +++ b/amd64_lower_mul_dword_qword.go @@ -0,0 +1,146 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerDwordToQwordMultiply(op Op, ins Instr) (bool, bool, error) { + form, known, err := x86DwordProductProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy dword product m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + if form.mmx { + if first.Kind == OpReg && first.Reg != ins.Args[form.first].Reg { + return true, false, fmt.Errorf("%w: Go GP-spelled dword product source names the MMX bank", ErrProbeNeedsContext) + } + var a string + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + var bytes string + bytes, err = c.loadX86PackedShiftBytes(first, 8, true) + if err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i8> %s to i64\n", value, bytes) + a = "%" + value + } + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + a = c.extendDwordScalar(c.lowDwordFromI64(a), false) + b = c.extendDwordScalar(c.lowDwordFromI64(b), false) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %s\n", product, a, b) + return true, false, c.storeReg(destination.Reg, "%"+product) + } + lanes := form.byteWidth / 8 + var mask, result string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.legacy && form.memory >= 0 { + b, err := c.loadPackedCompareLanes(second, 16, 64, false) + if err != nil { + return true, false, err + } + // Keep the original SSE opcode's complete m128 access and alignment + // behavior, including when the numerical result is dead. + result, err = c.emitPackedBinaryNativeMemory(first, 64, b, form.spec.nativeOpcode) + } else { + // EVEX E4 uses qword fault-suppression lanes, including m64bcst; + // only each qword's low dword participates in the multiplication. + var a, b string + a, err = c.loadDwordMultiplyEvenLanes(first, form.byteWidth, form.broadcast, mask) + if err == nil { + b, err = c.loadDwordMultiplyEvenLanes(second, form.byteWidth, false, "") + } + if err == nil { + a = c.extendDwordMultiplyLanes(a, lanes, form.spec.signed) + b = c.extendDwordMultiplyLanes(b, lanes, form.spec.signed) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul <%d x i64> %s, %s\n", product, lanes, a, b) + result = "%" + product + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, 64, false) + if err != nil { + return true, false, err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, 64, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", out, lanes, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func isAMD64XRegOperand(op Operand) bool { + return op.Kind == OpReg && isAMD64XReg(op.Reg) +} + +func (c *amd64Ctx) loadDwordMultiplyEvenLanes(op Operand, byteWidth int, broadcast bool, mask string) (string, error) { + lanes := byteWidth / 8 + qwords, err := c.loadX86PackedShiftLanes(op, byteWidth, 64, broadcast, mask) + if err != nil { + return "", err + } + even := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc <%d x i64> %s to <%d x i32>\n", even, lanes, qwords, lanes) + return "%" + even, nil +} + +func (c *amd64Ctx) extendDwordMultiplyLanes(value string, lanes int, signed bool) string { + extended := c.newTmp() + op := "zext" + if signed { + op = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s <%d x i32> %s to <%d x i64>\n", extended, op, lanes, value, lanes) + return "%" + extended +} + +func (c *amd64Ctx) lowDwordFromI64(value string) string { + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", low, value) + return "%" + low +} + +func (c *amd64Ctx) extendDwordScalar(value string, signed bool) string { + extended := c.newTmp() + op := "zext" + if signed { + op = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s i32 %s to i64\n", extended, op, value) + return "%" + extended +} diff --git a/amd64_lower_mul_qword_test.go b/amd64_lower_mul_qword_test.go new file mode 100644 index 000000000..6f0b7fc75 --- /dev/null +++ b/amd64_lower_mul_qword_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedQwordMultiplyCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + source := fmt.Sprintf(` +TEXT packedqwordmultiplyforms(SB),4,$0-0 + VPMULLQ X1, X20, X21 + VPMULLQ 8(AX), Y20, Y21 + VPMULLQ Z1, Z2, Z%d + VPMULLQ.BCST 40(AX), X20, X21 + VPMULLQ.BCST 48(AX), Y20, Y21 + VPMULLQ.BCST 56(AX), Z2, Z%d +`, lastZ, lastZ) + if target.goarch == "amd64" { + source += ` + VPMULLQ X1, X20, K1, X21 + VPMULLQ.Z Y1, Y20, K2, Y21 + VPMULLQ.BCST.Z 104(AX), Z20, K3, Z21 +` + } + source += "\tRET\n" + if target.goarch == "386" { + // Go accepts these high X/Y spellings, but syntax acceptance + // does not provide a physical high register in 32-bit mode. + // Their separate Context regression retains the original forms. + source = strings.NewReplacer("X20", "X6", "X21", "X7", "Y20", "Y6", "Y21", "Y7").Replace(source) + } + goAssemblerOracleOutput(t, target.goarch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"packedqwordmultiplyforms": {Name: "packedqwordmultiplyforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-qword-multiply-"+target.name+".ll", "packed-qword-multiply-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedQwordMultiplyRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPMULLQ X0, X1", + "VPMULLQ X0, Y1, Y2", + "VPMULLQ X0, (AX), X2", + "VPMULLQ X0, X1, K0, X2", + "VPMULLQ X0, X1, AX", + "VPMULLQ.Z X0, X1, X2", + "VPMULLQ.BCST X0, X1, X2", + "VPMULLQ.Z.BCST (AX), X1, K1, X2", + "VPMULLQ.RN_SAE X0, X1, X2", + } { + assertX86PackedQwordMultiplyRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + } + assertX86PackedQwordMultiplyRejected(t, "386", "i386-unknown-linux-gnu", "VPMULLQ X0, X1, K1, X2") + assertX86PackedQwordMultiplyRejected(t, "386", "i386-unknown-linux-gnu", "VPMULLQ Z0, Z1, Z8") +} + +func assertX86PackedQwordMultiplyRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPMULLQ forms", instruction) + } +} diff --git a/amd64_lower_muldiv_scalar.go b/amd64_lower_muldiv_scalar.go new file mode 100644 index 000000000..fee1d2b13 --- /dev/null +++ b/amd64_lower_muldiv_scalar.go @@ -0,0 +1,362 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarMultiplyDivide implements the complete scalar x86 multiply and +// divide families described by Go 1.27's ydivb, ydivl, yimul, and yimul3 +// tables. The one-operand forms use the architectural accumulator pairs; +// IMULW/L/Q also have the table's explicit two- and three-operand forms. +func (c *amd64Ctx) lowerScalarMultiplyDivide(op Op, ins Instr) (ok bool, terminated bool, err error) { + name := strings.ToUpper(string(op)) + stem, bits, threeOperandName, recognized := x86ScalarMultiplyDivideProperties(name) + if !recognized { + return false, false, nil + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", name, ins.Raw) + } + + if threeOperandName { + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects immediate, source, destination register: %q", c.goarch, name, ins.Raw) + } + form, _, err := x86ExplicitIMULProperties(Op(name), ins, c.goarch) + if err != nil { + return true, false, err + } + return true, false, c.emitX86ExplicitIMUL(bits, form.immediate, form.source, form.destination) + } + + if stem == "IMUL" && bits != 8 && len(ins.Args) != 1 { + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects one or two operands: %q", c.goarch, name, ins.Raw) + } + form, _, err := x86ExplicitIMULProperties(Op(name), ins, c.goarch) + if err != nil { + return true, false, err + } + return true, false, c.emitX86ExplicitIMUL(bits, Operand{}, form.source, form.destination) + } + + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one GP-or-memory source: %q", c.goarch, name, ins.Raw) + } + if !isX86ScalarMultiplyDivideSource(ins.Args[0], bits, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's %s class: %q", c.goarch, name, x86ScalarMultiplyDivideSourceClass(bits), ins.Raw) + } + + switch stem { + case "MUL": + return true, false, c.emitX86ImplicitMultiply(bits, false, ins.Args[0]) + case "IMUL": + return true, false, c.emitX86ImplicitMultiply(bits, true, ins.Args[0]) + case "DIV": + return true, false, c.emitX86ImplicitDivide(bits, false, ins.Args[0]) + case "IDIV": + return true, false, c.emitX86ImplicitDivide(bits, true, ins.Args[0]) + default: + panic("unreachable scalar multiply/divide stem") + } +} + +func x86ScalarMultiplyDivideProperties(op string) (stem string, bits int, threeOperandName bool, ok bool) { + switch op { + case "MULB": + return "MUL", 8, false, true + case "MULW": + return "MUL", 16, false, true + case "MULL": + return "MUL", 32, false, true + case "MULQ": + return "MUL", 64, false, true + case "IMULB": + return "IMUL", 8, false, true + case "IMULW": + return "IMUL", 16, false, true + case "IMULL": + return "IMUL", 32, false, true + case "IMULQ": + return "IMUL", 64, false, true + case "IMUL3W": + return "IMUL", 16, true, true + case "IMUL3L": + return "IMUL", 32, true, true + case "IMUL3Q": + return "IMUL", 64, true, true + case "DIVB": + return "DIV", 8, false, true + case "DIVW": + return "DIV", 16, false, true + case "DIVL": + return "DIV", 32, false, true + case "DIVQ": + return "DIV", 64, false, true + case "IDIVB": + return "IDIV", 8, false, true + case "IDIVW": + return "IDIV", 16, false, true + case "IDIVL": + return "IDIV", 32, false, true + case "IDIVQ": + return "IDIV", 64, false, true + default: + return "", 0, false, false + } +} + +func x86ScalarMultiplyDivideSourceClass(bits int) string { + if bits == 8 { + return "Ymb" + } + return "Yml" +} + +func isX86ScalarMultiplyDivideSource(operand Operand, bits int, goarch string) bool { + if operand.Kind == OpReg { + if bits == 8 { + return isGoYmbRegisterForArch(operand.Reg, goarch) + } + return isX86YrlRegisterForArch(operand.Reg, goarch) + } + if operand.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + return false + } + return isAMD64MemoryOperand(operand) +} + +func validateX86ExplicitIMUL(goarch, op string, bits int, ins Instr, threeOperands bool) error { + immediate := Operand{} + source := ins.Args[0] + destination := ins.Args[1] + if threeOperands { + immediate = ins.Args[0] + source = ins.Args[1] + destination = ins.Args[2] + } + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, goarch) { + return fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", goarch, op, ins.Raw) + } + if threeOperands { + if !isX86IMULImmediate(immediate, goarch) { + return fmt.Errorf("%s %s immediate is outside Go 1.27's Yi8/Yi32 classes: %q", goarch, op, ins.Raw) + } + if !isX86ScalarMultiplyDivideSource(source, bits, goarch) { + return fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", goarch, op, ins.Raw) + } + return nil + } + if source.Kind == OpImm { + if !isX86IMULImmediate(source, goarch) { + return fmt.Errorf("%s %s immediate is outside Go 1.27's Yi8/Yi32 classes: %q", goarch, op, ins.Raw) + } + return nil + } + if !isX86ScalarMultiplyDivideSource(source, bits, goarch) { + return fmt.Errorf("%s %s source is outside Go 1.27's Yml class: %q", goarch, op, ins.Raw) + } + return nil +} + +func isX86IMULImmediate(operand Operand, goarch string) bool { + if operand.Kind != OpImm { + return false + } + // The 386 assembler classifies constants after truncating them to int32. + // In amd64 mode Yi32 covers signed int32 through unsigned uint32. + return goarch == "386" || (operand.Imm >= -2147483648 && operand.Imm <= 4294967295) +} + +func (c *amd64Ctx) emitX86ImplicitMultiply(bits int, signed bool, source Operand) error { + narrow := amd64IntegerTypeForBits(bits) + wide := x86ScalarMultiplyDivideWideType(bits) + src, err := c.evalIntSized(source, narrow) + if err != nil { + return err + } + acc, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, narrow) + if err != nil { + return err + } + lhs := c.extendX86MulDivValue(acc, narrow, wide, signed) + rhs := c.extendX86MulDivValue(src, narrow, wide, signed) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %s, %s\n", product, wide, lhs, rhs) + + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", low, wide, product, narrow) + if bits == 8 { + if err := c.storeRegSized(AX, I16, "%"+product); err != nil { + return err + } + } else { + shifted := c.newTmp() + shift := "lshr" + if signed { + shift = "ashr" + } + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %d\n", shifted, shift, wide, product, bits) + high := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", high, wide, shifted, narrow) + if err := c.storeRegSized(AX, narrow, "%"+low); err != nil { + return err + } + if err := c.storeRegSized(DX, narrow, "%"+high); err != nil { + return err + } + } + c.setX86MultiplyOverflowFlags(narrow, wide, "%"+low, "%"+product, signed) + return nil +} + +func (c *amd64Ctx) emitX86ExplicitIMUL(bits int, immediate, source, destination Operand) error { + typ := amd64IntegerTypeForBits(bits) + wide := x86ScalarMultiplyDivideWideType(bits) + right := "" + var err error + if source.Kind == OpImm { + right = x86IMULImmediateValue(source.Imm, bits) + } else { + right, err = c.evalIntSized(source, typ) + if err != nil { + return err + } + } + left := "" + if immediate.Kind == OpImm { + left = x86IMULImmediateValue(immediate.Imm, bits) + } else { + left, err = c.evalIntSized(destination, typ) + if err != nil { + return err + } + } + lhs := c.extendX86MulDivValue(left, typ, wide, true) + rhs := c.extendX86MulDivValue(right, typ, wide, true) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %s, %s\n", product, wide, lhs, rhs) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", low, wide, product, typ) + if err := c.storeRegSized(destination.Reg, typ, "%"+low); err != nil { + return err + } + c.setX86MultiplyOverflowFlags(typ, wide, "%"+low, "%"+product, true) + return nil +} + +func x86IMULImmediateValue(value int64, bits int) string { + return fmt.Sprintf("%d", x86IMULEncodedImmediate(value, bits)) +} + +func x86IMULEncodedImmediate(value int64, bits int) int64 { + switch bits { + case 16: + return int64(int16(value)) + case 32: + return int64(int32(value)) + case 64: + // x86-64 IMUL encodes this table's immediate as imm32 and sign + // extends it to the qword operand width. + return int64(int32(value)) + default: + panic("unsupported explicit IMUL width") + } +} + +func (c *amd64Ctx) emitX86ImplicitDivide(bits int, signed bool, source Operand) error { + narrow := amd64IntegerTypeForBits(bits) + wide := x86ScalarMultiplyDivideWideType(bits) + divisorNarrow, err := c.evalIntSized(source, narrow) + if err != nil { + return err + } + divisor := c.extendX86MulDivValue(divisorNarrow, narrow, wide, signed) + + var dividend string + if bits == 8 { + dividend, err = c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I16) + if err != nil { + return err + } + } else { + low, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, narrow) + if err != nil { + return err + } + high, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, narrow) + if err != nil { + return err + } + lowWide := c.extendX86MulDivValue(low, narrow, wide, false) + highWide := c.extendX86MulDivValue(high, narrow, wide, signed) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s %s, %d\n", shifted, wide, highWide, bits) + joined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", joined, wide, shifted, lowWide) + dividend = "%" + joined + } + + divide := "udiv" + remainder := "urem" + if signed { + divide = "sdiv" + remainder = "srem" + } + quotientWide := c.newTmp() + remainderWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", quotientWide, divide, wide, dividend, divisor) + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", remainderWide, remainder, wide, dividend, divisor) + quotient := c.newTmp() + remainderValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", quotient, wide, quotientWide, narrow) + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", remainderValue, wide, remainderWide, narrow) + if bits == 8 { + q16 := c.extendX86MulDivValue("%"+quotient, I8, I16, false) + r16 := c.extendX86MulDivValue("%"+remainderValue, I8, I16, false) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i16 %s, 8\n", shifted, r16) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i16 %%%s, %s\n", combined, shifted, q16) + return c.storeRegSized(AX, I16, "%"+combined) + } + if err := c.storeRegSized(AX, narrow, "%"+quotient); err != nil { + return err + } + return c.storeRegSized(DX, narrow, "%"+remainderValue) +} + +func (c *amd64Ctx) extendX86MulDivValue(value string, from, to LLVMType, signed bool) string { + tmp := c.newTmp() + op := "zext" + if signed { + op = "sext" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", tmp, op, from, value, to) + return "%" + tmp +} + +func x86ScalarMultiplyDivideWideType(bits int) LLVMType { + if bits == 64 { + return LLVMType("i128") + } + return amd64IntegerTypeForBits(bits * 2) +} + +func (c *amd64Ctx) setX86MultiplyOverflowFlags(narrow, wide LLVMType, low, product string, signed bool) { + var overflow string + if signed { + extended := c.extendX86MulDivValue(low, narrow, wide, true) + cmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %s, %s\n", cmp, wide, product, extended) + overflow = "%" + cmp + } else { + extended := c.extendX86MulDivValue(low, narrow, wide, false) + cmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %s, %s\n", cmp, wide, product, extended) + overflow = "%" + cmp + } + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", overflow, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", overflow, c.flagsOFSlot) +} diff --git a/amd64_lower_mulword.go b/amd64_lower_mulword.go new file mode 100644 index 000000000..261a6ab57 --- /dev/null +++ b/amd64_lower_mulword.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedWordMultiplyMode uint8 + +const ( + amd64PackedWordMultiplyLow amd64PackedWordMultiplyMode = iota + amd64PackedWordMultiplySignedHigh + amd64PackedWordMultiplyUnsignedHigh + amd64PackedWordMultiplyRoundedSignedHigh +) + +func (c *amd64Ctx) lowerPackedWordMultiply(op Op, ins Instr) (bool, bool, error) { + form, recognized, err := x86PackedWordMultiplyProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed word multiply m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + lanes := form.byteWidth / 2 + if form.mmx { + // A GP source accepted through ycover encodes an MMX register; it + // must never consume the unrelated GP shadow as its input value. + if first.Kind == OpReg && first.Reg != ins.Args[form.first].Reg { + return true, false, fmt.Errorf("%w: Go GP-spelled packed multiply source names the MMX bank", ErrProbeNeedsContext) + } + var a string + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedWordMultiply(lanes, c.bitcastI64ToIntegerLanes(lanes, 16, a), c.bitcastI64ToIntegerLanes(lanes, 16, b), form.spec.mode) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to i64\n", out, lanes, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, 16, false) + if err != nil { + return true, false, err + } + var mask, value string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.legacy && form.memory >= 0 { + value, err = c.emitPackedBinaryNativeMemory(first, 16, b, strings.ToLower(string(op))) + } else { + // These four EVEX opcodes are E4.nb: masked-off word lanes do not + // read native memory. The full possible envelope still belongs to + // the independent source/caller proof, never to the mask value. + a, loadErr := c.loadX86PackedShiftLanes(first, form.byteWidth, 16, false, mask) + err = loadErr + if err == nil { + value = c.emitPackedWordMultiply(lanes, a, b, form.spec.mode) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, 16, false) + if err != nil { + return true, false, err + } + } + value = amd64ApplyIntegerLaneMask(c, lanes, 16, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x i8>\n", out, lanes, value, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedWordMultiply(lanes int, first, second string, mode amd64PackedWordMultiplyMode) string { + wordType := fmt.Sprintf("<%d x i16>", lanes) + if mode == amd64PackedWordMultiplyLow { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %s, %s\n", result, wordType, second, first) + return "%" + result + } + + wideType := fmt.Sprintf("<%d x i32>", lanes) + extend := "sext" + shift := int64(16) + if mode == amd64PackedWordMultiplyUnsignedHigh { + extend = "zext" + } + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", firstWide, extend, wordType, first, wideType) + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", secondWide, extend, wordType, second, wideType) + product := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul %s %%%s, %%%s\n", product, wideType, secondWide, firstWide) + value := "%" + product + if mode == amd64PackedWordMultiplyRoundedSignedHigh { + rounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", rounded, wideType, value, llvmSplatSignedInteger(lanes, 32, 16384)) + value = "%" + rounded + shift = 15 + } + shifted := c.newTmp() + shiftOp := "ashr" + if mode == amd64PackedWordMultiplyUnsignedHigh { + shiftOp = "lshr" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", shifted, shiftOp, wideType, value, llvmSplatSignedInteger(lanes, 32, shift)) + value = "%" + shifted + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, wideType, value, wordType) + return "%" + result +} diff --git a/amd64_lower_mxcsr.go b/amd64_lower_mxcsr.go new file mode 100644 index 000000000..0fab1b247 --- /dev/null +++ b/amd64_lower_mxcsr.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerMXCSR(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + switch baseOp { + case "LDMXCSR", "VLDMXCSR", "STMXCSR", "VSTMXCSR": + // handled below + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 1 || !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("amd64 %s expects one memory operand: %q", baseOp, ins.Raw) + } + slot := c.newTmp() + fmt.Fprintf(c.b, " %%%s = alloca i32\n", slot) + switch baseOp { + case "LDMXCSR", "VLDMXCSR": + value, err := c.evalIntSized(ins.Args[0], I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " store i32 %s, ptr %%%s\n", value, slot) + fmt.Fprintf(c.b, " call void @llvm.x86.sse.ldmxcsr(ptr %%%s)\n", slot) + return true, false, nil + case "STMXCSR", "VSTMXCSR": + value := c.newTmp() + fmt.Fprintf(c.b, " call void @llvm.x86.sse.stmxcsr(ptr %%%s)\n", slot) + fmt.Fprintf(c.b, " %%%s = load i32, ptr %%%s\n", value, slot) + if err := c.storeMXCSRMemory(ins.Args[0], "%"+value); err != nil { + return true, false, err + } + return true, false, nil + default: + panic("MXCSR opcode precheck drift") + } +} + +func (c *amd64Ctx) storeMXCSRMemory(dst Operand, value string) error { + switch dst.Kind { + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i32 %s, %s %s, align 1\n", value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i32 %s, ptr %s, align 1\n", value, ptr) + return nil + case OpFP: + return c.storeFPResult(dst.FPOffset, I32, value) + default: + return fmt.Errorf("amd64: unsupported MXCSR memory operand %s", dst.String()) + } +} diff --git a/amd64_lower_narrow_single_source.go b/amd64_lower_narrow_single_source.go new file mode 100644 index 000000000..6143bacd7 --- /dev/null +++ b/amd64_lower_narrow_single_source.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SingleSourceNarrowSpec struct { + inputBits int + outputBits int + saturating bool + mode amd64PackedNarrowMode +} + +var amd64SingleSourceNarrowSpecs = map[string]amd64SingleSourceNarrowSpec{ + "VPMOVDB": {inputBits: 32, outputBits: 8}, + "VPMOVDW": {inputBits: 32, outputBits: 16}, + "VPMOVQB": {inputBits: 64, outputBits: 8}, + "VPMOVQD": {inputBits: 64, outputBits: 32}, + "VPMOVQW": {inputBits: 64, outputBits: 16}, + "VPMOVWB": {inputBits: 16, outputBits: 8}, + "VPMOVSDB": {inputBits: 32, outputBits: 8, saturating: true, mode: amd64PackedNarrowSigned}, + "VPMOVSDW": {inputBits: 32, outputBits: 16, saturating: true, mode: amd64PackedNarrowSigned}, + "VPMOVSQB": {inputBits: 64, outputBits: 8, saturating: true, mode: amd64PackedNarrowSigned}, + "VPMOVSQD": {inputBits: 64, outputBits: 32, saturating: true, mode: amd64PackedNarrowSigned}, + "VPMOVSQW": {inputBits: 64, outputBits: 16, saturating: true, mode: amd64PackedNarrowSigned}, + "VPMOVSWB": {inputBits: 16, outputBits: 8, saturating: true, mode: amd64PackedNarrowSigned}, + "VPMOVUSDB": {inputBits: 32, outputBits: 8, saturating: true, mode: amd64PackedNarrowUnsigned}, + "VPMOVUSDW": {inputBits: 32, outputBits: 16, saturating: true, mode: amd64PackedNarrowUnsigned}, + "VPMOVUSQB": {inputBits: 64, outputBits: 8, saturating: true, mode: amd64PackedNarrowUnsigned}, + "VPMOVUSQD": {inputBits: 64, outputBits: 32, saturating: true, mode: amd64PackedNarrowUnsigned}, + "VPMOVUSQW": {inputBits: 64, outputBits: 16, saturating: true, mode: amd64PackedNarrowUnsigned}, + "VPMOVUSWB": {inputBits: 16, outputBits: 8, saturating: true, mode: amd64PackedNarrowUnsigned}, +} + +// lowerSingleSourceIntegerNarrow implements the complete Go 1.27 +// _yvpmovdb/_yvpmovdw family: truncating, signed-saturating, and +// signed-to-unsigned-saturating conversions from X/Y/Z into X/Y or memory. +func (c *amd64Ctx) lowerSingleSourceIntegerNarrow(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, handled := amd64SingleSourceNarrowSpecs[baseOp] + if !handled { + return false, false, nil + } + if suffix != "" && suffix != "Z" { + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvpmovdb/_yvpmovdw tables: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects vector source, [K1-K7 mask,] vector or memory destination: %q", c.goarch, baseOp, ins.Raw) + } + + source := ins.Args[0] + inputBytes := amd64VectorByteWidth(source.Reg) + if source.Kind != OpReg || inputBytes != 16 && inputBytes != 32 && inputBytes != 64 || !c.isGoEVEXVectorRegister(source, inputBytes) { + return true, false, fmt.Errorf("%s %s source must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if suffix == "Z" && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + var mask string + if masked { + maskArg := ins.Args[1] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + index, valid := amd64ParseKReg(maskArg.Reg) + if !valid || index == 0 { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + destination := ins.Args[len(ins.Args)-1] + lanes := inputBytes * 8 / spec.inputBits + outputBytes := lanes * spec.outputBits / 8 + destinationBytes := 16 + if outputBytes > 16 { + destinationBytes = 32 + } + if destination.Kind == OpReg { + if amd64VectorByteWidth(destination.Reg) != destinationBytes || !c.isGoEVEXVectorRegister(destination, destinationBytes) { + return true, false, fmt.Errorf("%s %s destination must be an in-range %s register for a %s source: %q", c.goarch, baseOp, map[int]string{16: "X", 32: "Y"}[destinationBytes], strings.ToUpper(string(source.Reg))[0:1], ins.Raw) + } + } else if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s %s destination must be the matching X/Y register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + sourceBytes, err := c.loadPackedCompareBytes(source, inputBytes) + if err != nil { + return true, false, err + } + sourceLanes := c.bitcastVectorBytesToIntegerLanes(inputBytes, lanes, spec.inputBits, sourceBytes) + narrowed := c.emitSingleSourceIntegerNarrow(spec, lanes, sourceLanes) + zeroing := suffix == "Z" + if destination.Kind == OpReg { + return true, false, c.storeSingleSourceNarrowRegister(destination.Reg, destinationBytes, lanes, spec.outputBits, narrowed, mask, zeroing) + } + return true, false, c.storeSingleSourceNarrowMemory(destination, outputBytes, lanes, spec.outputBits, narrowed, mask, zeroing) +} + +func (c *amd64Ctx) emitSingleSourceIntegerNarrow(spec amd64SingleSourceNarrowSpec, lanes int, source string) string { + inputType := fmt.Sprintf("<%d x i%d>", lanes, spec.inputBits) + outputType := fmt.Sprintf("<%d x i%d>", lanes, spec.outputBits) + if !spec.saturating { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, inputType, source, outputType) + return "%" + result + } + result := "poison" + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", value, inputType, source, lane) + narrowed := c.saturatePackedInteger("%"+value, spec.inputBits, spec.outputBits, spec.mode) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, outputType, result, spec.outputBits, narrowed, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) storeSingleSourceNarrowRegister(destination Reg, destinationBytes, lanes, outputBits int, narrowed, mask string, zeroing bool) error { + physicalLanes := destinationBytes * 8 / outputBits + result := "zeroinitializer" + var old string + if mask != "" && !zeroing { + oldBytes, err := c.loadPackedCompareBytes(Operand{Kind: OpReg, Reg: destination}, destinationBytes) + if err != nil { + return err + } + old = c.bitcastVectorBytesToIntegerLanes(destinationBytes, physicalLanes, outputBits, oldBytes) + } + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, lanes, outputBits, narrowed, lane) + selected := "%" + value + if mask != "" { + fallback := "0" + if !zeroing { + oldValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", oldValue, physicalLanes, outputBits, old, lane) + fallback = "%" + oldValue + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %s, i%d %s\n", masked, amd64MaskBitI1(c, mask, lane), outputBits, selected, outputBits, fallback) + selected = "%" + masked + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %s, i32 %d\n", inserted, physicalLanes, outputBits, result, outputBits, selected, lane) + result = "%" + inserted + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, physicalLanes, outputBits, result, destinationBytes) + return c.storeVectorBytes(destination, destinationBytes, "%"+out) +} + +func (c *amd64Ctx) storeSingleSourceNarrowMemory(destination Operand, outputBytes, lanes, outputBits int, narrowed, mask string, zeroing bool) error { + result := narrowed + if mask != "" { + var old string + if !zeroing { + oldBytes, err := c.loadNarrowMemoryBytes(destination, outputBytes) + if err != nil { + return err + } + old = c.bitcastVectorBytesToIntegerLanes(outputBytes, lanes, outputBits, oldBytes) + } + result = "poison" + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, lanes, outputBits, narrowed, lane) + fallback := "0" + if !zeroing { + oldValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", oldValue, lanes, outputBits, old, lane) + fallback = "%" + oldValue + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %s\n", selected, amd64MaskBitI1(c, mask, lane), outputBits, value, outputBits, fallback) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, outputBits, result, outputBits, selected, lane) + result = "%" + inserted + } + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, outputBits, result, outputBytes) + return c.storeNarrowMemoryBytes(destination, outputBytes, "%"+out) +} + +func (c *amd64Ctx) loadNarrowMemoryBytes(source Operand, byteWidth int) (string, error) { + if byteWidth >= 16 { + return c.loadPackedCompareBytes(source, byteWidth) + } + typ := amd64IntegerTypeForBits(byteWidth * 8) + value, err := c.evalIntSized(source, typ) + if err != nil { + return "", err + } + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytesValue, typ, value, byteWidth) + return "%" + bytesValue, nil +} + +func (c *amd64Ctx) storeNarrowMemoryBytes(destination Operand, byteWidth int, value string) error { + if byteWidth >= 16 { + return c.storeVectorBytesOperand(destination, byteWidth, value) + } + typ := amd64IntegerTypeForBits(byteWidth * 8) + integer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", integer, byteWidth, value, typ) + return c.storeVectorScalarMemory(destination, byteWidth*8, "%"+integer, "") +} diff --git a/amd64_lower_narrow_single_source_test.go b/amd64_lower_narrow_single_source_test.go new file mode 100644 index 000000000..73fb07a96 --- /dev/null +++ b/amd64_lower_narrow_single_source_test.go @@ -0,0 +1,127 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86SingleSourceIntegerNarrowCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 31 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT singlesourcenarrowforms(SB),$0-0\n") + for _, op := range []string{ + "VPMOVDB", "VPMOVDW", "VPMOVQB", "VPMOVQD", "VPMOVQW", "VPMOVWB", + "VPMOVSDB", "VPMOVSDW", "VPMOVSQB", "VPMOVSQD", "VPMOVSQW", "VPMOVSWB", + "VPMOVUSDB", "VPMOVUSDW", "VPMOVUSQB", "VPMOVUSQD", "VPMOVUSQW", "VPMOVUSWB", + } { + inputBits, outputBits := x86SingleSourceNarrowTestBits(op) + for _, width := range []struct { + name string + bytes int + }{ + {name: "X", bytes: 16}, + {name: "Y", bytes: 32}, + {name: "Z", bytes: 64}, + } { + outputBytes := width.bytes * outputBits / inputBits + destinationWidth := "X" + if outputBytes > 16 { + destinationWidth = "Y" + } + fmt.Fprintf(&source, "\t%s %s%d, %s%d\n", op, width.name, last, destinationWidth, last) + fmt.Fprintf(&source, "\t%s %s%d, K1, %s%d\n", op, width.name, last, destinationWidth, last) + fmt.Fprintf(&source, "\t%s.Z %s%d, K2, %s%d\n", op, width.name, last, destinationWidth, last) + fmt.Fprintf(&source, "\t%s %s%d, 8(AX)\n", op, width.name, last) + fmt.Fprintf(&source, "\t%s %s%d, K3, 16(AX)\n", op, width.name, last) + fmt.Fprintf(&source, "\t%s.Z %s%d, K4, 24(AX)\n", op, width.name, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "singlesourcenarrowforms": {Name: "singlesourcenarrowforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "single-source-integer-narrow.ll", "single-source-integer-narrow.o", ll) + }) + } +} + +func x86SingleSourceNarrowTestBits(op string) (inputBits, outputBits int) { + pair := strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(op, "VPMOV"), "US"), "S") + bits := func(letter byte) int { + switch letter { + case 'B': + return 8 + case 'W': + return 16 + case 'D': + return 32 + case 'Q': + return 64 + default: + panic("bad narrow element letter") + } + } + return bits(pair[0]), bits(pair[1]) +} + +func TestTranslateX86SingleSourceIntegerNarrowRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPMOVSDB 8(AX), X0", + "VPMOVSDB X0, Y0", + "VPMOVSDW Z0, X0", + "VPMOVSDB X0, K0, X1", + "VPMOVSDB.Z X0, X1", + "VPMOVSDB.BCST X0, X1", + "VPMOVSDB X0, K1, K2, X1", + "VPMOVSDB X0, AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvpmovdb/_yvpmovdw tables", instruction) + } + }) + } +} diff --git a/amd64_lower_neg.go b/amd64_lower_neg.go new file mode 100644 index 000000000..3870aeff5 --- /dev/null +++ b/amd64_lower_neg.go @@ -0,0 +1,63 @@ +package plan9asm + +import "fmt" + +// Both complete unary families use Go's yscond(Ymb) grammar, including +// byte-register spellings of wide operations. Only NEG changes flags. +type x86UnaryYmbSpec struct { + bits int + negate bool +} + +var x86UnaryYmbSpecs = map[Op]x86UnaryYmbSpec{ + "NEGB": {bits: 8, negate: true}, + "NEGW": {bits: 16, negate: true}, + "NEGL": {bits: 32, negate: true}, + "NEGQ": {bits: 64, negate: true}, + "NOTB": {bits: 8}, + "NOTW": {bits: 16}, + "NOTL": {bits: 32}, + "NOTQ": {bits: 64}, +} + +func (c *amd64Ctx) lowerScalarUnaryYmb(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, recognized := x86UnaryYmbSpecs[op] + if !recognized { + return false, false, nil + } + bits := spec.bits + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", op, ins.Raw) + } + if len(ins.Args) != 1 { + return true, false, fmt.Errorf("%s %s expects one register/memory operand: %q", c.goarch, op, ins.Raw) + } + destination := ins.Args[0] + if destination.Kind == OpReg { + if !isGoYmbRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s register is outside Go 1.27's Ymb class: %q", c.goarch, op, ins.Raw) + } + destination.Reg = amd64YmbEffectiveRegister(destination.Reg, bits) + } else if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s %s operand is outside Go 1.27's Ymb class: %q", c.goarch, op, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + value, store, err := c.loadIntDestination(destination, typ) + if err != nil { + return true, false, err + } + result := c.newTmp() + if !spec.negate { + fmt.Fprintf(c.b, " %%%s = xor %s %s, -1\n", result, typ, value) + return true, false, store("%" + result) + } + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", result, typ, value) + if err := store("%" + result); err != nil { + return true, false, err + } + // NEG has the same arithmetic flags as subtraction from zero: CF is set + // exactly for nonzero inputs and OF is set only for the signed minimum. + c.setScalarAddSubFlags(typ, false, "0", value, "%"+result) + return true, false, nil +} diff --git a/amd64_lower_neg_test.go b/amd64_lower_neg_test.go new file mode 100644 index 000000000..7ab2773ff --- /dev/null +++ b/amd64_lower_neg_test.go @@ -0,0 +1,294 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86UnaryYmbGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`\{A((?:NEG|NOT)[BWLQ]),\s*yscond,`).FindAllStringSubmatch(string(data), -1) + if len(rows) != len(x86UnaryYmbSpecs) || len(rows) != 8 { + t.Fatalf("Go encoder has %d NEG/NOT Ymb rows; grammar has %d", len(rows), len(x86UnaryYmbSpecs)) + } + for _, row := range rows { + name := row[1] + spec, ok := x86UnaryYmbSpecs[Op(name)] + width := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[name[len(name)-1]] + if !ok || spec.bits != width || spec.negate != strings.HasPrefix(name, "NEG") { + t.Errorf("Go %s encoder row disagrees with grammar: %+v, present=%v", name, spec, ok) + } + } +} + +func x86NEGCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "negdata" + size := 8 + if goarch == "386" { + symbol = "negdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $1\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT negforms(SB),$0-0\n") + if goarch == "amd64" { + source.WriteString("\tNEGB AH\n\tNEGB R11\n") + } else { + source.WriteString("\tNEGB AH\n\tNEGB BP\n") + } + source.WriteString("\tNEGB 8(BX)\n\tNEGB " + symbol + "(SB)\n") + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + // yscond uses Ymb even for wide opcodes. Byte spellings therefore + // select the full register with the same raw ModRM number. + for _, reg := range []string{"AL", "CL", "DL", "BL", "AH", "CH", "DH", "BH"} { + fmt.Fprintf(&source, "\tNEG%s %s\n", width, reg) + } + fmt.Fprintf(&source, "\tNEG%s DX\n", width) + if goarch == "amd64" { + for _, reg := range []string{"SP", "BPB", "SIB", "DIB", "R8", "R15", "R8B", "R15B"} { + fmt.Fprintf(&source, "\tNEG%s %s\n", width, reg) + } + } else { + fmt.Fprintf(&source, "\tNEG%s SI\n", width) + } + fmt.Fprintf(&source, "\tNEG%s 8(BX)\n\tNEG%s %s(SB)\n", width, width, symbol) + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86NEGCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86NEGCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"negforms": {Name: "negforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"sub i8 0", "sub i16 0", "sub i32 0", "store i1"} { + if !strings.Contains(ll, want) { + t.Fatalf("NEG lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "sub i64 0") { + t.Fatalf("NEGQ lowering omitted sub i64:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "neg-"+target.name+".ll", "neg-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86NEGRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "NEGB X0"}, + {goarch: "amd64", instruction: "NEGW $1"}, + {goarch: "amd64", instruction: "NEGL AX, BX"}, + {goarch: "amd64", instruction: "NEGQ"}, + {goarch: "amd64", instruction: "NEGQ.Z AX"}, + {goarch: "386", instruction: "NEGB SP"}, + {goarch: "386", instruction: "NEGW SP"}, + {goarch: "386", instruction: "NEGL SP"}, + {goarch: "386", instruction: "NEGL R8"}, + {goarch: "386", instruction: "NEGQ AX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's NEG yscond table", test.instruction) + } + }) + } +} + +func TestAMD64NEGRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT negsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x1122, AX + NEGB AH + MOVQ AX, 0(DI) + MOVQ $0x1122334455660001, AX + NEGW AX + MOVQ AX, 8(DI) + MOVQ $0x1122334480000000, AX + NEGL AX + MOVQ AX, 16(DI) + SETCS 32(DI) + SETOS 33(DI) + SETEQ 34(DI) + SETMI 35(DI) + SETPS 36(DI) + MOVQ $5, 24(DI) + NEGQ 24(DI) + MOVB $0, 40(DI) + NEGB 40(DI) + SETCS 41(DI) + SETOS 42(DI) + SETEQ 43(DI) + SETMI 44(DI) + SETPS 45(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "negsemantics": { + Name: "negsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void negsemantics(uint8_t *); +int main(void) { + uint8_t out[48] = {0}; + negsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0xef22)) return 10; + if (*(uint64_t *)(out+8) != UINT64_C(0x112233445566ffff)) return 11; + if (*(uint64_t *)(out+16) != UINT64_C(0x80000000)) return 12; + if (*(uint64_t *)(out+24) != UINT64_C(0xfffffffffffffffb)) return 13; + if (out[32] != 1 || out[33] != 1 || out[34] != 0 || out[35] != 1 || out[36] != 1) return 14; + if (out[40] != 0 || out[41] != 0 || out[42] != 0 || out[43] != 1 || out[44] != 0 || out[45] != 1) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "neg_semantics", triple, ll, mainC, runPrefix) +} + +func TestAMD64UnaryYmbAliasesRuntime(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, checks strings.Builder + source.WriteString("TEXT unaryaliases(SB),$0-8\n\tMOVQ out+0(FP), R14\n") + index := 0 + for _, family := range []string{"NEG", "NOT"} { + for _, width := range []struct { + suffix string + bits uint + }{{"W", 16}, {"L", 32}, {"Q", 64}} { + for _, reg := range []struct{ spelling, effective string }{ + {"AL", "AX"}, {"CL", "CX"}, {"DL", "DX"}, {"BL", "BX"}, + {"AH", "SP"}, {"CH", "BP"}, {"DH", "SI"}, {"BH", "DI"}, + {"BPB", "BP"}, {"SIB", "SI"}, {"DIB", "DI"}, {"R8B", "R8"}, {"R15B", "R15"}, + } { + const input uint64 = 0x1122334455667701 + want := ^input + if family == "NEG" { + want++ + } + if width.bits == 16 { + want = (input &^ 0xffff) | (want & 0xffff) + } + if width.bits == 32 { + want &= 0xffffffff + } + fmt.Fprintf(&source, "\tMOVQ $%#x, %s\n\t%s%s %s\n\tMOVQ %s, %d(R14)\n", input, reg.effective, family, width.suffix, reg.spelling, reg.effective, index*8) + fmt.Fprintf(&checks, " if (out[%d] != UINT64_C(%#x)) return %d;\n", index, want, index+1) + index++ + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple, runPrefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{ + "unaryaliases": {Name: "unaryaliases", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}}, + }}) + if err != nil { + t.Fatal(err) + } + mainC := fmt.Sprintf("#include \nextern void unaryaliases(uint64_t *);\nint main(void) {\n uint64_t out[%d] = {0};\n unaryaliases(out);\n%s return 0;\n}\n", index, checks.String()) + compileAndRunRuntimeTestForTarget(t, llc, clang, "unary_aliases", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_non_temporal_move.go b/amd64_lower_non_temporal_move.go new file mode 100644 index 000000000..59320c099 --- /dev/null +++ b/amd64_lower_non_temporal_move.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerNonTemporalVectorMove implements Go 1.27's complete vector +// non-temporal move family. The VEX/EVEX store table is shared by VMOVNTDQ, +// VMOVNTPD, and VMOVNTPS and accepts X/Y/Z register-to-memory forms. Its +// VMOVNTDQA counterpart accepts the inverse memory-to-X/Y/Z forms. The legacy +// instructions are the same directions at X width; Go spells MOVNTDQ as +// MOVNTO so that VMOVNTDQ remains available for the VEX/EVEX instruction. +func (c *amd64Ctx) lowerNonTemporalVectorMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + vectorStore, vectorLoad := false, false + legacyStore := false + switch baseOp { + case "VMOVNTDQ", "VMOVNTPD", "VMOVNTPS": + vectorStore = true + case "VMOVNTDQA": + vectorLoad = true + case "MOVNTO", "MOVNTPD", "MOVNTPS": + legacyStore = true + case "MOVNTDQA": + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed forms in Go 1.27's non-temporal vector-move tables: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects exactly two operands: %q", c.goarch, baseOp, ins.Raw) + } + + if vectorStore { + source, destination := ins.Args[0], ins.Args[1] + byteWidth := 0 + if source.Kind == OpReg { + byteWidth = amd64VectorByteWidth(source.Reg) + } + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s %s destination must be memory: %q", c.goarch, baseOp, ins.Raw) + } + value, loadErr := c.loadPackedCompareBytes(source, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + return true, false, c.storeVectorBytesOperandWithMetadata(destination, byteWidth, value, x86NonTemporalMetadata) + } + + if vectorLoad { + source, destination := ins.Args[0], ins.Args[1] + if !isAMD64MemoryOperand(source) || destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects a memory source and X, Y, or Z destination: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + value, loadErr := c.loadPackedCompareBytes(source, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + return true, false, c.storeVectorBytesOperand(destination, byteWidth, value) + } + + if legacyStore { + source, destination := ins.Args[0], ins.Args[1] + if !c.isGoLegacyNonTemporalVectorRegister(source) || !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s %s expects an in-range X source and memory destination: %q", c.goarch, baseOp, ins.Raw) + } + value, loadErr := c.loadPackedCompareBytes(source, 16) + if loadErr != nil { + return true, false, loadErr + } + return true, false, c.storeVectorBytesOperandWithMetadata(destination, 16, value, x86NonTemporalMetadata) + } + + source, destination := ins.Args[0], ins.Args[1] + if !isAMD64MemoryOperand(source) || !c.isGoLegacyNonTemporalVectorRegister(destination) { + return true, false, fmt.Errorf("%s %s expects a memory source and in-range X destination: %q", c.goarch, baseOp, ins.Raw) + } + value, loadErr := c.loadPackedCompareBytes(source, 16) + if loadErr != nil { + return true, false, loadErr + } + return true, false, c.storeVectorBytesOperand(destination, 16, value) +} + +func (c *amd64Ctx) isGoLegacyNonTemporalVectorRegister(operand Operand) bool { + if !amd64VEXVectorRegister(operand, 16) { + return false + } + index, _ := amd64ParseXReg(operand.Reg) + return c.goarch != "386" || index < 8 +} diff --git a/amd64_lower_non_temporal_move_test.go b/amd64_lower_non_temporal_move_test.go new file mode 100644 index 000000000..40b254f14 --- /dev/null +++ b/amd64_lower_non_temporal_move_test.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86NonTemporalVectorMoveCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses _yvmovntdq for VMOVNTDQ/PD/PS register-to-memory + // stores and _yvmovntdqa for the inverse aligned memory loads. Both + // tables cover VEX/EVEX X, Y, and Z widths. The legacy table covers the + // corresponding X-only forms (with MOVNTO as Go's MOVNTDQ spelling). + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 31 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT nontemporalmoveforms(SB),$0-0\n") + for index, op := range []string{"VMOVNTDQ", "VMOVNTPD", "VMOVNTPS"} { + fmt.Fprintf(&source, "\t%s X%d, %d(AX)\n", op, 20+index, index*256) + fmt.Fprintf(&source, "\t%s Y%d, %d(AX)\n", op, 20+index, index*256+64) + fmt.Fprintf(&source, "\t%s Z%d, %d(AX)\n", op, lastZ, index*256+128) + } + source.WriteString("\tVMOVNTDQA 1024(AX), X20\n") + source.WriteString("\tVMOVNTDQA 1088(AX), Y21\n") + fmt.Fprintf(&source, "\tVMOVNTDQA 1152(AX), Z%d\n", lastZ) + source.WriteString("\tMOVNTO X1, 1216(AX)\n") + source.WriteString("\tMOVNTPD X2, 1280(AX)\n") + source.WriteString("\tMOVNTPS X3, 1344(AX)\n") + source.WriteString("\tMOVNTDQA 1408(AX), X4\n") + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "nontemporalmoveforms": {Name: "nontemporalmoveforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, vectorType := range []string{"<16 x i8>", "<32 x i8>", "<64 x i8>"} { + if !strings.Contains(ir, "load "+vectorType) || !strings.Contains(ir, "store "+vectorType) { + t.Fatalf("translation omitted %s load/store forms:\n%s", vectorType, ir) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "non-temporal-move-"+target.name+".ll", "non-temporal-move-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86NonTemporalVectorMoveRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VMOVNTDQ 8(AX), X0"}, + {goarch: "amd64", instruction: "VMOVNTPD X0, X1"}, + {goarch: "amd64", instruction: "VMOVNTPS X0, K1, 8(AX)"}, + {goarch: "amd64", instruction: "VMOVNTDQ.Z Z0, 8(AX)"}, + {goarch: "amd64", instruction: "VMOVNTDQA X0, X1"}, + {goarch: "amd64", instruction: "VMOVNTDQA 8(AX), K1, X0"}, + {goarch: "amd64", instruction: "MOVNTPD 8(AX), X0"}, + {goarch: "amd64", instruction: "MOVNTPS X0, X1"}, + {goarch: "amd64", instruction: "MOVNTDQA X0, 8(AX)"}, + {goarch: "386", instruction: "VMOVNTDQ Z8, 8(AX)"}, + {goarch: "386", instruction: "VMOVNTDQA 8(AX), Z8"}, + {goarch: "386", instruction: "MOVNTO X8, 8(AX)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's non-temporal vector-move tables for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestX86RawDecoderRecoversVEXNonTemporalVectorMoveWidths(t *testing.T) { + for _, test := range []struct { + name string + code []byte + op Op + from string + to string + }{ + {name: "vmovntdq x store", code: []byte{0xc5, 0xf9, 0xe7, 0x03}, op: "VMOVNTDQ", from: "X0", to: "0(BX)"}, + {name: "vmovntdq y store", code: []byte{0xc5, 0xfd, 0xe7, 0x03}, op: "VMOVNTDQ", from: "Y0", to: "0(BX)"}, + {name: "vmovntdq y sib store", code: []byte{0xc4, 0xa1, 0x7d, 0xe7, 0x04, 0x03}, op: "VMOVNTDQ", from: "Y0", to: "0(BX)(R8*1)"}, + {name: "vmovntpd x store", code: []byte{0xc5, 0xf9, 0x2b, 0x03}, op: "VMOVNTPD", from: "X0", to: "0(BX)"}, + {name: "vmovntpd y store", code: []byte{0xc5, 0xfd, 0x2b, 0x03}, op: "VMOVNTPD", from: "Y0", to: "0(BX)"}, + {name: "vmovntps x store", code: []byte{0xc5, 0xf8, 0x2b, 0x03}, op: "VMOVNTPS", from: "X0", to: "0(BX)"}, + {name: "vmovntps y store", code: []byte{0xc5, 0xfc, 0x2b, 0x03}, op: "VMOVNTPS", from: "Y0", to: "0(BX)"}, + {name: "vmovntdqa x load", code: []byte{0xc4, 0xe2, 0x79, 0x2a, 0x03}, op: "VMOVNTDQA", from: "0(BX)", to: "X0"}, + {name: "vmovntdqa y load", code: []byte{0xc4, 0xe2, 0x7d, 0x2a, 0x03}, op: "VMOVNTDQA", from: "0(BX)", to: "Y0"}, + } { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.op || len(got.Instrs[0].Args) != 2 || got.Instrs[0].Args[0].String() != test.from || got.Instrs[0].Args[1].String() != test.to { + t.Fatalf("decoded %#x as %#v, want %s %s, %s", test.code, got.Instrs, test.op, test.from, test.to) + } + }) + } +} diff --git a/amd64_lower_nop_width_test.go b/amd64_lower_nop_width_test.go new file mode 100644 index 000000000..1d0866518 --- /dev/null +++ b/amd64_lower_nop_width_test.go @@ -0,0 +1,160 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86SizedNOPCompleteFormsSource(goarch string) string { + var source strings.Builder + source.WriteString("DATA nopdata+0(SB)/8, $1\n") + source.WriteString("GLOBL nopdata(SB), $8\n") + source.WriteString("TEXT sizednopforms(SB),$0-0\n") + for _, op := range []string{"NOPW", "NOPL"} { + fmt.Fprintf(&source, "\t%s AX\n", op) + fmt.Fprintf(&source, "\t%s SP\n", op) + fmt.Fprintf(&source, "\t%s 8(BX)\n", op) + fmt.Fprintf(&source, "\t%s nopdata(SB)\n", op) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s R11\n", op) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86SizedNOPCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86SizedNOPCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"sizednopforms": {Name: "sizednopforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "sized-nop-"+target.name+".ll", "sized-nop-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SizedNOPRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "NOPW"}, + {goarch: "amd64", instruction: "NOPL $1"}, + {goarch: "amd64", instruction: "NOPW X0"}, + {goarch: "amd64", instruction: "NOPL AX, BX"}, + {goarch: "amd64", instruction: "NOPL.Z AX"}, + {goarch: "386", instruction: "NOPW R8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's NOPW/NOPL ydivl table", test.instruction) + } + }) + } +} + +func TestAMD64SizedNOPRuntimePreservesState(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT sizednopsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x123456789abcdef, AX + MOVL $0x7fffffff, R8 + ADDL $1, R8 + STC + NOPW AX + NOPL 4096(DI) + MOVQ AX, 0(DI) + SETCS 8(DI) + SETOS 9(DI) + SETEQ 10(DI) + SETMI 11(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "sizednopsemantics": { + Name: "sizednopsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void sizednopsemantics(uint8_t *); +int main(void) { + uint8_t out[16] = {0}; + sizednopsemantics(out); + uint64_t value; + memcpy(&value, out, sizeof(value)); + if (value != UINT64_C(0x123456789abcdef)) return 10; + if (out[8] != 1 || out[9] != 1 || out[10] != 0 || out[11] != 1) return 11; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "sized_nop_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_numerical_read.go b/amd64_lower_numerical_read.go new file mode 100644 index 000000000..6478599d5 --- /dev/null +++ b/amd64_lower_numerical_read.go @@ -0,0 +1,70 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerNumericalRead(ins Instr) (bool, error) { + if ins.x86NumericalRead == nil { + return false, nil + } + read, form, valid := numericalCertifiedRead(ins, c.goarch) + cert := ins.x86NumericalRead + if !valid || c.sig.Name != read.Function || c.targetTriple != cert.binding.options.TargetTriple { + return true, numericalSBContext("emitted function/profile/read certificate changed") + } + var comparison x86ScalarCompareForm + if form.spec.compare { + var known bool + var err error + comparison, known, err = x86ScalarCompareProperties(ins.Op, ins, c.goarch) + if !known || err != nil { + return true, numericalSBContext("certified comparison lost its complete operand form: %v", err) + } + } + if c.fixedLocal == nil { + if err := c.scalarFPResultInitialized(); err != nil { + return true, err + } + } else if c.fixedLocal.binding == nil || c.fixedLocal.binding != c.sig.x86FixedLocal || + c.fixedLocal.binding.numericalReads == nil || c.fixedLocal.binding.numericalReads.source != cert.binding.source { + return true, numericalSBContext("numerical read lost its independent complete fixed-local proof") + } + // No ptrFromSB call: only this exact typed read can consume the immutable + // native source offset. The global remains externally mutable, and every + // read is volatile even when its result/condition is later overwritten. + pointer := llvmGlobal(read.Variable.NativeName) + if read.Offset != 0 { + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, ptr %s, i64 %d\n", address, pointer, read.Offset) + pointer = "%" + address + } + sourceType := amd64IntegerTypeForBits(int(read.Width * 8)) + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile %s, ptr %s, align 1\n", loaded, sourceType, pointer) + value := "%" + loaded + if form.spec.compare { + // Certification already checked this shared complete operand grammar. + // Preserve Go's immediate truncation/sign extension and byte views. + partner := comparison.args[form.partner] + other, err := c.evalIntSized(partner, sourceType) + if err != nil { + return true, err + } + if form.operand == 0 { + c.setCmpFlagsSized(sourceType, value, other) + } else { + c.setCmpFlagsSized(sourceType, other, value) + } + return true, nil + } + destinationType := amd64IntegerTypeForBits(int(form.spec.gpBytes * 8)) + if read.Width < form.spec.gpBytes { + extension := "zext" + if amd64ScalarExtensionMoveSpecs[ins.Op].signed { + extension = "sext" + } + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", extended, extension, sourceType, value, destinationType) + value = "%" + extended + } + return true, c.storeRegSized(ins.Args[form.partner].Reg, destinationType, value) +} diff --git a/amd64_lower_pack.go b/amd64_lower_pack.go new file mode 100644 index 000000000..76ab79f5e --- /dev/null +++ b/amd64_lower_pack.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedNarrowMode uint8 + +const ( + amd64PackedNarrowSigned amd64PackedNarrowMode = iota + amd64PackedNarrowUnsigned +) + +// All legacy/VEX/EVEX forms share one typed grammar. Packing effects remain +// separate: clamp signed inputs, then concatenate each 128-bit source pair. +func (c *amd64Ctx) lowerPackedSaturatingNarrow(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedNarrowSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("packed narrow dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86PackedNarrowProperties(ins, c.goarch) + if err != nil || !known { + return known, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.mmx && first.Kind == OpReg && first.Reg != form.source.Reg { + return true, false, fmt.Errorf("%w: packed narrow GP spelling needs its physical MMX transport", ErrProbeNeedsContext) + } + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy pack m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + inputBits, outputBits := form.spec.inputBits, form.spec.outputBits + inputLanes := form.byteWidth * 8 / inputBits + if form.mmx { + var firstBits string + if form.memory >= 0 { + firstBits, err = c.loadPackedUnpackMMXSource(first, form.memoryBytes) + } else { + firstBits, err = c.loadReg(first.Reg) + } + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedSaturatingNarrow(form.byteWidth, inputBits, outputBits, form.spec.mode, + c.bitcastI64ToIntegerLanes(inputLanes, inputBits, firstBits), + c.bitcastI64ToIntegerLanes(inputLanes, inputBits, secondBits)) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, 64/outputBits, outputBits, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + secondValue, err := c.loadPackedCompareLanes(second, form.byteWidth, inputBits, false) + if err != nil { + return true, false, err + } + outputLanes := form.byteWidth * 8 / outputBits + var value string + if form.legacy && form.memory >= 0 { + mnemonic := strings.ToLower(base) + if base == "PACKSSLW" { + mnemonic = "packssdw" + } + // Preserve legacy native alignment/fault behavior. The native result + // is already packed: reinterpret it, do not narrow it a second time. + packed, loadErr := c.emitPackedBinaryNativeMemory(first, inputBits, secondValue, mnemonic) + err = loadErr + if err == nil { + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i%d>\n", out, inputLanes, inputBits, packed, outputLanes, outputBits) + value = "%" + out + } + } else { + // Intel's E4NF/E4NF.nb pack reads the entire tuple even with a zero + // output mask or dead result. Never pass that mask to an E4 reader. + firstValue, loadErr := c.loadX86PackedShiftLanes(first, form.byteWidth, inputBits, form.broadcast, "") + err = loadErr + if err == nil { + value = c.emitPackedSaturatingNarrow(form.byteWidth, inputBits, outputBits, form.spec.mode, firstValue, secondValue) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, outputBits, false) + if err != nil { + return true, false, err + } + } + value = amd64ApplyIntegerLaneMask(c, outputLanes, outputBits, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, outputLanes, outputBits, value, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedSaturatingNarrow(byteWidth, inputBits, outputBits int, mode amd64PackedNarrowMode, first, second string) string { + inputLanes := byteWidth * 8 / inputBits + outputLanes := byteWidth * 8 / outputBits + chunkBits := 128 + if byteWidth < 16 { + chunkBits = byteWidth * 8 + } + inputLanesPerChunk := chunkBits / inputBits + outputLanesPerChunk := chunkBits / outputBits + chunks := byteWidth * 8 / chunkBits + result := "zeroinitializer" + for chunk := 0; chunk < chunks; chunk++ { + for sourceHalf, source := range []string{second, first} { + for lane := 0; lane < inputLanesPerChunk; lane++ { + inputIndex := chunk*inputLanesPerChunk + lane + outputIndex := chunk*outputLanesPerChunk + sourceHalf*inputLanesPerChunk + lane + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, inputLanes, inputBits, source, inputIndex) + narrowed := c.saturatePackedInteger("%"+value, inputBits, outputBits, mode) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %s, i32 %d\n", inserted, outputLanes, outputBits, result, outputBits, narrowed, outputIndex) + result = "%" + inserted + } + } + } + return result +} + +func (c *amd64Ctx) saturatePackedInteger(value string, inputBits, outputBits int, mode amd64PackedNarrowMode) string { + minValue := -(int64(1) << (outputBits - 1)) + maxValue := (int64(1) << (outputBits - 1)) - 1 + clamped := value + if mode == amd64PackedNarrowUnsigned { + minValue = 0 + maxValue = (int64(1) << outputBits) - 1 + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %s, 0\n", below, inputBits, clamped) + nonnegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %s\n", nonnegative, below, inputBits, inputBits, clamped) + clamped = "%" + nonnegative + } + above := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt i%d %s, %d\n", above, inputBits, clamped, maxValue) + capped := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %s\n", capped, above, inputBits, maxValue, inputBits, clamped) + clamped = "%" + capped + if mode == amd64PackedNarrowSigned { + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i%d %s, %d\n", below, inputBits, clamped, minValue) + floored := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d %s\n", floored, below, inputBits, minValue, inputBits, clamped) + clamped = "%" + floored + } + narrowed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %s to i%d\n", narrowed, inputBits, clamped, outputBits) + return "%" + narrowed +} diff --git a/amd64_lower_packed_logical.go b/amd64_lower_packed_logical.go new file mode 100644 index 000000000..8605323b5 --- /dev/null +++ b/amd64_lower_packed_logical.go @@ -0,0 +1,90 @@ +package plan9asm + +import "fmt" + +type amd64PackedLogicalMode uint8 + +const ( + amd64PackedLogicalAnd amd64PackedLogicalMode = iota + amd64PackedLogicalAndNot + amd64PackedLogicalOr + amd64PackedLogicalXor +) + +type amd64PackedLogicalSpec struct { + laneBits int + mode amd64PackedLogicalMode +} + +// Compatibility helper entry points all use the same dispatch identity and +// complete typed grammar; none has a separate permissive operand fallback. +func (c *amd64Ctx) lowerPackedFloatingLogical(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedLogicalFamily(op, ins) +} + +func (c *amd64Ctx) lowerEVEXPackedLogical(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedLogicalFamily(op, ins) +} + +func (c *amd64Ctx) lowerPackedAndNot(op Op, ins Instr) (bool, bool, error) { + return c.lowerPackedLogicalFamily(op, ins) +} + +func (c *amd64Ctx) emitPackedLogical(lanes int, spec amd64PackedLogicalSpec, first, second string) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + if spec.mode == amd64PackedLogicalAndNot { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", inverted, vectorType, second, llvmSplatInteger(lanes, spec.laneBits, ^uint64(0))) + second = "%" + inverted + } + result := c.newTmp() + op := "and" + switch spec.mode { + case amd64PackedLogicalOr: + op = "or" + case amd64PackedLogicalXor: + op = "xor" + } + fmt.Fprintf(c.b, " %%%s = %s %s %s, %s\n", result, op, vectorType, second, first) + return "%" + result +} + +func (c *amd64Ctx) isGoEVEXVectorRegister(op Operand, byteWidth int) bool { + return amd64GoPackedVectorMoveRegister(op, byteWidth, c.goarch) +} + +func amd64VectorByteWidth(r Reg) int { + switch { + case isAMD64XReg(r): + return 16 + case isAMD64YReg(r): + return 32 + case isAMD64ZReg(r): + return 64 + default: + return 0 + } +} + +func amd64VectorRegisterHasWidth(op Operand, byteWidth int) bool { + return op.Kind == OpReg && amd64VectorByteWidth(op.Reg) == byteWidth +} + +func (c *amd64Ctx) bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits int, value string) string { + cast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to <%d x i%d>\n", cast, byteWidth, value, lanes, laneBits) + return "%" + cast +} + +func (c *amd64Ctx) storeVectorBytes(dst Reg, byteWidth int, value string) error { + switch byteWidth { + case 16: + return c.storeX(dst, value) + case 32: + return c.storeY(dst, value) + case 64: + return c.storeZ(dst, value) + default: + return fmt.Errorf("unsupported packed vector width %d", byteWidth) + } +} diff --git a/amd64_lower_packed_move_frame_test.go b/amd64_lower_packed_move_frame_test.go new file mode 100644 index 000000000..e731eb61b --- /dev/null +++ b/amd64_lower_packed_move_frame_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86PackedMoveFrameRejectsUndeclaredSlots(t *testing.T) { + for _, op := range []string{"VMOVAPS", "VMOVAPD", "VMOVUPS", "VMOVUPD", "VMOVDQA32", "VMOVDQA64", "VMOVDQU8", "VMOVDQU16", "VMOVDQU32", "VMOVDQU64"} { + for _, reg := range []string{"X0", "Y0", "Z0"} { + for _, instruction := range []string{fmt.Sprintf("%s %s,K7,out+0(FP)", op, reg), fmt.Sprintf("%s out+0(FP),K7,%s", op, reg)} { + for _, arch := range []string{"386", "amd64"} { + file, err := Parse(ArchAMD64, "TEXT framebounds(SB),4,$0-8\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{ + "framebounds": {Name: "framebounds", Ret: I64, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}}, + }}) + if err == nil { + t.Errorf("%s accepted %s beyond a single 8-byte frame slot", arch, instruction) + } + } + } + } + } +} + +func TestX86PackedMoveFrameSlotsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + for index, spec := range x86PackedMoveRuntimeRows { + name := fmt.Sprintf("packedframe%d", index) + body := fmt.Sprintf("TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ mask+24(FP),BX\nKMOVQ BX,K7\nVMOVUPS (DI),X0\n%s a+8(FP),K7,X0\nVMOVUPS X0,(DI)\nVMOVUPS 16(DI),X1\n%s X1,K7,a+8(FP)\nMOVQ a+8(FP),AX\nMOVQ AX,16(DI)\nMOVQ b+16(FP),AX\nMOVQ AX,24(DI)\nRET\n", name, spec.op, spec.op) + sig := FuncSig{Name: name, Args: []LLVMType{Ptr, I64, I64, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}, {Offset: 16, Type: I64, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + if x86PackedMoveRuntimeNativeRow(spec.op, true) { + // Preserve each old fixture as an explicit original-Go accepted + // Context boundary, not an invented aligned LLVM aggregate. + goAssemblerOracleOutput(t, "amd64", body, "", true) + file, err := Parse(ArchAMD64, body) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{name: sig}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual aligned FP fixture %s: %v", spec.op, err) + } + continue + } + source.WriteString(body) + sigs[name] = sig + fmt.Fprintf(&declarations, "extern void %s(void *,uint64_t,uint64_t,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d)) return %d;\n", name, spec.bits/8, index+1) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "x86_64-w64-windows-gnu"} { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "packed-frame.ll", "packed-frame.o", ir) + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; object compilation above is required on every host") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +` + declarations.String() + ` +static int check(void (*fn)(void *,uint64_t,uint64_t,uint64_t),int lane) { + uint64_t words[]={UINT64_C(0x123456789abcdef0),UINT64_C(0xfedcba9876543210)}; + uint64_t masks[]={0,1,UINT64_C(0xaaaa),UINT64_MAX}; + uint8_t frame[16]; memcpy(frame,words,16); + for(int m=0;m<4;m++) { + _Alignas(64) uint8_t out[32],old[32]; + for(int i=0;i<32;i++) out[i]=old[i]=(uint8_t)(19+i*7); + fn(out,words[0],words[1],masks[m]); + for(int i=0;i<16;i++) { + int active=(masks[m]>>(i/lane))&1; + if(out[i]!=(active ? frame[i] : old[i]) || out[16+i]!=(active ? old[16+i] : frame[i])) { + fprintf(stderr,"packed FP mismatch lane=%d mask=%d byte=%d\n",lane,m,i);return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_move_frame", triple, ir, mainC, prefix) +} diff --git a/amd64_lower_packed_move_memory_test.go b/amd64_lower_packed_move_memory_test.go new file mode 100644 index 000000000..cf94eeb48 --- /dev/null +++ b/amd64_lower_packed_move_memory_test.go @@ -0,0 +1,144 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86PackedMoveMaskedMemoryRuntime(t *testing.T) { + x86PackedMoveMaskedMemoryRuntime(t, false) +} + +func x86PackedMoveMaskedMemoryRuntime(t *testing.T, native bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + if native { + execution = x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, spec := range x86PackedMoveRuntimeRows { + if x86PackedMoveRuntimeNativeRow(spec.op, true) != native { + continue + } + for width, reg := range []string{"X0", "Y0", "Z0"} { + bytes := 16 << width + name := fmt.Sprintf("packedmemory%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ in+0(FP),BX\nMOVQ mem+8(FP),AX\nMOVQ out+16(FP),DI\nMOVQ mask+24(FP),CX\nKMOVQ CX,K7\nVMOVUPS (BX),%s\n%s (AX),K7,%s\nVMOVUPS %s,(DI)\nVMOVUPS (BX),%s\n%s.Z (AX),K7,%s\nVMOVUPS %s,%d(DI)\nVMOVUPS (BX),%s\n%s %s,K7,(AX)\nRET\n", name, reg, spec.op, reg, reg, reg, spec.op, reg, reg, bytes, reg, spec.op, reg) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(const void *,void *,void *,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d)) return %d;\n", name, bytes, spec.bits/8, index) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,void *,void *,uint64_t),int bytes,int lane) { + uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX}; + for (int m=0;m<4;m++) { + _Alignas(64) uint8_t in[64],mem[64],out[128],old[64]; + for(int i=0;i<64;i++) { in[i]=(uint8_t)(17+i*3); mem[i]=old[i]=(uint8_t)(255-i*7); } + fn(in,mem,out,masks[m]); + for(int i=0;i>(i/lane))&1; + if(out[i]!=(active ? old[i] : in[i]) || out[bytes+i]!=(active ? old[i] : 0) || mem[i]!=(active ? in[i] : old[i])) { + fprintf(stderr,"packed move bytes=%d lane=%d mask=%d byte=%d mismatch\n",bytes,lane,m,i);return 1; + } + } + if(!masks[m]) { fn(in,0,out,0); if(memcmp(in,out,bytes)) return 1; for(int i=0;i= 3 { + input = prefix + "1" + } + op, mask := spec.op, "" + if form%3 != 0 { + mask = "K7," + } + if form%3 == 2 { + op += ".Z" + } + fmt.Fprintf(&source, "%s %s,%s%s0\nVMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n", op, input, mask, prefix) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d)) return %d;\n", name, 16< +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,const void *,void *,uint64_t),int bytes,int lane,int form) { + uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX}; + for(int m=0;m<4;m++) { + _Alignas(64) uint8_t in[64],old[64],out[112]; + for(int i=0;i<64;i++) {in[i]=(uint8_t)(17+i*3);old[i]=(uint8_t)(255-i*7);} + fn(in,old,out,masks[m]); + for(int view=0;view<3;view++) for(int i=0;i<(64>>view);i++) { + int active=form==0 || ((masks[m]>>(i/lane))&1); + uint8_t want=i>=bytes ? 0 : active ? in[i] : form==2 ? 0 : old[i]; + int offset=view==0 ? 0 : view==1 ? 64 : 96; + if(out[offset+i]!=want) {fprintf(stderr,"packed views width=%d lane=%d form=%d mask=%d view=%d byte=%d got=%d want=%d\n",bytes,lane,form,m,view,i,out[offset+i],want);return 1;} + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + execution.compileAndRunLLVM(t, "packed-move-views", ir, mainC, "-O2") +} diff --git a/amd64_lower_packed_numeric_conversion.go b/amd64_lower_packed_numeric_conversion.go new file mode 100644 index 000000000..1b9a9e45b --- /dev/null +++ b/amd64_lower_packed_numeric_conversion.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedNumericKind uint8 + +const ( + amd64PackedNumericFloat amd64PackedNumericKind = iota + amd64PackedNumericSigned + amd64PackedNumericUnsigned +) + +type amd64PackedNumericShape struct { + sourceBytes int + destinationBytes int + lanes int +} + +type amd64PackedNumericConversionSpec struct { + sourceKind amd64PackedNumericKind + destinationKind amd64PackedNumericKind + sourceBits int + destinationBits int + truncate bool + explicit string + shapes []amd64PackedNumericShape +} + +var ( + amd64PackedNumericSame32 = []amd64PackedNumericShape{ + {sourceBytes: 16, destinationBytes: 16, lanes: 4}, + {sourceBytes: 32, destinationBytes: 32, lanes: 8}, + {sourceBytes: 64, destinationBytes: 64, lanes: 16}, + } + amd64PackedNumericSame64 = []amd64PackedNumericShape{ + {sourceBytes: 16, destinationBytes: 16, lanes: 2}, + {sourceBytes: 32, destinationBytes: 32, lanes: 4}, + {sourceBytes: 64, destinationBytes: 64, lanes: 8}, + } + amd64PackedNumericWiden32To64 = []amd64PackedNumericShape{ + {sourceBytes: 16, destinationBytes: 16, lanes: 2}, + {sourceBytes: 16, destinationBytes: 32, lanes: 4}, + {sourceBytes: 32, destinationBytes: 64, lanes: 8}, + } +) + +// amd64PackedNumericConversionSpecs describes the complete Go 1.27 EVEX +// packed numeric-conversion family not covered by the legacy/VEX conversion +// handlers. Source/destination element types and physical width shapes remain +// independent axes, avoiding one lowerer per mnemonic and register width. +var amd64PackedNumericConversionSpecs = map[Op]amd64PackedNumericConversionSpec{ + "VCVTPD2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTPD2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTTPD2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 64, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericSame64}, + "VCVTTPD2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericSame64}, + "VCVTPS2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 32, explicit: "round", shapes: amd64PackedNumericSame32}, + "VCVTTPS2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 32, truncate: true, explicit: "sae", shapes: amd64PackedNumericSame32}, + "VCVTUDQ2PS": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 32, destinationBits: 32, explicit: "round", shapes: amd64PackedNumericSame32}, + "VCVTQQ2PD": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTUQQ2PD": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericSame64}, + "VCVTPS2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 32, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericWiden32To64}, + "VCVTPS2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 64, explicit: "round", shapes: amd64PackedNumericWiden32To64}, + "VCVTTPS2QQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, sourceBits: 32, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericWiden32To64}, + "VCVTTPS2UQQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 32, destinationBits: 64, truncate: true, explicit: "sae", shapes: amd64PackedNumericWiden32To64}, + "VCVTUDQ2PD": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 32, destinationBits: 64, shapes: amd64PackedNumericWiden32To64}, + "VCVTPD2UDQX": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTPD2UDQY": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTPD2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, explicit: "round", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, + "VCVTTPD2UDQX": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, truncate: true, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTTPD2UDQY": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, truncate: true, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTTPD2UDQ": {sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericUnsigned, sourceBits: 64, destinationBits: 32, truncate: true, explicit: "sae", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, + "VCVTQQ2PSX": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTQQ2PSY": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTQQ2PS": {sourceKind: amd64PackedNumericSigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, explicit: "round", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, + "VCVTUQQ2PSX": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}}}, + "VCVTUQQ2PSY": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, shapes: []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}}}, + "VCVTUQQ2PS": {sourceKind: amd64PackedNumericUnsigned, destinationKind: amd64PackedNumericFloat, sourceBits: 64, destinationBits: 32, explicit: "round", shapes: []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}}}, +} + +func (c *amd64Ctx) lowerPackedNumericConversion(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedConversionSpecification(Op(base)); !known { + return false, false, nil + } + if !strings.EqualFold(string(op), string(ins.Op)) { + return true, false, fmt.Errorf("packed conversion dispatch/source mismatch: %s/%s", op, ins.Op) + } + form, _, err := x86PackedConversionProperties(ins, c.goarch) + if err != nil { + return true, false, err + } + if form.mmx || !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: packed conversion lacks native target or physical MMX/x87 caller contract", ErrProbeNeedsContext) + } + source := ins.Args[0] + if source.Kind == OpFP { + if err := c.validateFPVectorSpan(source.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + return true, false, fmt.Errorf("%w: packed FP conversion source lacks original native backing", ErrProbeNeedsContext) + } + if ins.x86AddressBits != 0 || source.Kind == OpMem && + !x86OrdinaryNativeMemoryOperand(source, c.goarch) && !c.fixedLocalMemoryCertified(ins, form.memory) { + return true, false, fmt.Errorf("%w: packed conversion lacks original native address binding", ErrProbeNeedsContext) + } + base = strings.ToLower(base) + mnemonic := strings.TrimSuffix(strings.TrimSuffix(base, "x"), "y") + if form.spec.legacy { + mnemonic = strings.ReplaceAll(base, "pl", "dq") + } + // Narrowing memory leaves X/Y source length ambiguous to an assembler: + // encode the source vector length, not the destination carrier width. + if form.memory >= 0 && !form.spec.legacy && !form.properties.broadcast && + form.spec.sourceBits > form.spec.destinationBits && form.shape.sourceBytes < 64 { + mnemonic += map[int]string{16: "x", 32: "y"}[form.shape.sourceBytes] + } + return true, false, c.emitNativePackedConversion(ins, x86NativePackedConversion{ + byteWidth: form.shape.destinationBytes, sourceBytes: form.shape.sourceBytes, + memoryBytes: form.memoryBytes, outputBits: form.spec.destinationBits, + lanes: form.shape.lanes, + destination: form.destination, mask: form.mask, legacy: form.spec.legacy, evex: form.evex, + broadcast: form.properties.broadcast, zeroing: form.properties.zeroing, + rounding: form.properties.rounding, sae: form.properties.sae, mnemonic: mnemonic, + }) +} + +func (c *amd64Ctx) packedNumericRoundingMode(rounding string) string { + switch rounding { + case "RN_SAE": + return "0" + case "RD_SAE": + return "1" + case "RU_SAE": + return "2" + case "RZ_SAE": + return "3" + default: + mxcsr := c.loadMXCSR() + shifted, mode := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i32 %s, 13\n", shifted, mxcsr) + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 3\n", mode, shifted) + return "%" + mode + } +} diff --git a/amd64_lower_packed_numeric_conversion_test.go b/amd64_lower_packed_numeric_conversion_test.go new file mode 100644 index 000000000..b590e692d --- /dev/null +++ b/amd64_lower_packed_numeric_conversion_test.go @@ -0,0 +1,299 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type packedNumericConversionTestShape struct { + source string + destination string +} + +type packedNumericConversionTestSpec struct { + op string + shapes []packedNumericConversionTestShape + explicit string +} + +func go127PackedNumericConversionSpecs() []packedNumericConversionTestSpec { + same := []packedNumericConversionTestShape{{"X", "X"}, {"Y", "Y"}, {"Z", "Z"}} + widen := []packedNumericConversionTestShape{{"X", "X"}, {"X", "Y"}, {"Y", "Z"}} + return []packedNumericConversionTestSpec{ + {op: "VCVTPD2QQ", shapes: same, explicit: "round"}, + {op: "VCVTPD2UQQ", shapes: same, explicit: "round"}, + {op: "VCVTTPD2QQ", shapes: same, explicit: "sae"}, + {op: "VCVTTPD2UQQ", shapes: same, explicit: "sae"}, + {op: "VCVTPS2UDQ", shapes: same, explicit: "round"}, + {op: "VCVTTPS2UDQ", shapes: same, explicit: "sae"}, + {op: "VCVTUDQ2PS", shapes: same, explicit: "round"}, + {op: "VCVTQQ2PD", shapes: same, explicit: "round"}, + {op: "VCVTUQQ2PD", shapes: same, explicit: "round"}, + {op: "VCVTPS2QQ", shapes: widen, explicit: "round"}, + {op: "VCVTPS2UQQ", shapes: widen, explicit: "round"}, + {op: "VCVTTPS2QQ", shapes: widen, explicit: "sae"}, + {op: "VCVTTPS2UQQ", shapes: widen, explicit: "sae"}, + {op: "VCVTUDQ2PD", shapes: widen}, + {op: "VCVTPD2UDQX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTPD2UDQY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTPD2UDQ", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "round"}, + {op: "VCVTTPD2UDQX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTTPD2UDQY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTTPD2UDQ", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "sae"}, + {op: "VCVTQQ2PSX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTQQ2PSY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTQQ2PS", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "round"}, + {op: "VCVTUQQ2PSX", shapes: []packedNumericConversionTestShape{{"X", "X"}}}, + {op: "VCVTUQQ2PSY", shapes: []packedNumericConversionTestShape{{"Y", "X"}}}, + {op: "VCVTUQQ2PS", shapes: []packedNumericConversionTestShape{{"Z", "Y"}}, explicit: "round"}, + } +} + +func TestAMD64PackedNumericConversionGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127PackedNumericConversionSpecs() + if len(amd64PackedNumericConversionSpecs) != len(expected) { + t.Fatalf("packed numeric conversion grammar has %d entries, want %d", len(amd64PackedNumericConversionSpecs), len(expected)) + } + for _, want := range expected { + if _, ok := amd64PackedNumericConversionSpecs[Op(want.op)]; !ok { + t.Errorf("packed numeric conversion grammar omitted %s", want.op) + } + } +} + +func TestTranslateX86PackedNumericConversionCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + last := 23 + if target.goarch == "386" { + last = 7 + } + var source strings.Builder + source.WriteString("TEXT packednumericconversionforms(SB),$0-0\n") + for index, spec := range go127PackedNumericConversionSpecs() { + for _, shape := range spec.shapes { + fmt.Fprintf(&source, "\t%s %s1, %s%d\n", spec.op, shape.source, shape.destination, last) + fmt.Fprintf(&source, "\t%s %d(AX), %s%d\n", spec.op, 8+index*8, shape.destination, last) + fmt.Fprintf(&source, "\t%s.BCST %d(AX), %s%d\n", spec.op, 16+index*8, shape.destination, last) + fmt.Fprintf(&source, "\t%s %s2, K1, %s%d\n", spec.op, shape.source, shape.destination, last) + fmt.Fprintf(&source, "\t%s.Z %d(AX), K2, %s%d\n", spec.op, 24+index*8, shape.destination, last) + fmt.Fprintf(&source, "\t%s.BCST.Z %d(AX), K3, %s%d\n", spec.op, 32+index*8, shape.destination, last) + } + if spec.explicit == "round" { + shape := spec.shapes[len(spec.shapes)-1] + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s %s3, %s%d\n", spec.op, rounding, shape.source, shape.destination, last) + } + fmt.Fprintf(&source, "\t%s.RN_SAE.Z %s4, K4, %s%d\n", spec.op, shape.source, shape.destination, last) + } else if spec.explicit == "sae" { + shape := spec.shapes[len(spec.shapes)-1] + fmt.Fprintf(&source, "\t%s.SAE %s3, %s%d\n", spec.op, shape.source, shape.destination, last) + fmt.Fprintf(&source, "\t%s.SAE.Z %s4, K4, %s%d\n", spec.op, shape.source, shape.destination, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"packednumericconversionforms": {Name: "packednumericconversionforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-numeric-conversion-"+target.name+".ll", "packed-numeric-conversion-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedNumericConversionRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VCVTPD2QQ X0"}, + {goarch: "amd64", instruction: "VCVTPD2QQ X0, Y1"}, + {goarch: "amd64", instruction: "VCVTPS2QQ Y0, Y1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PD Y0, Y1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQX Y0, X1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQY X0, X1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQ Z0, Z1"}, + {goarch: "amd64", instruction: "VCVTQQ2PSX X0, Y1"}, + {goarch: "amd64", instruction: "VCVTQQ2PSY X0, X1"}, + {goarch: "amd64", instruction: "VCVTQQ2PS Z0, Z1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PS.Z X0, X1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PS X0, K0, X1"}, + {goarch: "amd64", instruction: "VCVTUDQ2PS.BCST X0, X1"}, + {goarch: "amd64", instruction: "VCVTPD2QQ.RN_SAE Y0, Y1"}, + {goarch: "amd64", instruction: "VCVTPD2QQ.RN_SAE 0(AX), Z1"}, + {goarch: "amd64", instruction: "VCVTTPD2QQ.RN_SAE Z0, Z1"}, + {goarch: "amd64", instruction: "VCVTPD2UDQX.RN_SAE X0, X1"}, + {goarch: "386", instruction: "VCVTPD2QQ Z8, Z1"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed numeric conversion tables", test.instruction) + } + }) + } +} + +func amd64PackedNumericConversionNativeRuntimeSemantics(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + const source = `TEXT packednumericconversionsemantics(SB),$0-64 + MOVQ out+0(FP), AX + MOVQ f64+8(FP), BX + MOVQ f32+16(FP), CX + MOVQ u32+24(FP), DX + MOVQ i64+32(FP), R8 + MOVQ u64+40(FP), R9 + MOVQ old64+48(FP), R10 + MOVQ mask+56(FP), R11 + KMOVQ R11, K1 + VMOVDQU64 0(BX), Z0 + VCVTPD2QQ Z0, Z1 + VMOVDQU64 Z1, 0(AX) + VCVTPD2UQQ Z0, Z2 + VMOVDQU64 Z2, 64(AX) + VCVTPD2QQ.RU_SAE Z0, Z1 + VMOVDQU64 Z1, 128(AX) + VCVTTPD2UQQ.SAE Z0, Z2 + VMOVDQU64 Z2, 192(AX) + VMOVDQU32 0(CX), Y3 + VCVTPS2QQ Y3, Z4 + VMOVDQU64 Z4, 256(AX) + VCVTTPS2UQQ Y3, Z5 + VMOVDQU64 Z5, 320(AX) + VMOVDQU32 0(CX), Z6 + VCVTPS2UDQ.BCST 0(CX), Z7 + VMOVDQU32 Z7, 384(AX) + VCVTTPS2UDQ Z6, Z7 + VMOVDQU32 Z7, 448(AX) + VMOVDQU32 0(DX), Z8 + VCVTUDQ2PS Z8, Z9 + VMOVDQU32 Z9, 512(AX) + VCVTUDQ2PS.RD_SAE Z8, Z9 + VMOVDQU32 Z9, 576(AX) + VCVTUDQ2PD 0(DX), Z10 + VMOVDQU64 Z10, 640(AX) + VMOVDQU64 0(R8), Z11 + VCVTQQ2PD Z11, Z12 + VMOVDQU64 Z12, 704(AX) + VMOVDQU64 0(R9), Z13 + VCVTUQQ2PD Z13, Z14 + VMOVDQU64 Z14, 768(AX) + VCVTQQ2PS Z11, Y15 + VMOVDQU32 Y15, 832(AX) + VCVTUQQ2PS.RU_SAE Z13, Y16 + VMOVDQU32 Y16, 864(AX) + VMOVDQU64 0(R10), Z17 + VCVTPD2QQ.Z Z0, K1, Z17 + VMOVDQU64 Z17, 896(AX) + VMOVDQU64 0(BX), X0 + VCVTPD2UDQX X0, X18 + VMOVDQU32 X18, 960(AX) + VMOVDQU64 0(BX), Y0 + VCVTTPD2UDQY Y0, X19 + VMOVDQU32 X19, 976(AX) + STC + SETCS 992(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := execution.triple + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"packednumericconversionsemantics": { + Name: "packednumericconversionsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#pragma STDC FENV_ACCESS ON +#include +#include +#include +#include +#include +#include +extern void packednumericconversionsemantics(uint8_t *, const double *, const float *, const uint32_t *, const int64_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t u32(const uint8_t *p){uint32_t v;memcpy(&v,p,4);return v;} static uint64_t u64(const uint8_t *p){uint64_t v;memcpy(&v,p,8);return v;} +static double rd(double x,int truncating,int fixed){if(truncating)return trunc(x);if(fixed==1)return nearbyint(x);int rc=fegetround();return rc==FE_DOWNWARD?floor(x):(rc==FE_UPWARD?ceil(x):(rc==FE_TOWARDZERO?trunc(x):nearbyint(x)));} +static float rf(float x,int truncating){if(truncating)return truncf(x);int rc=fegetround();return rc==FE_DOWNWARD?floorf(x):(rc==FE_UPWARD?ceilf(x):(rc==FE_TOWARDZERO?truncf(x):nearbyintf(x)));} +static uint64_t f64i(double x,int uns,int truncating,int fixed){double r=rd(x,truncating,fixed);if(uns){if(!(r>=0.0&&r<18446744073709551616.0))return UINT64_MAX;return (uint64_t)r;}if(!(r>=-9223372036854775808.0&&r<9223372036854775808.0))return UINT64_C(0x8000000000000000);return (uint64_t)(int64_t)r;} +static uint64_t f32i64(float x,int uns,int truncating){float r=rf(x,truncating);if(uns){if(!(r>=0.0f&&r<18446744073709551616.0f))return UINT64_MAX;return (uint64_t)r;}if(!(r>=-9223372036854775808.0f&&r<9223372036854775808.0f))return UINT64_C(0x8000000000000000);return (uint64_t)(int64_t)r;} +static uint32_t f32u32(float x,int truncating){float r=rf(x,truncating);if(!(r>=0.0f&&r<4294967296.0f))return UINT32_MAX;return (uint32_t)r;} +static uint32_t fb(float x){uint32_t u;memcpy(&u,&x,4);return u;} static uint64_t db(double x){uint64_t u;memcpy(&u,&x,8);return u;} +static float cu32f(uint32_t x){volatile uint32_t v=x;return (float)v;} static double cu32d(uint32_t x){volatile uint32_t v=x;return (double)v;} +static float ci64f(int64_t x){volatile int64_t v=x;return (float)v;} static double ci64d(int64_t x){volatile int64_t v=x;return (double)v;} +static float cu64f(uint64_t x){volatile uint64_t v=x;return (float)v;} static double cu64d(uint64_t x){volatile uint64_t v=x;return (double)v;} +int main(void){ + const double d[8]={1.5,2.5,-1.5,-2.5,9223372036854775808.0,-9223372036854775808.0,18446744073709551616.0,NAN}; + const float f[16]={1.5f,2.5f,-1.5f,-2.5f,0.0f,-0.0f,4294967296.0f,NAN,16777217.0f,2147483648.0f,-2147483904.0f,3.75f,INFINITY,-INFINITY,0x1p-149f,0x1.fffffep127f}; + const uint32_t ui32[16]={0,1,16777215,16777217,UINT32_MAX,2147483648u,3,5,7,9,11,13,15,17,19,21}; + const int64_t si64[8]={0,1,-1,INT64_C(9007199254740993),-INT64_C(9007199254740993),INT64_MAX,INT64_MIN,123456789}; + const uint64_t ui64[8]={0,1,UINT64_C(9007199254740993),UINT64_C(16777217),UINT64_MAX,UINT64_C(1)<<63,3,123456789}; + const uint64_t old[8]={10,11,12,13,14,15,16,17},mask=UINT64_C(0xa5); unsigned original=_mm_getcsr(); const int modes[4]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + for(int rc=0;rc<4;rc++){uint8_t out[993]={0};fesetround(modes[rc]);packednumericconversionsemantics(out,d,f,ui32,si64,ui64,old,mask); + for(int i=0;i<8;i++){if(u64(out+i*8)!=f64i(d[i],0,0,0))return 10+rc;if(u64(out+64+i*8)!=f64i(d[i],1,0,0))return 20+rc; + int save=fegetround();fesetround(FE_UPWARD);uint64_t ru=f64i(d[i],0,0,0);fesetround(save);if(u64(out+128+i*8)!=ru)return 30+rc;if(u64(out+192+i*8)!=f64i(d[i],1,1,0))return 40+rc; + if(u64(out+256+i*8)!=f32i64(f[i],0,0))return 50+rc;if(u64(out+320+i*8)!=f32i64(f[i],1,1))return 60+rc;} + for(int i=0;i<16;i++){if(u32(out+384+i*4)!=f32u32(f[0],0))return 70+rc;if(u32(out+448+i*4)!=f32u32(f[i],1))return 80+rc;if(u32(out+512+i*4)!=fb(cu32f(ui32[i])))return 90+rc; + int save=fegetround();fesetround(FE_DOWNWARD);uint32_t down=fb(cu32f(ui32[i]));fesetround(save);if(u32(out+576+i*4)!=down){fprintf(stderr,"rd rc=%d i=%d got=%08x want=%08x\n",rc,i,u32(out+576+i*4),down);return 100+i;}} + for(int i=0;i<8;i++){if(u64(out+640+i*8)!=db(cu32d(ui32[i])))return 110+rc;if(u64(out+704+i*8)!=db(ci64d(si64[i])))return 120+rc;if(u64(out+768+i*8)!=db(cu64d(ui64[i])))return 130+rc;if(u32(out+832+i*4)!=fb(ci64f(si64[i])))return 140+rc; + int save=fegetround();fesetround(FE_UPWARD);uint32_t up=fb(cu64f(ui64[i]));fesetround(save);if(u32(out+864+i*4)!=up)return 150+rc;if(u64(out+896+i*8)!=(((mask>>i)&1)?f64i(d[i],0,0,0):0))return 160+rc;} + if(u32(out+960)!=f32u32((float)d[0],0)||u32(out+964)!=f32u32((float)d[1],0)||u64(out+968)!=0){fprintf(stderr,"narrow rc=%d got=%08x,%08x,%016llx want=%08x,%08x,0\n",rc,u32(out+960),u32(out+964),(unsigned long long)u64(out+968),f32u32((float)d[0],0),f32u32((float)d[1],0));return 170+rc;} + for(int i=0;i<4;i++)if(u32(out+976+i*4)!=f32u32((float)d[i],1))return 180+rc;if(out[992]!=1)return 190+rc; + } + _mm_setcsr(original);fesetround(FE_TONEAREST);return 0; +} +` + execution.compileAndRunLLVM(t, "packed_numeric_conversion_semantics", ir, mainC, "-O2") +} diff --git a/amd64_lower_packed_rotate.go b/amd64_lower_packed_rotate.go new file mode 100644 index 000000000..c4b58a79f --- /dev/null +++ b/amd64_lower_packed_rotate.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" +) + +type amd64PackedRotateSpec struct { + laneBits int + left bool + variable bool +} + +// amd64PackedRotateSpecs covers Go 1.27's complete _yvprold immediate and +// _yvblendmpd per-lane variable rotate families. +var amd64PackedRotateSpecs = map[Op]amd64PackedRotateSpec{ + "VPROLD": {laneBits: 32, left: true}, + "VPROLQ": {laneBits: 64, left: true}, + "VPRORD": {laneBits: 32}, + "VPRORQ": {laneBits: 64}, + "VPROLVD": {laneBits: 32, left: true, variable: true}, + "VPROLVQ": {laneBits: 64, left: true, variable: true}, + "VPRORVD": {laneBits: 32, variable: true}, + "VPRORVQ": {laneBits: 64, variable: true}, +} + +func (c *amd64Ctx) lowerPackedRotate(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64PackedRotateSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's EVEX encoding: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s has an operand count absent from its Go 1.27 table: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed Go 1.27's assembler operand limit: %q", baseOp, ins.Raw) + } + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + + firstIndex := 0 + if !spec.variable { + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("%s %s first operand must be Go 1.27's unsigned-imm8 class: %q", c.goarch, baseOp, ins.Raw) + } + firstIndex = 1 + } + dstArg := ins.Args[len(ins.Args)-1] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be an X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(dstArg.Reg) + if (byteWidth != 16 && byteWidth != 32 && byteWidth != 64) || !c.isGoEVEXVectorRegister(dstArg, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's EVEX vector class: %q", c.goarch, baseOp, ins.Raw) + } + + firstArg := ins.Args[firstIndex] + if broadcast { + if !isAMD64MemoryOperand(firstArg) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory first source: %q", c.goarch, baseOp, ins.Raw) + } + } else if firstArg.Kind == OpReg { + if !c.isGoEVEXVectorRegister(firstArg, byteWidth) { + return true, false, fmt.Errorf("%s %s first source must match the destination width and register class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(firstArg) { + return true, false, fmt.Errorf("%s %s first source must be a vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if spec.variable && !c.isGoEVEXVectorRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("%s %s data source must be a same-width EVEX vector register: %q", c.goarch, baseOp, ins.Raw) + } + + var mask string + if masked { + if !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + + return c.emitPackedRotateInstruction(baseOp, spec, ins, firstArg, dstArg, mask, broadcast, zeroing) +} + +func (c *amd64Ctx) emitPackedRotateInstruction(baseOp string, spec amd64PackedRotateSpec, ins Instr, firstArg, dstArg Operand, mask string, broadcast, zeroing bool) (bool, bool, error) { + byteWidth := amd64VectorByteWidth(dstArg.Reg) + lanes := byteWidth * 8 / spec.laneBits + loadLanes := func(arg Operand, allowBroadcast bool) (string, error) { + if allowBroadcast && broadcast { + scalar, err := c.evalIntSized(arg, amd64IntegerTypeForBits(spec.laneBits)) + if err != nil { + return "", err + } + return amd64SplatInteger(c, lanes, spec.laneBits, scalar), nil + } + bytesValue, err := c.loadPackedCompareBytes(arg, byteWidth) + if err != nil { + return "", err + } + return c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, bytesValue), nil + } + + var data, counts string + var err error + if spec.variable { + counts, err = loadLanes(firstArg, true) + if err == nil { + data, err = loadLanes(ins.Args[1], false) + } + } else { + data, err = loadLanes(firstArg, true) + counts = amd64SplatInteger(c, lanes, spec.laneBits, strconv.Itoa(int(ins.Args[0].Imm)&(spec.laneBits-1))) + } + if err != nil { + return true, false, err + } + result := c.emitPackedRotate(lanes, spec.laneBits, data, counts, spec.left) + if mask != "" { + oldBytes, err := c.loadPackedCompareBytes(dstArg, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + if err := c.storeVectorBytes(dstArg.Reg, byteWidth, "%"+out); err != nil { + return true, false, fmt.Errorf("%s %s destination: %w", c.goarch, baseOp, err) + } + return true, false, nil +} + +func (c *amd64Ctx) emitPackedRotate(lanes, laneBits int, data, counts string, left bool) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + laneMask := amd64SplatInteger(c, lanes, laneBits, strconv.Itoa(laneBits-1)) + normalized := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", normalized, vectorType, counts, laneMask) + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s zeroinitializer, %%%s\n", negated, vectorType, normalized) + inverse := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", inverse, vectorType, negated, laneMask) + + low, high := c.newTmp(), c.newTmp() + if left { + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", low, vectorType, data, normalized) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", high, vectorType, data, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", low, vectorType, data, normalized) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", high, vectorType, data, inverse) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", result, vectorType, low, high) + return "%" + result +} diff --git a/amd64_lower_packed_shift.go b/amd64_lower_packed_shift.go new file mode 100644 index 000000000..2ea121bf6 --- /dev/null +++ b/amd64_lower_packed_shift.go @@ -0,0 +1,204 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerX86PackedShift(op Op, ins Instr) (bool, bool, error) { + form, recognized, err := x86PackedShiftProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + local := ins + local.Op = op + if err := x86LegacyMMXReadEmitterContext(local, c.goarch); err != nil { + return true, false, err + } + if x86PackedShiftHasNativeCount(form) && ins.Args[form.count].Kind == OpFP { + return true, false, fmt.Errorf("legacy m128 count needs actual native FP backing: %w", ErrProbeNeedsContext) + } + if form.memory >= 0 && ins.Args[form.memory].Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(ins.Args[form.memory].FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + dataArg := ins.Args[form.data] + var data string + if form.mmx { + bits, err := c.loadReg(dataArg.Reg) + if err != nil { + return true, false, err + } + data = c.bitcastI64ToIntegerLanes(64/form.laneBits, form.laneBits, bits) + } else if form.countMode == x86PackedShiftBytes { + data, err = c.loadX86PackedShiftBytes(dataArg, form.byteWidth, true) + } else { + dataMask := "" + if form.faultSuppressed && form.memory == form.data { + dataMask = mask + } + if form.memory == form.data && !form.faultSuppressed { + bytes, loadErr := c.loadX86PackedShiftBytes(dataArg, form.byteWidth, true) + err = loadErr + if err == nil { + data = c.bitcastVectorBytesToIntegerLanes(form.byteWidth, form.byteWidth*8/form.laneBits, form.laneBits, bytes) + } + } else { + data, err = c.loadX86PackedShiftLanes(dataArg, form.byteWidth, form.laneBits, form.broadcast && form.memory == form.data, dataMask) + } + } + if err != nil { + return true, false, err + } + lanes := form.byteWidth * 8 / form.laneBits + var result string + if form.countMode == x86PackedShiftBytes { + result = c.emitPackedByteShift(form.byteWidth, data, uint8(form.countOperand.Imm), form.left) + } else { + if form.countMode == x86PackedShiftPerLane { + counts, err := c.loadX86PackedShiftLanes(ins.Args[form.count], form.byteWidth, form.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + operation := "lshr" + if form.left { + operation = "shl" + } + result = c.emitPerLaneVariableShift(amd64PerLaneVariableShiftSpec{laneBits: form.laneBits, operation: operation}, lanes, counts, data) + } else if x86PackedShiftHasNativeCount(form) { + result, err = c.emitX86PackedShiftNativeCount(form, ins.Args[form.count], data) + if err != nil { + return true, false, err + } + } else { + count, err := c.loadPackedUniformLogicalShiftCount(form.countOperand, form.mmx) + if err != nil { + return true, false, err + } + result = c.emitPackedLogicalShift(amd64PackedLogicalShiftSpec{laneBits: form.laneBits, left: form.left}, lanes, data, count) + } + if form.mask >= 0 { + old, err := c.loadPackedCompareLanes(ins.Args[form.destination], form.byteWidth, form.laneBits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + if form.mmx { + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, lanes, form.laneBits, result) + return true, false, c.storeReg(ins.Args[form.destination].Reg, "%"+out) + } + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.laneBits, result, form.byteWidth) + result = "%" + out + } + if form.legacy { + return true, false, c.storeLegacyPackedLogical(ins.Args[form.destination], result) + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, result) +} + +// Native reads remain observable even when the shift result is a known zero. +// Only actual EVEX mask bits can suppress the corresponding memory accesses; +// LLVM dead-value elimination is not a native fault-suppression contract. +func (c *amd64Ctx) loadX86PackedShiftLanes(source Operand, width, bits int, broadcast bool, mask string) (string, error) { + if source.Kind != OpMem && source.Kind != OpSym && source.Kind != OpFP { + return c.loadMaskedPackedCompareLanes(source, width, bits, broadcast, mask) + } + lanes := width * 8 / bits + if mask == "" { + bytes := width + if broadcast { + bytes = bits / 8 + } + value, err := c.loadX86PackedShiftBytes(source, bytes, true) + if err != nil { + return "", err + } + if !broadcast { + return c.bitcastVectorBytesToIntegerLanes(width, lanes, bits, value), nil + } + scalar := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", scalar, bytes, value, bits) + return amd64SplatInteger(c, lanes, bits, "%"+scalar), nil + } + load := func(offset int) func() (string, error) { + return func() (string, error) { + operand := source + if operand.Kind == OpFP { + operand.FPOffset += int64(offset) + bytes, err := c.loadX86PackedShiftBytes(operand, bits/8, true) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", value, bits/8, bytes, bits) + return "%" + value, nil + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(operand) + if err != nil { + return "", err + } + at, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", at, pointerType, pointer, offset) + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %%%s, align 1\n", value, bits, pointerType, at) + return "%" + value, nil + } + } + if broadcast { + relevant, active := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", relevant, mask, uint64(1)< %s, i%d %s, i32 %d\n", inserted, lanes, bits, result, bits, value, lane) + result = "%" + inserted + } + return result, nil +} + +func (c *amd64Ctx) loadX86PackedShiftBytes(source Operand, width int, observable bool) (string, error) { + if source.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(source.FPOffset, width); err != nil { + return "", err + } + if c.classicFrame == "" { + // Every native ABI0 byte is bound; preserve its little-endian + // value without pretending independent allocas are contiguous. + return c.loadFPVectorBytes(source.FPOffset, width) + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, ptr %s, align 1\n", + value, width, c.classicFramePtr(source.FPOffset)) + return "%" + value, nil + } + if !observable || source.Kind != OpMem && source.Kind != OpSym { + return c.loadPackedCompareBytes(source, width) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, width, pointerType, pointer) + return "%" + value, nil +} diff --git a/amd64_lower_packed_testmask.go b/amd64_lower_packed_testmask.go new file mode 100644 index 000000000..e955d81f2 --- /dev/null +++ b/amd64_lower_packed_testmask.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedTestMaskSpec struct { + laneBits int + testZero bool +} + +var amd64PackedTestMaskSpecs = map[string]amd64PackedTestMaskSpec{ + "VPTESTMB": {laneBits: 8}, + "VPTESTMW": {laneBits: 16}, + "VPTESTMD": {laneBits: 32}, + "VPTESTMQ": {laneBits: 64}, + "VPTESTNMB": {laneBits: 8, testZero: true}, + "VPTESTNMW": {laneBits: 16, testZero: true}, + "VPTESTNMD": {laneBits: 32, testZero: true}, + "VPTESTNMQ": {laneBits: 64, testZero: true}, +} + +// lowerPackedTestMask implements the complete Go 1.27 _yvpshufbitqmb +// operand table shared by VPTESTM{B,W,D,Q} and VPTESTNM{B,W,D,Q}. +// VPTESTM sets each K bit when the corresponding source-lane intersection is +// nonzero; VPTESTNM sets it when that intersection is zero. +func (c *amd64Ctx) lowerPackedTestMask(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, ok := amd64PackedTestMaskSpecs[baseOp] + if !ok { + return false, false, nil + } + broadcast := suffix == "BCST" + if suffix != "" && !broadcast { + return true, false, fmt.Errorf("%s %s accepts only the .BCST suffix supported by Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects first source, second source, [K mask,] K destination: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && len(ins.Args) == 4 && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s write-mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + + second := ins.Args[1] + byteWidth, validWidth := amd64PackedCompareWidth(second) + if !validWidth || !c.isGoPackedVectorMoveRegister(second, byteWidth) { + return true, false, fmt.Errorf("%s %s second source must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + first := ins.Args[0] + if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("%s %s register sources must have matching widths and be in range: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s first source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if broadcast { + if spec.laneBits != 32 && spec.laneBits != 64 { + return true, false, fmt.Errorf("%s %s does not enable broadcast in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory first source: %q", c.goarch, baseOp, ins.Raw) + } + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be K0-K7: %q", c.goarch, baseOp, ins.Raw) + } + if _, validDestination := amd64ParseKReg(destination.Reg); !validDestination { + return true, false, fmt.Errorf("%s %s destination must be K0-K7: %q", c.goarch, baseOp, ins.Raw) + } + + writeMask := "" + if len(ins.Args) == 4 { + mask := ins.Args[2] + if mask.Kind != OpReg { + return true, false, fmt.Errorf("%s %s write mask must be K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + maskIndex, validMask := amd64ParseKReg(mask.Reg) + if !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("%s %s write mask must be K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + writeMask, err = c.loadK(mask.Reg) + if err != nil { + return true, false, err + } + } + + firstValue, err := c.loadPackedCompareLanes(first, byteWidth, spec.laneBits, broadcast) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + intersection := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", intersection, vectorType, firstValue, secondValue) + predicate := "ne" + if spec.testZero { + predicate = "eq" + } + tested := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s %s %%%s, zeroinitializer\n", tested, predicate, vectorType, intersection) + packedType := fmt.Sprintf("i%d", lanes) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %%%s to %s\n", packed, lanes, tested, packedType) + result := "%" + packed + if lanes < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if writeMask != "" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", masked, result, writeMask) + result = "%" + masked + } + return true, false, c.storeK(destination.Reg, result) +} diff --git a/amd64_lower_packed_testmask_test.go b/amd64_lower_packed_testmask_test.go new file mode 100644 index 000000000..1ecee0a6e --- /dev/null +++ b/amd64_lower_packed_testmask_test.go @@ -0,0 +1,126 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedTestMaskCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 gives VPTESTM{B,W,D,Q} and VPTESTNM{B,W,D,Q} the same + // _yvpshufbitqmb operand table: X/Y/Z r/m first source, a matching + // vector second source, an optional K1-K7 write mask, and a K + // destination. D/Q additionally enable scalar-memory broadcast. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedtestmaskforms(SB),$0-0\n") + for _, op := range []string{ + "VPTESTMB", "VPTESTMW", "VPTESTMD", "VPTESTMQ", + "VPTESTNMB", "VPTESTNMW", "VPTESTNMD", "VPTESTNMQ", + } { + zLast := 31 + if target.goarch == "386" { + zLast = 7 + } + fmt.Fprintf(&source, "\t%s X1, X31, K0\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), X31, K2\n", op) + fmt.Fprintf(&source, "\t%s Y1, Y31, K3\n", op) + fmt.Fprintf(&source, "\t%s 40(AX), Y31, K5\n", op) + fmt.Fprintf(&source, "\t%s Z1, Z%d, K6\n", op, zLast) + fmt.Fprintf(&source, "\t%s 104(AX), Z%d, K0\n", op, zLast) + fmt.Fprintf(&source, "\t%s 0, Z%d, K0\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s 112(AX), X31, K1, K2\n", op) + fmt.Fprintf(&source, "\t%s 120(AX), Y31, K4, K5\n", op) + fmt.Fprintf(&source, "\t%s 128(AX), Z31, K7, K0\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST 168(AX), X31, K1\n", op) + fmt.Fprintf(&source, "\t%s.BCST 176(AX), Y31, K3\n", op) + fmt.Fprintf(&source, "\t%s.BCST 184(AX), Z%d, K4\n", op, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST 192(AX), Y31, K2, K3\n", op) + } + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedtestmaskforms": {Name: "packedtestmaskforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"icmp ne", "icmp eq", "store i64"} { + if !strings.Contains(ll, want) { + t.Fatalf("translation omitted %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-test-mask-"+target.name+".ll", "packed-test-mask-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedTestMaskRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPTESTMD X0, X1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, Y1, K1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, (AX), K1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, X1, K0, K1"}, + {goarch: "amd64", instruction: "VPTESTMD X0, X1, AX"}, + {goarch: "amd64", instruction: "VPTESTMD.Z X0, X1, K1"}, + {goarch: "amd64", instruction: "VPTESTMB.BCST (AX), X1, K1"}, + {goarch: "amd64", instruction: "VPTESTMW.BCST (AX), X1, K1"}, + {goarch: "amd64", instruction: "VPTESTMD.BCST X0, X1, K1"}, + {goarch: "amd64", instruction: "VPTESTNMQ X32, X1, K1"}, + {goarch: "386", instruction: "VPTESTMD X1, X2, K1, K2"}, + {goarch: "386", instruction: "VPTESTMD Z8, Z0, K1"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed test-mask forms for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_paddd_alias_test.go b/amd64_lower_paddd_alias_test.go new file mode 100644 index 000000000..897c2f902 --- /dev/null +++ b/amd64_lower_paddd_alias_test.go @@ -0,0 +1,199 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64PackedAddGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64PackedAddSpec{ + "PADDB": {canonical: "PADDB", form: amd64PackedAddLegacyYMM, laneBits: 8, mode: amd64PackedAddWrap}, + "PADDW": {canonical: "PADDW", form: amd64PackedAddLegacyYMM, laneBits: 16, mode: amd64PackedAddWrap}, + "PADDL": {canonical: "PADDL", form: amd64PackedAddLegacyYMM, laneBits: 32, mode: amd64PackedAddWrap}, + "PADDD": {canonical: "PADDL", form: amd64PackedAddLegacyYMM, laneBits: 32, mode: amd64PackedAddWrap}, + "PADDQ": {canonical: "PADDQ", form: amd64PackedAddLegacyYXM, laneBits: 64, mode: amd64PackedAddWrap}, + "PADDSB": {canonical: "PADDSB", form: amd64PackedAddLegacyYMM, laneBits: 8, mode: amd64PackedAddSignedSaturating}, + "PADDSW": {canonical: "PADDSW", form: amd64PackedAddLegacyYMM, laneBits: 16, mode: amd64PackedAddSignedSaturating}, + "PADDUSB": {canonical: "PADDUSB", form: amd64PackedAddLegacyYMM, laneBits: 8, mode: amd64PackedAddUnsignedSaturating}, + "PADDUSW": {canonical: "PADDUSW", form: amd64PackedAddLegacyYMM, laneBits: 16, mode: amd64PackedAddUnsignedSaturating}, + "VPADDB": {canonical: "VPADDB", form: amd64PackedAddVEXEVEX, laneBits: 8, mode: amd64PackedAddWrap}, + "VPADDW": {canonical: "VPADDW", form: amd64PackedAddVEXEVEX, laneBits: 16, mode: amd64PackedAddWrap}, + "VPADDD": {canonical: "VPADDD", form: amd64PackedAddVEXEVEX, laneBits: 32, mode: amd64PackedAddWrap, allowBroadcast: true}, + "VPADDQ": {canonical: "VPADDQ", form: amd64PackedAddVEXEVEX, laneBits: 64, mode: amd64PackedAddWrap, allowBroadcast: true}, + "VPADDSB": {canonical: "VPADDSB", form: amd64PackedAddVEXEVEX, laneBits: 8, mode: amd64PackedAddSignedSaturating}, + "VPADDSW": {canonical: "VPADDSW", form: amd64PackedAddVEXEVEX, laneBits: 16, mode: amd64PackedAddSignedSaturating}, + "VPADDUSB": {canonical: "VPADDUSB", form: amd64PackedAddVEXEVEX, laneBits: 8, mode: amd64PackedAddUnsignedSaturating}, + "VPADDUSW": {canonical: "VPADDUSW", form: amd64PackedAddVEXEVEX, laneBits: 16, mode: amd64PackedAddUnsignedSaturating}, + } + if len(amd64PackedAddSpecs) != len(expected) { + t.Fatalf("packed-add grammar has %d entries, want %d", len(amd64PackedAddSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64PackedAddSpecs[op]; !ok { + t.Errorf("packed-add grammar omitted %s", op) + } else if got != want { + t.Errorf("packed-add grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86PADDDGoAliasCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastX := 15 + if target.goarch == "386" { + lastX = 7 + } + var source strings.Builder + source.WriteString("TEXT padddaliasforms(SB),$0-0\n") + for _, op := range []string{"PADDL", "PADDD"} { + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s M0, M7\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), M7\n", op) + } + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, lastX) + fmt.Fprintf(&source, "\t%s 16(AX), X%d\n", op, lastX) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"padddaliasforms": {Name: "padddaliasforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "add <4 x i32>") { + t.Fatalf("PADDL/PADDD lowering omitted dword addition:\n%s", ir) + } + if target.goarch == "amd64" && !strings.Contains(ir, "bitcast i64") { + t.Fatalf("PADDL/PADDD lowering omitted MMX forms:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "paddd-alias-"+target.name+".ll", "paddd-alias-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PADDDGoAliasRejectsFormsOutsidePADDLTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PADDD M0, X1"}, + {goarch: "amd64", instruction: "PADDD X0, M1"}, + {goarch: "amd64", instruction: "PADDD Y0, Y1"}, + {goarch: "amd64", instruction: "PADDD.Z X0, X1"}, + {goarch: "386", instruction: "PADDD M0, M1"}, + {goarch: "386", instruction: "PADDL M0, M1"}, + {goarch: "386", instruction: "PADDD X0, X8"}, + {goarch: "386", instruction: "PADDD 8(R9), X0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside PADDD/PADDL's Go table for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64PADDDGoAliasRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT padddaliassemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ a+8(FP), M0 + MOVQ b+16(FP), M1 + STC + PADDD M0, M1 + MOVQ M1, 0(AX) + SETCS 8(AX) + EMMS + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "padddaliassemantics": { + Name: "padddaliassemantics", Args: []LLVMType{Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I64, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void padddaliassemantics(uint8_t *, uint64_t, uint64_t); +int main(void) { + uint8_t out[9] = {0}; + uint64_t a = UINT64_C(0xffffffff00000002); + uint64_t b = UINT64_C(0x0000000200000003); + padddaliassemantics(out, a, b); + uint64_t got; + memcpy(&got, out, 8); + if (got != UINT64_C(0x0000000100000005)) return 10; + if (out[8] != 1) return 11; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "paddd_alias_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_parallel_bits.go b/amd64_lower_parallel_bits.go new file mode 100644 index 000000000..c9a94b716 --- /dev/null +++ b/amd64_lower_parallel_bits.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ParallelBitSpec struct { + bits, prefix int + deposit bool +} + +var amd64ParallelBitSpecs = map[string]amd64ParallelBitSpec{ + "PDEPL": {bits: 32, prefix: 3, deposit: true}, + "PDEPQ": {bits: 64, prefix: 3, deposit: true}, + "PEXTL": {bits: 32, prefix: 2}, + "PEXTQ": {bits: 64, prefix: 2}, +} + +// lowerParallelBitDepositExtract implements the complete Go 1.27 PDEP/PEXT +// family. All four instructions share _yandnl's single Yml, Yrl, Yrl row. +func (c *amd64Ctx) lowerParallelBitDepositExtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, supported := amd64ParallelBitSpecs[baseOp] + if !supported { + return false, false, nil + } + bits := spec.bits + // Go accepts Q names on 386, but VEX.W is ignored in not64 mode. + // In particular, a memory mask must not accidentally read eight bytes. + if c.goarch == "386" { + bits = 32 + } + if rawOp != baseOp { + return true, false, fmt.Errorf("amd64 %s has no instruction suffixes in Go 1.27's optab: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("amd64 %s expects mask, source, destination: %q", baseOp, ins.Raw) + } + maskOperand := ins.Args[0] + if maskOperand.Kind == OpReg { + if !isX86YrlRegisterForArch(maskOperand.Reg, c.goarch) { + return true, false, fmt.Errorf("amd64 %s first operand is outside Go 1.27's Yml class: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(maskOperand) { + return true, false, fmt.Errorf("amd64 %s first operand is outside Go 1.27's Yml class: %q", baseOp, ins.Raw) + } + if ins.Args[1].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[1].Reg, c.goarch) { + return true, false, fmt.Errorf("amd64 %s second operand is outside Go 1.27's Yrl class: %q", baseOp, ins.Raw) + } + if ins.Args[2].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[2].Reg, c.goarch) { + return true, false, fmt.Errorf("amd64 %s destination is outside Go 1.27's Yrl class: %q", baseOp, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + mask, err := c.evalIntSized(maskOperand, typ) + if err != nil { + return true, false, err + } + source, err := c.evalIntSized(ins.Args[1], typ) + if err != nil { + return true, false, err + } + result := c.emitParallelBitOperation(spec.deposit, bits, mask, source) + return true, false, c.storeRegSized(ins.Args[2].Reg, typ, result) +} + +// emitParallelBitOperation emits target-independent LLVM so translated code +// retains PDEP/PEXT semantics even when the LLVM execution host lacks BMI2. +// cursor is at most bit before each iteration, so every variable shift stays +// strictly below the integer width and cannot produce LLVM poison. +func (c *amd64Ctx) emitParallelBitOperation(deposit bool, bits int, mask, source string) string { + typ := fmt.Sprintf("i%d", bits) + result := "0" + cursor := "0" + for bit := 0; bit < bits; bit++ { + shiftedMask := c.newTmp() + maskBit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %d\n", shiftedMask, typ, mask, bit) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", maskBit, typ, shiftedMask) + + sourceShift := cursor + destinationShift := fmt.Sprintf("%d", bit) + if !deposit { + sourceShift = fmt.Sprintf("%d", bit) + destinationShift = cursor + } + shiftedSource := c.newTmp() + sourceBit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", shiftedSource, typ, source, sourceShift) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", sourceBit, typ, shiftedSource) + placed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", placed, typ, sourceBit, destinationShift) + enabledMask := c.newTmp() + enabled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %%%s\n", enabledMask, typ, maskBit) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", enabled, typ, placed, enabledMask) + nextResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %s, %%%s\n", nextResult, typ, result, enabled) + result = "%" + nextResult + nextCursor := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add %s %s, %%%s\n", nextCursor, typ, cursor, maskBit) + cursor = "%" + nextCursor + } + return result +} diff --git a/amd64_lower_permute.go b/amd64_lower_permute.go new file mode 100644 index 000000000..7aa226b06 --- /dev/null +++ b/amd64_lower_permute.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerVariableDwordPermute implements the complete Go 1.27 _yvpermd +// operand family shared by VPERMD and VPERMPS. +func (c *amd64Ctx) lowerVariableDwordPermute(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + switch baseOp { + case "VPERMD", "VPERMPS": + default: + return false, false, nil + } + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvpermd encodings: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects data, indices, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + dstArg := ins.Args[len(ins.Args)-1] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be Y or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(dstArg.Reg) + if byteWidth != 32 && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s destination must be Y or Z: %q", c.goarch, baseOp, ins.Raw) + } + validRegister := func(arg Operand) bool { + if !amd64EVEXVectorRegister(arg, byteWidth) { + return false + } + if c.goarch == "386" && byteWidth == 64 { + index, _ := amd64VectorRegisterIndex(arg.Reg, byteWidth) + return index < 8 + } + return true + } + if !validRegister(dstArg) || !validRegister(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s indices and destination must be matching Y or Z registers: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind == OpReg { + if !validRegister(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s data register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s data must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if broadcast && !isAMD64MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory data source: %q", c.goarch, baseOp, ins.Raw) + } + + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + var mask string + if masked { + maskArg := ins.Args[2] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + index, ok := amd64ParseKReg(maskArg.Reg) + if !ok || index == 0 { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + evex := masked || suffix != "" || byteWidth == 64 || !amd64VEXVectorRegister(dstArg, byteWidth) || !amd64VEXVectorRegister(ins.Args[1], byteWidth) + if ins.Args[0].Kind == OpReg && !amd64VEXVectorRegister(ins.Args[0], byteWidth) { + evex = true + } + if !evex && byteWidth != 32 { + return true, false, fmt.Errorf("%s %s VEX form is Y-only in Go 1.27's _yvpermd table: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth / 4 + var data string + if broadcast { + scalar, err := c.evalIntSized(ins.Args[0], I32) + if err != nil { + return true, false, err + } + data = amd64SplatInteger(c, lanes, 32, scalar) + } else { + dataBytes, err := c.loadPackedCompareBytes(ins.Args[0], byteWidth) + if err != nil { + return true, false, err + } + data = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, 32, dataBytes) + } + indicesBytes, err := c.loadPackedCompareBytes(ins.Args[1], byteWidth) + if err != nil { + return true, false, err + } + indices := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, 32, indicesBytes) + result := c.emitVariableDwordPermute(data, indices, lanes) + if masked { + oldBytes, err := c.loadPackedCompareBytes(dstArg, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, 32, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, 32, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i32> %s to <%d x i8>\n", out, lanes, result, byteWidth) + return true, false, c.storeVectorBytes(dstArg.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitVariableDwordPermute(data, indices string, lanes int) string { + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + index := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i32> %s, i32 %d\n", index, lanes, indices, lane) + maskedIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, %d\n", maskedIndex, index, lanes-1) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i32> %s, i32 %%%s\n", value, lanes, data, maskedIndex) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i32> %s, i32 %%%s, i32 %d\n", inserted, lanes, result, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_permute128_test.go b/amd64_lower_permute128_test.go new file mode 100644 index 000000000..72ee240f6 --- /dev/null +++ b/amd64_lower_permute128_test.go @@ -0,0 +1,160 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestTranslateX86Permute128CompleteGoAssemblerForms(t *testing.T) { + const source = `TEXT permute128forms(SB),$0-0 + VPERM2F128 $0, Y0, Y1, Y2 + VPERM2F128 $255, 8(AX), Y15, Y15 + VPERM2I128 $0, Y0, Y1, Y2 + VPERM2I128 $255, 8(AX), Y15, Y15 + RET +` + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "permute128forms": {Name: "permute128forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "permute-128-"+target.name+".ll", "permute-128-"+target.name+".o", ll) + }) + } + requireX86GoAssemblerResult(t, "386", source, false) +} + +func TestTranslateX86Permute128RejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERM2F128 $-1, Y0, Y1, Y2", + "VPERM2F128 $256, Y0, Y1, Y2", + "VPERM2I128 $-1, Y0, Y1, Y2", + "VPERM2I128 $256, Y0, Y1, Y2", + "VPERM2I128 $1, X0, Y1, Y2", + "VPERM2I128 $1, Y0, 8(AX), Y2", + "VPERM2I128 $1, Y0, X1, Y2", + "VPERM2I128 $1, Y0, Y1, X2", + "VPERM2I128.Z $1, Y0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86Permute128Rejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86Permute128Rejected(t, "386", "i386-unknown-linux-gnu", "VPERM2I128 $1, Y0, Y1, Y2") +} + +func assertX86Permute128Rejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPERM2F128/VPERM2I128 table for %s", instruction, goarch) + } +} + +func TestAMD64Permute128RuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT permute128integer(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + VMOVDQU (BX), Y0 + VMOVDQU (CX), Y1 + VPERM2I128 $0x21, Y0, Y1, Y2 + VMOVDQU Y2, (AX) + VZEROUPPER + RET +TEXT permute128float(SB),NOSPLIT,$0-24 + MOVQ out+0(FP), AX + MOVQ first+8(FP), BX + MOVQ second+16(FP), CX + VMOVDQU (BX), Y0 + VMOVDQU (CX), Y1 + VPERM2F128 $0x12, Y0, Y1, Y2 + VMOVDQU Y2, (AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "permute128integer": {Name: "permute128integer", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + "permute128float": {Name: "permute128float", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void permute128integer(uint8_t *, const uint8_t *, const uint8_t *); +extern void permute128float(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t first[32], second[32], out[32]; + for (int i = 0; i < 32; i++) { first[i] = (uint8_t)i; second[i] = (uint8_t)(100+i); out[i] = 0; } + permute128integer(out, first, second); + for (int i = 0; i < 16; i++) if (out[i] != second[i+16]) return 10+i; + for (int i = 16; i < 32; i++) if (out[i] != first[i-16]) return 50+i; + permute128float(out, first, second); + for (int i = 0; i < 16; i++) if (out[i] != first[i]) return 90+i; + for (int i = 16; i < 32; i++) if (out[i] != second[i]) return 130+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "permute_128", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_permute_index_family_test.go b/amd64_lower_permute_index_family_test.go new file mode 100644 index 000000000..7fbb0ec0f --- /dev/null +++ b/amd64_lower_permute_index_family_test.go @@ -0,0 +1,274 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64IndexedPermuteGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64IndexedPermuteSpec{ + "VPERMB": {laneBits: 8, mode: amd64IndexedPermuteSingle}, + "VPERMW": {laneBits: 16, mode: amd64IndexedPermuteSingle}, + "VPERMI2B": {laneBits: 8, mode: amd64IndexedPermuteI2}, + "VPERMI2W": {laneBits: 16, mode: amd64IndexedPermuteI2}, + "VPERMI2D": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PS": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2Q": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PD": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMT2B": {laneBits: 8, mode: amd64IndexedPermuteT2}, + "VPERMT2W": {laneBits: 16, mode: amd64IndexedPermuteT2}, + "VPERMT2D": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PS": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2Q": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PD": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, + } + if len(amd64IndexedPermuteSpecs) != len(expected) { + t.Fatalf("indexed-permute grammar has %d entries, want %d", len(amd64IndexedPermuteSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64IndexedPermuteSpecs[op]; !ok { + t.Errorf("indexed-permute grammar omitted %s", op) + } else if got != want { + t.Errorf("indexed-permute grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86IndexedPermuteCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT indexedpermuteforms(SB),$0-0\n") + for _, family := range []struct { + op string + broadcast bool + }{ + {op: "VPERMB"}, {op: "VPERMW"}, + {op: "VPERMI2B"}, {op: "VPERMI2W"}, + {op: "VPERMI2D", broadcast: true}, {op: "VPERMI2Q", broadcast: true}, + {op: "VPERMI2PS", broadcast: true}, {op: "VPERMI2PD", broadcast: true}, + {op: "VPERMT2B"}, {op: "VPERMT2W"}, + {op: "VPERMT2D", broadcast: true}, {op: "VPERMT2Q", broadcast: true}, + {op: "VPERMT2PS", broadcast: true}, {op: "VPERMT2PD", broadcast: true}, + } { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s1, %s2, %s%d\n", family.op, width, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s2, %s%d\n", family.op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s1, %s2, K1, %s%d\n", family.op, width, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 16(AX), %s2, K2, %s%d\n", family.op, width, width, last) + } + if family.broadcast { + fmt.Fprintf(&source, "\t%s.BCST 24(AX), %s2, %s%d\n", family.op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 32(AX), %s2, K3, %s%d\n", family.op, width, width, last) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"indexedpermuteforms": {Name: "indexedpermuteforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "indexed-permute-"+target.name+".ll", "indexed-permute-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86IndexedPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPERMB X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VPERMW.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VPERMT2B.BCST 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VPERMT2D.BCST X0, X1, X2"}, + {goarch: "amd64", instruction: "VPERMT2D.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VPERMI2Q X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VPERMT2PS X0, X1, 0(AX)"}, + {goarch: "amd64", instruction: "VPERMI2D.Z.BCST 0(AX), X1, K1, X2"}, + {goarch: "amd64", instruction: "VPERMB X0, X1"}, + {goarch: "386", instruction: "VPERMT2D X0, X1, K1, X2"}, + {goarch: "386", instruction: "VPERMW Z0, Z1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's indexed-permute table", test.instruction) + } + }) + } +} + +func TestAMD64IndexedPermuteRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT indexedpermutesemantics(SB),$0-48 + MOVQ out+0(FP), AX + MOVQ tableA+8(FP), BX + MOVQ tableB+16(FP), CX + MOVQ indices+24(FP), DX + MOVQ old+32(FP), R8 + MOVQ mask+40(FP), R9 + KMOVQ R9, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VMOVDQU64 0(DX), Z2 + VMOVDQU64 0(R8), Z3 + STC + VPERMB Z0, Z2, Z4 + VMOVDQU64 Z4, 0(AX) + VPERMW Z0, Z2, Z4 + VMOVDQU64 Z4, 64(AX) + VPERMI2B Z0, Z1, Z2 + VMOVDQU64 Z2, 128(AX) + VMOVDQU64 0(DX), Z2 + VPERMT2B Z0, Z2, Z3 + VMOVDQU64 Z3, 192(AX) + VMOVDQU64 0(R8), Z3 + VPERMB Z0, Z2, K1, Z3 + VMOVDQU64 Z3, 256(AX) + VMOVDQU64 0(R8), Z3 + VPERMT2W.Z Z0, Z2, K1, Z3 + VMOVDQU64 Z3, 320(AX) + VMOVDQU64 0(DX), Z2 + VPERMI2Q.BCST 0(BX), Z1, K1, Z2 + VMOVDQU64 Z2, 384(AX) + VMOVDQU64 0(DX), Z2 + VMOVDQU64 0(R8), Z3 + VPERMT2D.BCST 0(BX), Z2, K1, Z3 + VMOVDQU64 Z3, 448(AX) + SETCS 512(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"indexedpermutesemantics": { + Name: "indexedpermutesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void indexedpermutesemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint16_t u16(const uint8_t *p) { uint16_t v; memcpy(&v,p,2); return v; } +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint64_t a64[8], b64[8], indices64[8], old64[8], output64[65] = {0}; + uint8_t *a=(uint8_t *)a64, *b=(uint8_t *)b64, *indices=(uint8_t *)indices64; + uint8_t *old=(uint8_t *)old64, *out=(uint8_t *)output64; + const uint64_t mask=UINT64_C(0xa55af00ff00fa55a); + for (int i=0;i<64;i++) { + a[i]=(uint8_t)(0x11u+3u*(unsigned)i); + b[i]=(uint8_t)(0x80u+5u*(unsigned)i); + indices[i]=(uint8_t)(7u+13u*(unsigned)i); + old[i]=(uint8_t)(0xe0u-(unsigned)i); + } + indexedpermutesemantics(out,a,b,indices,old,mask); + for (int i=0;i<64;i++) { + unsigned index=indices[i]&63u; + if (out[i]!=a[index]) return 10; + unsigned two=indices[i]&127u; + uint8_t want=two<64u ? b[two] : a[two-64u]; + if (out[128+i]!=want) return 11; + want=two<64u ? old[two] : a[two-64u]; + if (out[192+i]!=want) return 12; + want=(mask>>i)&1u ? a[index] : old[i]; + if (out[256+i]!=want) return 13; + } + for (int i=0;i<32;i++) { + unsigned index=u16(indices+2*i)&31u; + if (u16(out+64+2*i)!=u16(a+2*index)) return 20; + unsigned two=u16(indices+2*i)&63u; + uint16_t want=two<32u ? u16(old+2*two) : u16(a+2*(two-32u)); + if (u16(out+320+2*i)!=(((mask>>i)&1u)?want:0)) return 21; + } + for (int i=0;i<8;i++) { + unsigned index=(unsigned)(u64(indices+8*i)&15u); + uint64_t value=index<8u ? u64(b+8*index) : u64(a); + uint64_t want=(mask>>i)&1u ? value : u64(indices+8*i); + if (u64(out+384+8*i)!=want) return 30; + } + for (int i=0;i<16;i++) { + unsigned index=u32(indices+4*i)&31u; + uint32_t value=index<16u ? u32(old+4*index) : u32(a); + uint32_t want=(mask>>i)&1u ? value : u32(old+4*i); + if (u32(out+448+4*i)!=want) return 31; + } + if (out[512]!=1) return 40; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "indexed_permute_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_permute_inlane.go b/amd64_lower_permute_inlane.go new file mode 100644 index 000000000..d7b48b61f --- /dev/null +++ b/amd64_lower_permute_inlane.go @@ -0,0 +1,218 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerInLaneFloatingPermute implements the complete Go 1.27 _yvpermilpd +// operand family shared by VPERMILPD and VPERMILPS. The permutation preserves +// element bits and never selects across a 128-bit lane. +func (c *amd64Ctx) lowerInLaneFloatingPermute(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + laneBits := 0 + switch baseOp { + case "VPERMILPD": + laneBits = 64 + case "VPERMILPS": + laneBits = 32 + default: + return false, false, nil + } + + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvpermilpd encodings: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects control/data, data, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + immediate := ins.Args[0].Kind == OpImm + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoInLanePermuteRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's vector-register class: %q", c.goarch, baseOp, ins.Raw) + } + + rmArg := ins.Args[0] + dataArg := ins.Args[1] + var imm uint8 + if immediate { + rmArg = ins.Args[1] + value := ins.Args[0].Imm + if value < -128 || value > 255 { + return true, false, fmt.Errorf("%s %s immediate is outside Go 1.27's signed-or-unsigned imm8 classes: %q", c.goarch, baseOp, ins.Raw) + } + imm = uint8(value) + if value < 0 { + vex := !masked && suffix == "" && byteWidth != 64 && amd64VEXVectorRegister(destination, byteWidth) + if rmArg.Kind == OpReg { + vex = vex && amd64VEXVectorRegister(rmArg, byteWidth) + } + if !vex { + return true, false, fmt.Errorf("%s %s signed imm8 exists only in its VEX X/Y rows: %q", c.goarch, baseOp, ins.Raw) + } + } + } else if !c.isGoInLanePermuteRegister(dataArg, byteWidth) { + return true, false, fmt.Errorf("%s %s data source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + + if broadcast { + if !isAMD64MemoryOperand(rmArg) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory r/m source: %q", c.goarch, baseOp, ins.Raw) + } + } else if rmArg.Kind == OpReg { + if !c.isGoInLanePermuteRegister(rmArg, byteWidth) { + return true, false, fmt.Errorf("%s %s r/m register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(rmArg) { + return true, false, fmt.Errorf("%s %s r/m source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth * 8 / laneBits + var data, control string + if immediate { + if broadcast { + scalar, err := c.evalIntSized(rmArg, amd64IntegerTypeForBits(laneBits)) + if err != nil { + return true, false, err + } + data = amd64SplatInteger(c, lanes, laneBits, scalar) + } else { + bytes, err := c.loadPackedCompareBytes(rmArg, byteWidth) + if err != nil { + return true, false, err + } + data = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, bytes) + } + } else { + dataBytes, err := c.loadPackedCompareBytes(dataArg, byteWidth) + if err != nil { + return true, false, err + } + data = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, dataBytes) + if broadcast { + scalar, err := c.evalIntSized(rmArg, amd64IntegerTypeForBits(laneBits)) + if err != nil { + return true, false, err + } + control = amd64SplatInteger(c, lanes, laneBits, scalar) + } else { + controlBytes, err := c.loadPackedCompareBytes(rmArg, byteWidth) + if err != nil { + return true, false, err + } + control = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, controlBytes) + } + } + + var result string + if immediate { + result = c.emitImmediateInLanePermute(data, lanes, laneBits, imm) + } else { + result = c.emitVariableInLanePermute(data, control, lanes, laneBits) + } + if masked { + mask, err := c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) isGoInLanePermuteRegister(arg Operand, byteWidth int) bool { + if !amd64EVEXVectorRegister(arg, byteWidth) { + return false + } + if c.goarch == "386" && byteWidth == 64 { + index, _ := amd64VectorRegisterIndex(arg.Reg, byteWidth) + return index < 8 + } + return true +} + +func (c *amd64Ctx) emitImmediateInLanePermute(data string, lanes, laneBits int, imm uint8) string { + result := "poison" + for lane := 0; lane < lanes; lane++ { + var sourceLane int + if laneBits == 32 { + groupBase := lane / 4 * 4 + sourceLane = groupBase + int(imm>>uint(2*(lane%4)))&3 + } else { + groupBase := lane / 2 * 2 + sourceLane = groupBase + int(imm>>uint(lane))&1 + } + value := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, lanes, laneBits, data, sourceLane) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, value, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitVariableInLanePermute(data, control string, lanes, laneBits int) string { + result := "poison" + groupLanes := 128 / laneBits + for lane := 0; lane < lanes; lane++ { + controlLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", controlLane, lanes, laneBits, control, lane) + selectorValue := "%" + controlLane + if laneBits == 64 { + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, 1\n", shifted, controlLane) + selectorValue = "%" + shifted + } + selector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", selector, laneBits, selectorValue, groupLanes-1) + index := "%" + selector + groupBase := lane / groupLanes * groupLanes + if groupBase != 0 { + withBase := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %d\n", withBase, laneBits, selector, groupBase) + index = "%" + withBase + } + value := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i%d %s\n", value, lanes, laneBits, data, laneBits, index) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_permute_inlane_test.go b/amd64_lower_permute_inlane_test.go new file mode 100644 index 000000000..e3942cafa --- /dev/null +++ b/amd64_lower_permute_inlane_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86InLaneFloatingPermuteCompleteGoAssemblerForms(t *testing.T) { + // VPERMILPD and VPERMILPS share Go 1.27's 18-row _yvpermilpd table: + // immediate and variable-control VEX X/Y forms, EVEX X/Y/Z forms, + // scalar-memory broadcast, and K1-K7 merge/zero masking. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + zLast := 21 + if target.goarch == "386" { + zLast = 7 + } + var src strings.Builder + src.WriteString("TEXT inlanefloatingpermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMILPD", "VPERMILPS"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\t%s $-128, %s0, %s1\n", op, width, width) + fmt.Fprintf(&src, "\t%s $255, (AX), %s2\n", op, width) + fmt.Fprintf(&src, "\t%s %s0, %s1, %s2\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s (AX), %s20, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s $7, %s20, %s21\n", op, width, width) + } + fmt.Fprintf(&src, "\t%s $255, Z0, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s Z0, Z6, Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST $7, (AX), Z%d\n", op, zLast) + fmt.Fprintf(&src, "\t%s.BCST (AX), Z6, Z%d\n", op, zLast) + if target.goarch == "amd64" { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\t%s $7, %s20, K1, %s21\n", op, width, width) + fmt.Fprintf(&src, "\t%s.Z $7, (AX), K2, %s22\n", op, width) + fmt.Fprintf(&src, "\t%s %s20, %s21, K3, %s22\n", op, width, width, width) + fmt.Fprintf(&src, "\t%s.BCST.Z (AX), %s21, K4, %s22\n", op, width, width) + } + } + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "inlanefloatingpermuteforms": {Name: "inlanefloatingpermuteforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "in-lane-floating-permute-"+target.name+".ll", "in-lane-floating-permute-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86InLaneFloatingPermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMILPS $-129, X0, X1", + "VPERMILPD $256, X0, X1", + "VPERMILPS $-1, X20, X21", + "VPERMILPD $-1, Z0, Z1", + "VPERMILPS X0, X1", + "VPERMILPD X0, Y1, Y2", + "VPERMILPS X0, X1, (AX)", + "VPERMILPD.BCST X0, X1, X2", + "VPERMILPS.BCST $7, X0, X1", + "VPERMILPD.Z $7, X0, X1", + "VPERMILPS X0, X1, K0, X2", + "VPERMILPD.SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86InLaneFloatingPermuteRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "VPERMILPS $7, Z8, Z0", + "VPERMILPD Z0, Z1, Z8", + "VPERMILPS $-1, X20, X21", + "VPERMILPD X0, X1, K1, X2", + "VPERMILPS.Z $7, X0, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86InLaneFloatingPermuteRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86InLaneFloatingPermuteRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvpermilpd forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_permute_qword.go b/amd64_lower_permute_qword.go new file mode 100644 index 000000000..d726af939 --- /dev/null +++ b/amd64_lower_permute_qword.go @@ -0,0 +1,180 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerQwordPermute implements the complete Go 1.27 _yvpermq operand family +// shared by VPERMQ and VPERMPD. It includes immediate-control VEX/EVEX forms +// and variable-control EVEX forms; both preserve qword bits. +func (c *amd64Ctx) lowerQwordPermute(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + switch baseOp { + case "VPERMQ", "VPERMPD": + default: + return false, false, nil + } + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's _yvpermq encodings: %q", c.goarch, baseOp, ins.Raw) + } + immediate := len(ins.Args) != 0 && ins.Args[0].Kind == OpImm + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects immediate/data, control, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + dstArg := ins.Args[len(ins.Args)-1] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be Y or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(dstArg.Reg) + if byteWidth != 32 && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s destination must be Y or Z: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoQwordPermuteVector(dstArg, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's vector range: %q", c.goarch, baseOp, ins.Raw) + } + + dataArg := ins.Args[0] + controlArg := ins.Args[1] + var imm uint8 + if immediate { + dataArg = ins.Args[1] + value := ins.Args[0].Imm + if value < -128 || value > 255 { + return true, false, fmt.Errorf("%s %s immediate is outside Go 1.27's signed-or-unsigned imm8 classes: %q", c.goarch, baseOp, ins.Raw) + } + imm = uint8(value) + if value < 0 { + if masked || suffix != "" || byteWidth != 32 || !amd64VEXVectorRegister(dstArg, byteWidth) { + return true, false, fmt.Errorf("%s %s signed imm8 exists only in the VEX Y form: %q", c.goarch, baseOp, ins.Raw) + } + if dataArg.Kind == OpReg && !amd64VEXVectorRegister(dataArg, byteWidth) { + return true, false, fmt.Errorf("%s %s signed imm8 source is outside the VEX Y range: %q", c.goarch, baseOp, ins.Raw) + } + } + } else if !c.isGoQwordPermuteVector(controlArg, byteWidth) { + return true, false, fmt.Errorf("%s %s variable control and destination must be matching Y or Z registers: %q", c.goarch, baseOp, ins.Raw) + } + + if broadcast { + if !isAMD64MemoryOperand(dataArg) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory data source: %q", c.goarch, baseOp, ins.Raw) + } + } else if dataArg.Kind == OpReg { + if !c.isGoQwordPermuteVector(dataArg, byteWidth) { + return true, false, fmt.Errorf("%s %s data register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(dataArg) { + return true, false, fmt.Errorf("%s %s data must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + var mask string + if masked { + if !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + lanes := byteWidth / 8 + var data string + if broadcast { + scalar, err := c.evalIntSized(dataArg, I64) + if err != nil { + return true, false, err + } + data = amd64SplatInteger(c, lanes, 64, scalar) + } else { + dataBytes, err := c.loadPackedCompareBytes(dataArg, byteWidth) + if err != nil { + return true, false, err + } + data = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, 64, dataBytes) + } + var result string + if immediate { + result = c.emitImmediateQwordPermute(data, lanes, imm) + } else { + controlBytes, err := c.loadPackedCompareBytes(controlArg, byteWidth) + if err != nil { + return true, false, err + } + control := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, 64, controlBytes) + result = c.emitVariableQwordPermute(data, control, lanes) + } + if masked { + oldBytes, err := c.loadPackedCompareBytes(dstArg, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, 64, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, 64, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", out, lanes, result, byteWidth) + return true, false, c.storeVectorBytes(dstArg.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) isGoQwordPermuteVector(arg Operand, byteWidth int) bool { + if !amd64EVEXVectorRegister(arg, byteWidth) { + return false + } + if c.goarch == "386" && byteWidth == 64 { + index, _ := amd64VectorRegisterIndex(arg.Reg, byteWidth) + return index < 8 + } + return true +} + +func (c *amd64Ctx) emitImmediateQwordPermute(data string, lanes int, imm uint8) string { + result := "poison" + for lane := 0; lane < lanes; lane++ { + groupBase := lane / 4 * 4 + selector := int(imm>>uint(2*(lane%4))) & 3 + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", value, lanes, data, groupBase+selector) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, lanes, result, value, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitVariableQwordPermute(data, control string, lanes int) string { + result := "poison" + for lane := 0; lane < lanes; lane++ { + index := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", index, lanes, control, lane) + maskedIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %d\n", maskedIndex, index, lanes-1) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i64 %%%s\n", value, lanes, data, maskedIndex) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, lanes, result, value, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_permute_two_source.go b/amd64_lower_permute_two_source.go new file mode 100644 index 000000000..66b7c3e07 --- /dev/null +++ b/amd64_lower_permute_two_source.go @@ -0,0 +1,238 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64IndexedPermuteMode uint8 + +const ( + amd64IndexedPermuteSingle amd64IndexedPermuteMode = iota + amd64IndexedPermuteI2 + amd64IndexedPermuteT2 +) + +type amd64IndexedPermuteSpec struct { + laneBits int + mode amd64IndexedPermuteMode + broadcast bool +} + +// amd64IndexedPermuteSpecs models the complete Go 1.27 _yvblendmpd grammar +// for the single-table VPERMB/W and both two-table VPERMI2*/VPERMT2* +// families. Broadcast capability is an opcode property, not a consequence of +// lane width alone. +var amd64IndexedPermuteSpecs = map[Op]amd64IndexedPermuteSpec{ + "VPERMB": {laneBits: 8, mode: amd64IndexedPermuteSingle}, + "VPERMW": {laneBits: 16, mode: amd64IndexedPermuteSingle}, + "VPERMI2B": {laneBits: 8, mode: amd64IndexedPermuteI2}, + "VPERMI2W": {laneBits: 16, mode: amd64IndexedPermuteI2}, + "VPERMI2D": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PS": {laneBits: 32, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2Q": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMI2PD": {laneBits: 64, mode: amd64IndexedPermuteI2, broadcast: true}, + "VPERMT2B": {laneBits: 8, mode: amd64IndexedPermuteT2}, + "VPERMT2W": {laneBits: 16, mode: amd64IndexedPermuteT2}, + "VPERMT2D": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PS": {laneBits: 32, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2Q": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, + "VPERMT2PD": {laneBits: 64, mode: amd64IndexedPermuteT2, broadcast: true}, +} + +type amd64TwoSourcePermuteSuffixes struct { + broadcast bool + zeroing bool +} + +func amd64ParseTwoSourcePermuteSuffixes(rawOp, baseOp string) (amd64TwoSourcePermuteSuffixes, error) { + var suffixes amd64TwoSourcePermuteSuffixes + dot := strings.IndexByte(rawOp, '.') + if dot < 0 { + return suffixes, nil + } + parts := strings.Split(rawOp[dot+1:], ".") + for index, part := range parts { + switch part { + case "BCST": + if suffixes.broadcast || suffixes.zeroing || index != 0 { + return suffixes, fmt.Errorf("amd64 %s has invalid or duplicate .BCST suffix", baseOp) + } + suffixes.broadcast = true + case "Z": + if suffixes.zeroing || index != len(parts)-1 { + return suffixes, fmt.Errorf("amd64 %s has invalid or duplicate .Z suffix", baseOp) + } + suffixes.zeroing = true + default: + return suffixes, fmt.Errorf("amd64 %s has unsupported suffix .%s", baseOp, part) + } + } + return suffixes, nil +} + +// lowerIndexedPermute implements Go 1.27's complete single- and two-table +// indexed-permute family. +func (c *amd64Ctx) lowerIndexedPermute(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, ok := amd64IndexedPermuteSpecs[Op(baseOp)] + if !ok { + return false, false, nil + } + suffixes, err := amd64ParseTwoSourcePermuteSuffixes(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + if suffixes.broadcast && !spec.broadcast { + return true, false, fmt.Errorf("amd64 %s does not enable broadcast in Go 1.27: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("amd64 %s expects first source, second source, [K mask,] destination: %q", baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if masked && c.goarch == "386" { + return true, false, fmt.Errorf("386 %s masked form exceeds the Go assembler frontend's operand limit: %q", baseOp, ins.Raw) + } + if suffixes.zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s .Z requires a K1-K7 mask: %q", baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + byteWidth := 0 + if destination.Kind == OpReg { + byteWidth = amd64VectorByteWidth(destination.Reg) + } + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 %s expects an in-range X, Y, or Z destination: %q", baseOp, ins.Raw) + } + second := ins.Args[1] + if !c.isGoPackedVectorMoveRegister(second, byteWidth) { + return true, false, fmt.Errorf("amd64 %s second source must match the destination width: %q", baseOp, ins.Raw) + } + first := ins.Args[0] + if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("amd64 %s first source register must match the destination width: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s first source must be a matching vector register or memory: %q", baseOp, ins.Raw) + } + if suffixes.broadcast && !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory first source: %q", baseOp, ins.Raw) + } + + mask := "" + if masked { + maskArg := ins.Args[2] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + maskIndex, validMask := amd64ParseKReg(maskArg.Reg) + if !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + lanes := byteWidth * 8 / spec.laneBits + firstValue, err := c.loadPackedCompareLanes(first, byteWidth, spec.laneBits, suffixes.broadcast) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + computed := "" + switch spec.mode { + case amd64IndexedPermuteSingle: + // Plan 9 lists Intel's r/m table first and its index vector second. + computed = c.emitSingleTablePermute(lanes, spec.laneBits, firstValue, secondValue) + case amd64IndexedPermuteI2: + // VPERMI2 gets indices from the old destination. Intel calls the + // Plan 9 second operand table 0 and the first operand table 1. + computed = c.emitTwoTablePermute(lanes, spec.laneBits, secondValue, firstValue, old) + case amd64IndexedPermuteT2: + // VPERMT2 gets indices from the Plan 9 second operand. The old + // destination is table 0 and the Plan 9 first operand is table 1. + computed = c.emitTwoTablePermute(lanes, spec.laneBits, old, firstValue, secondValue) + } + if masked { + computed = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, computed, old, mask, suffixes.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, computed, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitSingleTablePermute(lanes, laneBits int, table, indices string) string { + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + indexValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", indexValue, lanes, laneBits, indices, lane) + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", bounded, laneBits, indexValue, lanes-1) + indexI32 := "%" + bounded + if laneBits < 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %%%s to i32\n", converted, laneBits, bounded) + indexI32 = "%" + converted + } else if laneBits > 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %%%s to i32\n", converted, laneBits, bounded) + indexI32 = "%" + converted + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %s\n", selected, lanes, laneBits, table, indexI32) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, selected, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitTwoTablePermute(lanes, laneBits int, table0, table1, indices string) string { + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + indexValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", indexValue, lanes, laneBits, indices, lane) + bounded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", bounded, laneBits, indexValue, 2*lanes-1) + useTable1 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge i%d %%%s, %d\n", useTable1, laneBits, bounded, lanes) + withinTable := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", withinTable, laneBits, bounded, lanes-1) + indexI32 := "%" + withinTable + if laneBits < 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %%%s to i32\n", converted, laneBits, withinTable) + indexI32 = "%" + converted + } else if laneBits > 32 { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i%d %%%s to i32\n", converted, laneBits, withinTable) + indexI32 = "%" + converted + } + fromTable0 := c.newTmp() + fromTable1 := c.newTmp() + selected := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %s\n", fromTable0, lanes, laneBits, table0, indexI32) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %s\n", fromTable1, lanes, laneBits, table1, indexI32) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", selected, useTable1, laneBits, fromTable1, laneBits, fromTable0) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, laneBits, result, laneBits, selected, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_permute_two_source_test.go b/amd64_lower_permute_two_source_test.go new file mode 100644 index 000000000..f81d75b07 --- /dev/null +++ b/amd64_lower_permute_two_source_test.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86TwoSourcePermuteCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 shares _yvblendmpd across VPERMI2{B,W,D,Q,PS,PD}: matching + // X/Y/Z sources and destination, optional K1-K7 masking, and .Z. The + // D/Q/PS/PD forms also accept .BCST on a memory first source. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT twosourcepermuteforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPERMI2B", "VPERMI2W", "VPERMI2D", "VPERMI2Q", "VPERMI2PS", "VPERMI2PD"} { + fmt.Fprintf(&source, "\t%s X1, X20, X21\n", op) + fmt.Fprintf(&source, "\t%s 8(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s Z1, Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K1, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z Y1, Y20, K2, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), Z20, K3, Z21\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") || strings.HasSuffix(op, "PS") || strings.HasSuffix(op, "PD") { + fmt.Fprintf(&source, "\t%s.BCST 104(AX), X20, X21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 112(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s.BCST 120(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 128(AX), Z20, K4, Z21\n", op) + } + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "twosourcepermuteforms": {Name: "twosourcepermuteforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "two-source-permute-"+target.name+".ll", "two-source-permute-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86TwoSourcePermuteRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPERMI2D X0, X1", + "VPERMI2D X0, Y1, Y2", + "VPERMI2D X0, (AX), X2", + "VPERMI2D X0, X1, K0, X2", + "VPERMI2D X0, X1, AX", + "VPERMI2D.Z X0, X1, X2", + "VPERMI2D.BCST X0, X1, X2", + "VPERMI2B.BCST (AX), X1, X2", + "VPERMI2W.BCST (AX), X1, X2", + "VPERMI2D.Z.BCST (AX), X1, K1, X2", + "VPERMI2D.RN_SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86TwoSourcePermuteRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86TwoSourcePermuteRejected(t, "386", "i386-unknown-linux-gnu", "VPERMI2D X0, X1, K1, X2") + assertX86TwoSourcePermuteRejected(t, "386", "i386-unknown-linux-gnu", "VPERMI2D Z0, Z1, Z8") +} + +func assertX86TwoSourcePermuteRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VPERMI2 forms", instruction) + } +} diff --git a/amd64_lower_popcount_vector.go b/amd64_lower_popcount_vector.go new file mode 100644 index 000000000..fd08c8f85 --- /dev/null +++ b/amd64_lower_popcount_vector.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedBitCountKind uint8 + +const ( + amd64PackedPopulationCount amd64PackedBitCountKind = iota + amd64PackedLeadingZeroCount +) + +type amd64PackedBitCountSpec struct { + laneBits int + kind amd64PackedBitCountKind + broadcast bool +} + +// amd64PackedBitCountSpecs models the complete Go 1.27 _yvexpandpd grammar +// for packed population-count and leading-zero-count instructions. +var amd64PackedBitCountSpecs = map[Op]amd64PackedBitCountSpec{ + "VPOPCNTB": {laneBits: 8, kind: amd64PackedPopulationCount}, + "VPOPCNTW": {laneBits: 16, kind: amd64PackedPopulationCount}, + "VPOPCNTD": {laneBits: 32, kind: amd64PackedPopulationCount, broadcast: true}, + "VPOPCNTQ": {laneBits: 64, kind: amd64PackedPopulationCount, broadcast: true}, + "VPLZCNTD": {laneBits: 32, kind: amd64PackedLeadingZeroCount, broadcast: true}, + "VPLZCNTQ": {laneBits: 64, kind: amd64PackedLeadingZeroCount, broadcast: true}, +} + +// lowerPackedBitCount implements X/Y/Z register or memory sources, K1-K7 +// merge and .Z, and each opcode's scalar-memory broadcast forms. +func (c *amd64Ctx) lowerPackedBitCount(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, ok := amd64PackedBitCountSpecs[Op(baseOp)] + if !ok { + return false, false, nil + } + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("%s %s has a suffix outside Go 1.27's _yvexpandpd form: %q", c.goarch, baseOp, ins.Raw) + } + if broadcast && !spec.broadcast { + return true, false, fmt.Errorf("%s %s has no broadcast form: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects source, [K1-K7,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s requires an X, Y, or Z destination: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth == 0 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go's EVEX register range: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) && !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must be matching X/Y/Z register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + mask := "" + if masked { + maskArg := ins.Args[1] + maskIndex, valid := amd64ParseKReg(maskArg.Reg) + if maskArg.Kind != OpReg || !valid || maskIndex == 0 { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + lanes := byteWidth * 8 / spec.laneBits + sourceValue, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, broadcast) + if err != nil { + return true, false, err + } + computed := c.newTmp() + if spec.kind == amd64PackedLeadingZeroCount { + fmt.Fprintf(c.b, " %%%s = call <%d x i%d> @llvm.ctlz.v%di%d(<%d x i%d> %s, i1 false)\n", + computed, lanes, spec.laneBits, lanes, spec.laneBits, lanes, spec.laneBits, sourceValue) + } else { + fmt.Fprintf(c.b, " %%%s = call <%d x i%d> @llvm.ctpop.v%di%d(<%d x i%d> %s)\n", + computed, lanes, spec.laneBits, lanes, spec.laneBits, lanes, spec.laneBits, sourceValue) + } + result := "%" + computed + if masked { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} diff --git a/amd64_lower_popcount_vector_test.go b/amd64_lower_popcount_vector_test.go new file mode 100644 index 000000000..3da7e45a9 --- /dev/null +++ b/amd64_lower_popcount_vector_test.go @@ -0,0 +1,210 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86PackedPopcountCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedpopcountforms(SB),$0-0\n") + for _, op := range []string{"VPOPCNTB", "VPOPCNTW", "VPOPCNTD", "VPOPCNTQ"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 22 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s0, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s %s0, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 8(AX), K7, %s%d\n", op, width, last) + if op == "VPOPCNTD" || op == "VPOPCNTQ" { + fmt.Fprintf(&source, "\t%s.BCST 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 8(AX), K2, %s%d\n", op, width, last) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedpopcountforms": {Name: "packedpopcountforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-popcount-"+target.name+".ll", "packed-popcount-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedPopcountRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VPOPCNTB.BCST 8(AX), X1", + "VPOPCNTW.BCST 8(AX), Z1", + "VPOPCNTD.BCST X0, X1", + "VPOPCNTQ X0, Y1", + "VPOPCNTB X0, K0, X1", + "VPOPCNTW.Z X0, X1", + "VPOPCNTD.SAE X0, X1", + "VPOPCNTQ X0, 8(AX)", + "VPOPCNTB AX, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedPopcountRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VPOPCNTD Z8, K1, Z0", + "VPOPCNTQ Z0, Z8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + if currentGoMinorAtLeast(27) { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } + assertX86PackedPopcountRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedPopcountRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvexpandpd forms for %s", instruction, goarch) + } +} + +func TestAMD64PackedPopcountRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT packedpopcount(SB),$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + MOVQ mask+24(FP), DX + KMOVQ DX, K1 + VMOVDQU64 (BX), Z0 + VMOVDQU64 (CX), Z1 + VPOPCNTB Z0, Z2 + VMOVDQU64 Z2, 0(AX) + VPOPCNTW Z0, K1, Z1 + VMOVDQU64 Z1, 64(AX) + VPOPCNTD.Z Z0, K1, Z1 + VMOVDQU64 Z1, 128(AX) + VPOPCNTQ (BX), Z2 + VMOVDQU64 Z2, 192(AX) + VPOPCNTD.BCST (BX), Z2 + VMOVDQU64 Z2, 256(AX) + VPOPCNTQ.BCST.Z (BX), K1, Z1 + VMOVDQU64 Z1, 320(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedpopcount": {Name: "packedpopcount", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +extern void packedpopcount(uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static unsigned pop64(uint64_t v) { + unsigned n = 0; + while (v) { n += v & 1; v >>= 1; } + return n; +} +int main(void) { + uint8_t source[64], old[64], out[384]; + const uint64_t mask = 0xa55a5aa5ULL; + for (int i = 0; i < 64; i++) { source[i] = (uint8_t)(i * 37 + 11); old[i] = (uint8_t)(0xe0 - i); } + memset(out, 0xcc, sizeof(out)); + packedpopcount(out, source, old, mask); + for (int i = 0; i < 64; i++) if (out[i] != pop64(source[i])) return 10 + i; + for (int i = 0; i < 32; i++) { + uint16_t src, prior, got; memcpy(&src, source+2*i, 2); memcpy(&prior, old+2*i, 2); memcpy(&got, out+64+2*i, 2); + uint16_t want = (mask >> i) & 1 ? (uint16_t)pop64(src) : prior; + if (got != want) return 80 + i; + } + for (int i = 0; i < 16; i++) { + uint32_t src, got; memcpy(&src, source+4*i, 4); memcpy(&got, out+128+4*i, 4); + uint32_t want = (mask >> i) & 1 ? pop64(src) : 0; + if (got != want) return 120 + i; + } + for (int i = 0; i < 8; i++) { + uint64_t src, got; memcpy(&src, source+8*i, 8); memcpy(&got, out+192+8*i, 8); + if (got != pop64(src)) return 150 + i; + } + uint32_t first32; memcpy(&first32, source, 4); + for (int i = 0; i < 16; i++) { uint32_t got; memcpy(&got, out+256+4*i, 4); if (got != pop64(first32)) return 170 + i; } + uint64_t first64; memcpy(&first64, source, 8); + for (int i = 0; i < 8; i++) { uint64_t got; memcpy(&got, out+320+8*i, 8); uint64_t want = (mask >> i) & 1 ? pop64(first64) : 0; if (got != want) return 190 + i; } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_popcount", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_portio.go b/amd64_lower_portio.go new file mode 100644 index 000000000..c48177ec7 --- /dev/null +++ b/amd64_lower_portio.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86PortScalarProperties(op Op) (input bool, typ LLVMType, ok bool) { + switch strings.ToUpper(string(op)) { + case "INB": + return true, I8, true + case "INW": + return true, I16, true + case "INL": + return true, I32, true + case "OUTB": + return false, I8, true + case "OUTW": + return false, I16, true + case "OUTL": + return false, I32, true + default: + return false, "", false + } +} + +func x86PortStringProperties(op Op) (input bool, width int, ok bool) { + switch strings.ToUpper(string(op)) { + case "INSB": + return true, 1, true + case "INSW": + return true, 2, true + case "INSL": + return true, 4, true + case "OUTSB": + return false, 1, true + case "OUTSW": + return false, 2, true + case "OUTSL": + return false, 4, true + default: + return false, 0, false + } +} + +// lowerPortIO implements the complete x86 port-I/O rows in Go's yin and +// ynone operand tables. These instructions are inherently target-specific, so +// preserving them as constrained LLVM inline assembly is both more accurate +// and safer than pretending that they are ordinary memory accesses. +func (c *amd64Ctx) lowerPortIO(op Op, ins Instr) (ok bool, terminated bool, err error) { + if input, typ, scalar := x86PortScalarProperties(op); scalar { + if c.repeatPrefix != "" { + prefix := c.repeatPrefix + c.repeatPrefix = "" + return true, false, fmt.Errorf("%s %s prefix is unsupported for %s", c.goarch, prefix, op) + } + if len(ins.Args) > 1 || (len(ins.Args) == 1 && ins.Args[0].Kind != OpImm) { + return true, false, fmt.Errorf("%s %s expects no operand or one immediate: %q", c.goarch, op, ins.Raw) + } + if len(ins.Args) == 1 && (ins.Args[0].Imm < -1<<31 || ins.Args[0].Imm > 1<<32-1) { + return true, false, fmt.Errorf("%s %s immediate is outside the Go assembler's 32-bit form: %q", c.goarch, op, ins.Raw) + } + return true, false, c.lowerScalarPortIO(strings.ToLower(string(op)), input, typ, ins.Args) + } + + input, _, stringOp := x86PortStringProperties(op) + if !stringOp { + return false, false, nil + } + prefix := c.repeatPrefix + c.repeatPrefix = "" + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, op, ins.Raw) + } + return true, false, c.lowerStringPortIO(strings.ToLower(string(op)), input, prefix) +} + +func (c *amd64Ctx) lowerScalarPortIO(asm string, input bool, typ LLVMType, args []Operand) error { + constraints := "~{dirflag},~{fpsr},~{flags}" + if input { + call := c.newTmp() + if len(args) == 0 { + port, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I16) + if err != nil { + return err + } + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(i16 %s)\n", call, typ, asm+" $1, $0", "={ax},{dx},"+constraints, port) + } else { + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", call, typ, fmt.Sprintf("%s $$%d, $0", asm, int64(args[0].Imm)), "={ax},"+constraints) + } + return c.storeRegSized(AX, typ, "%"+call) + } + + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) + if err != nil { + return err + } + if len(args) == 0 { + port, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I16) + if err != nil { + return err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s, i16 %s)\n", asm+" $0, $1", "{ax},{dx},"+constraints, typ, value, port) + return nil + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", fmt.Sprintf("%s $0, $$%d", asm, int64(args[0].Imm)), "{ax},"+constraints, typ, value) + return nil +} + +func (c *amd64Ctx) lowerStringPortIO(asm string, input bool, prefix string) error { + indexReg := SI + indexConstraint := "{si}" + if input { + indexReg = DI + indexConstraint = "{di}" + } + index, err := c.loadReg(indexReg) + if err != nil { + return err + } + port, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I16) + if err != nil { + return err + } + wordType := I64 + if c.goarch == "386" { + wordType = I32 + index = c.truncI64(index, I32) + } + clobbers := "~{memory},~{dirflag},~{fpsr},~{flags}" + if prefix == "" { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(i16 %s, %s %s)\n", call, wordType, asm, "="+indexConstraint+",{dx},0,"+clobbers, port, wordType, index) + return c.storeRegSized(indexReg, wordType, "%"+call) + } + + count, err := c.loadReg(CX) + if err != nil { + return err + } + if c.goarch == "386" { + count = c.truncI64(count, I32) + } + call := c.newTmp() + resultType := fmt.Sprintf("{ %s, %s }", wordType, wordType) + asmPrefix := "rep" + if prefix == "REPN" { + asmPrefix = "repne" + } + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(i16 %s, %s %s, %s %s)\n", call, resultType, asmPrefix+"; "+asm, "="+indexConstraint+",={cx},{dx},0,1,"+clobbers, port, wordType, index, wordType, count) + nextIndex := c.newTmp() + nextCount := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", nextIndex, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", nextCount, resultType, call) + if err := c.storeRegSized(indexReg, wordType, "%"+nextIndex); err != nil { + return err + } + return c.storeRegSized(CX, wordType, "%"+nextCount) +} diff --git a/amd64_lower_qword_bit_lookup.go b/amd64_lower_qword_bit_lookup.go new file mode 100644 index 000000000..dd9575824 --- /dev/null +++ b/amd64_lower_qword_bit_lookup.go @@ -0,0 +1,189 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64QwordBitLookupOutput uint8 + +const ( + amd64QwordBitLookupBytes amd64QwordBitLookupOutput = iota + amd64QwordBitLookupMask +) + +type amd64QwordBitLookupSpec struct { + output amd64QwordBitLookupOutput + broadcast bool +} + +// amd64QwordBitLookupSpecs groups the two AVX-512 byte-control operations +// that select bits inside corresponding qword lanes. They share the same core +// rotate/lookup grammar but produce either bytes or a packed K mask. +var amd64QwordBitLookupSpecs = map[Op]amd64QwordBitLookupSpec{ + "VPMULTISHIFTQB": {output: amd64QwordBitLookupBytes, broadcast: true}, + "VPSHUFBITQMB": {output: amd64QwordBitLookupMask}, +} + +func (c *amd64Ctx) lowerQwordBitLookup(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64QwordBitLookupSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + + suffixes, err := amd64ParseTwoSourcePermuteSuffixes(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + if spec.output == amd64QwordBitLookupMask && (suffixes.broadcast || suffixes.zeroing) { + return true, false, fmt.Errorf("%s %s has no instruction suffixes in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if suffixes.broadcast && !spec.broadcast { + return true, false, fmt.Errorf("%s %s does not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects first source, second source, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s masked form exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if suffixes.zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + first, second := ins.Args[0], ins.Args[1] + destination := ins.Args[len(ins.Args)-1] + byteWidth := 0 + switch spec.output { + case amd64QwordBitLookupBytes: + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth = amd64VectorByteWidth(destination.Reg) + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's vector class: %q", c.goarch, baseOp, ins.Raw) + } + case amd64QwordBitLookupMask: + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be K0-K7: %q", c.goarch, baseOp, ins.Raw) + } + if _, valid := amd64ParseKReg(destination.Reg); !valid { + return true, false, fmt.Errorf("%s %s destination must be K0-K7: %q", c.goarch, baseOp, ins.Raw) + } + if second.Kind == OpReg { + byteWidth = amd64VectorByteWidth(second.Reg) + } + } + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(second, byteWidth) { + return true, false, fmt.Errorf("%s %s second source must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("%s %s first source register must match the vector width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s first source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if suffixes.broadcast && !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s.BCST requires a memory qword source: %q", c.goarch, baseOp, ins.Raw) + } + + var dataQwords, controls string + switch spec.output { + case amd64QwordBitLookupBytes: + dataQwords, err = c.loadPackedCompareLanes(first, byteWidth, 64, suffixes.broadcast) + if err == nil { + controls, err = c.loadPackedCompareBytes(second, byteWidth) + } + case amd64QwordBitLookupMask: + dataQwords, err = c.loadPackedCompareLanes(second, byteWidth, 64, false) + if err == nil { + controls, err = c.loadPackedCompareBytes(first, byteWidth) + } + } + if err != nil { + return true, false, err + } + selectedBytes := c.emitQwordBitLookupBytes(byteWidth, dataQwords, controls) + + var writeMask string + if masked { + writeMask, err = c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + } + if spec.output == amd64QwordBitLookupBytes { + if masked { + old, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + selectedBytes = amd64ApplyIntegerLaneMask(c, byteWidth, 8, selectedBytes, old, writeMask, suffixes.zeroing) + } + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, selectedBytes) + } + + ones := c.newTmp() + predicates := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %s, %s\n", ones, byteWidth, selectedBytes, llvmSplatInteger(byteWidth, 8, 1)) + fmt.Fprintf(c.b, " %%%s = icmp ne <%d x i8> %%%s, zeroinitializer\n", predicates, byteWidth, ones) + packedType := fmt.Sprintf("i%d", byteWidth) + packed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i1> %%%s to %s\n", packed, byteWidth, predicates, packedType) + result := "%" + packed + if byteWidth < 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", wide, packedType, result) + result = "%" + wide + } + if masked { + maskedResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %s\n", maskedResult, result, writeMask) + result = "%" + maskedResult + } + return true, false, c.storeK(destination.Reg, result) +} + +// emitQwordBitLookupBytes rotates each corresponding data qword right by the +// low six bits of a control byte. Its low byte is VPMULTISHIFTQB's result; +// its low bit is VPSHUFBITQMB's gathered mask bit. +func (c *amd64Ctx) emitQwordBitLookupBytes(byteWidth int, dataQwords, controls string) string { + result := "zeroinitializer" + qwords := byteWidth / 8 + for lane := 0; lane < byteWidth; lane++ { + data := c.newTmp() + controlByte := c.newTmp() + controlWide := c.newTmp() + control := c.newTmp() + right := c.newTmp() + negated := c.newTmp() + wrappedLeftCount := c.newTmp() + left := c.newTmp() + rotated := c.newTmp() + selected := c.newTmp() + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i64> %s, i32 %d\n", data, qwords, dataQwords, lane/8) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", controlByte, byteWidth, controls, lane) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", controlWide, controlByte) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", control, controlWide) + fmt.Fprintf(c.b, " %%%s = lshr i64 %%%s, %%%s\n", right, data, control) + fmt.Fprintf(c.b, " %%%s = sub i64 0, %%%s\n", negated, control) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, 63\n", wrappedLeftCount, negated) + fmt.Fprintf(c.b, " %%%s = shl i64 %%%s, %%%s\n", left, data, wrappedLeftCount) + fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", rotated, right, left) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i8\n", selected, rotated) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, result, selected, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_qword_bit_lookup_test.go b/amd64_lower_qword_bit_lookup_test.go new file mode 100644 index 000000000..b94e5e108 --- /dev/null +++ b/amd64_lower_qword_bit_lookup_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64QwordBitLookupGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64QwordBitLookupSpec{ + "VPMULTISHIFTQB": {output: amd64QwordBitLookupBytes, broadcast: true}, + "VPSHUFBITQMB": {output: amd64QwordBitLookupMask}, + } + if len(amd64QwordBitLookupSpecs) != len(expected) { + t.Fatalf("qword-bit-lookup grammar has %d entries, want %d", len(amd64QwordBitLookupSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64QwordBitLookupSpecs[op]; !ok { + t.Errorf("qword-bit-lookup grammar omitted %s", op) + } else if got != want { + t.Errorf("qword-bit-lookup grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86QwordBitLookupCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT qwordbitlookupforms(SB),$0-0\n") + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\tVPMULTISHIFTQB %s1, %s2, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB 8(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB.BCST 16(AX), %s2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPSHUFBITQMB %s1, %s2, K0\n", width, width) + fmt.Fprintf(&source, "\tVPSHUFBITQMB 24(AX), %s2, K7\n", width) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\tVPMULTISHIFTQB %s1, %s2, K1, %s%d\n", width, width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB.Z 32(AX), %s2, K2, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPMULTISHIFTQB.BCST.Z 40(AX), %s2, K3, %s%d\n", width, width, last) + fmt.Fprintf(&source, "\tVPSHUFBITQMB %s1, %s2, K4, K7\n", width, width) + fmt.Fprintf(&source, "\tVPSHUFBITQMB 48(AX), %s2, K5, K6\n", width) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"qwordbitlookupforms": {Name: "qwordbitlookupforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "qword-bit-lookup-"+target.name+".ll", "qword-bit-lookup-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86QwordBitLookupRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VPMULTISHIFTQB X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB.BCST X0, X1, X2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VPMULTISHIFTQB X0, X1, 0(AX)"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB.Z X0, X1, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB.BCST 0(AX), X1, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB X0, Y1, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB X0, X1, K0, K2"}, + {goarch: "amd64", instruction: "VPSHUFBITQMB X0, X1, AX"}, + {goarch: "386", instruction: "VPMULTISHIFTQB X0, X1, K1, X2"}, + {goarch: "386", instruction: "VPSHUFBITQMB X0, X1, K1, K2"}, + {goarch: "386", instruction: "VPMULTISHIFTQB Z0, Z1, Z8"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's qword-bit-lookup table", test.instruction) + } + }) + } +} + +func TestAMD64QwordBitLookupRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT qwordbitlookupsemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ data+8(FP), BX + MOVQ controls+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), R8 + KMOVQ R8, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + VMOVDQU64 0(DX), Z2 + STC + VPMULTISHIFTQB Z0, Z1, Z3 + VMOVDQU64 Z3, 0(AX) + VPMULTISHIFTQB Z0, Z1, K1, Z2 + VMOVDQU64 Z2, 64(AX) + VPMULTISHIFTQB.Z Z0, Z1, K1, Z3 + VMOVDQU64 Z3, 128(AX) + VPMULTISHIFTQB.BCST 0(BX), Z1, Z3 + VMOVDQU64 Z3, 192(AX) + VPSHUFBITQMB Z1, Z0, K2 + KMOVQ K2, 256(AX) + VPSHUFBITQMB Z1, Z0, K1, K3 + KMOVQ K3, 264(AX) + SETCS 272(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"qwordbitlookupsemantics": { + Name: "qwordbitlookupsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void qwordbitlookupsemantics(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, uint64_t); +static uint64_t ror64(uint64_t value, unsigned count) { + count &= 63u; + return (value >> count) | (value << ((0u-count)&63u)); +} +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint64_t data64[8], controls64[8], old64[8], output64[35] = {0}; + uint8_t *data=(uint8_t *)data64, *controls=(uint8_t *)controls64; + uint8_t *old=(uint8_t *)old64, *out=(uint8_t *)output64; + const uint64_t mask=UINT64_C(0xa55af00ff00fa55a); + for (int q=0;q<8;q++) data64[q]=UINT64_C(0x0123456789abcdef)^((uint64_t)q*UINT64_C(0x11100f0e0d0c0b0a)); + for (int i=0;i<64;i++) { controls[i]=(uint8_t)(13u*(unsigned)i+((unsigned)i>>2)); old[i]=(uint8_t)(0xe0u-(unsigned)i); } + qwordbitlookupsemantics(out,data,controls,old,mask); + uint64_t gathered=0; + for (int i=0;i<64;i++) { + uint64_t qword=data64[i/8]; + uint8_t selected=(uint8_t)ror64(qword,controls[i]); + if (out[i]!=selected) return 10; + if (out[64+i]!=(((mask>>i)&1u)?selected:old[i])) return 11; + if (out[128+i]!=(((mask>>i)&1u)?selected:0)) return 12; + uint8_t broadcast=(uint8_t)ror64(data64[0],controls[i]); + if (out[192+i]!=broadcast) return 13; + gathered|=((qword>>(controls[i]&63u))&1u)<= 0 { + baseOp = rawOp[:dot] + } + + family := "" + bits := 0 + for _, candidate := range []string{"RDRAND", "RDSEED"} { + if !strings.HasPrefix(baseOp, candidate) { + continue + } + family = strings.ToLower(candidate) + switch strings.TrimPrefix(baseOp, candidate) { + case "W": + bits = 16 + case "L": + bits = 32 + case "Q": + bits = 64 + default: + return true, false, fmt.Errorf("%s %s requires a W, L, or Q width suffix: %q", c.goarch, candidate, ins.Raw) + } + break + } + if family == "" { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 1 || ins.Args[0].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[0].Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s expects one general register: %q", c.goarch, baseOp, ins.Raw) + } + + typ := amd64IntegerTypeForBits(bits) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call { %s, i32 } @llvm.x86.%s.%d()\n", call, typ, family, bits) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i32 } %%%s, 0\n", value, typ, call) + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i32 } %%%s, 1\n", status, typ, call) + + // SP is a legal Yrl destination in Go's 386 table. This instruction is the + // modeled write itself, so permit storeRegSized to update the virtual SP. + previousAllowSPWrite := c.allowSPWrite + if c.goarch == "386" && ins.Args[0].Reg == SP { + c.allowSPWrite = true + } + err = c.storeRegSized(ins.Args[0].Reg, typ, "%"+value) + c.allowSPWrite = previousAllowSPWrite + if err != nil { + return true, false, err + } + + carry := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", carry, status) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", carry, c.flagsCFSlot) + c.clearX86StatusFlags(x86StatusAll &^ x86StatusCF) + return true, false, nil +} diff --git a/amd64_lower_random_test.go b/amd64_lower_random_test.go new file mode 100644 index 000000000..4ee482abc --- /dev/null +++ b/amd64_lower_random_test.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86RandomCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + widths []string + registers []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", widths: []string{"W", "L"}, registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}}, + {goarch: "386", triple: "i686-pc-windows-msvc", widths: []string{"W", "L"}, registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}}, + {goarch: "amd64", triple: "x86_64-apple-darwin", widths: []string{"W", "L", "Q"}, registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", widths: []string{"W", "L", "Q"}, registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc", widths: []string{"W", "L", "Q"}, registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + } { + t.Run(target.goarch+"/"+target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT randomforms(SB),$0-0\n") + for _, family := range []string{"RDRAND", "RDSEED"} { + for _, width := range target.widths { + for _, register := range target.registers { + fmt.Fprintf(&source, "\t%s%s %s\n", family, width, register) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "randomforms": {Name: "randomforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "@llvm.x86.rdrand.16()", "@llvm.x86.rdrand.32()", + "@llvm.x86.rdseed.16()", "@llvm.x86.rdseed.32()", + `"target-features"="+rdrnd,+rdseed"`, + } { + if !strings.Contains(ll, want) { + t.Fatalf("random lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" { + for _, want := range []string{"@llvm.x86.rdrand.64()", "@llvm.x86.rdseed.64()"} { + if !strings.Contains(ll, want) { + t.Fatalf("random lowering omitted %q:\n%s", want, ll) + } + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-random.ll", "x86-random.o", ll) + }) + } +} + +func TestTranslateX86RandomRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "386", instruction: "RDRANDQ AX"}, + {goarch: "386", instruction: "RDSEEDQ AX"}, + {goarch: "386", instruction: "RDRANDL R8"}, + {goarch: "386", instruction: "RDSEEDW R8"}, + {goarch: "amd64", instruction: "RDRANDB AL"}, + {goarch: "amd64", instruction: "RDSEEDB AL"}, + {goarch: "amd64", instruction: "RDRANDQ 0(AX)"}, + {goarch: "amd64", instruction: "RDSEEDQ 0(AX)"}, + {goarch: "amd64", instruction: "RDRANDQ AX, DX"}, + {goarch: "amd64", instruction: "RDSEEDQ"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", ".", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT badrandom(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"badrandom": {Name: "badrandom", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go's RDRAND/RDSEED optab", test.instruction) + } + }) + } +} + +func TestX86RawDecoderNormalizesCompleteRandomFamily(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + reg Reg + }{ + {name: "rdrand word low", code: []byte{0x66, 0x0f, 0xc7, 0xf2}, want: "RDRANDW", reg: DX}, + {name: "rdrand word extended", code: []byte{0x66, 0x41, 0x0f, 0xc7, 0xf3}, want: "RDRANDW", reg: Reg("R11")}, + {name: "rdrand long low", code: []byte{0x0f, 0xc7, 0xf2}, want: "RDRANDL", reg: DX}, + {name: "rdrand long extended", code: []byte{0x41, 0x0f, 0xc7, 0xf3}, want: "RDRANDL", reg: Reg("R11")}, + {name: "rdrand quad low", code: []byte{0x48, 0x0f, 0xc7, 0xf2}, want: "RDRANDQ", reg: DX}, + {name: "rdrand quad extended", code: []byte{0x49, 0x0f, 0xc7, 0xf3}, want: "RDRANDQ", reg: Reg("R11")}, + {name: "rdseed word low", code: []byte{0x66, 0x0f, 0xc7, 0xfa}, want: "RDSEEDW", reg: DX}, + {name: "rdseed word extended", code: []byte{0x66, 0x41, 0x0f, 0xc7, 0xfb}, want: "RDSEEDW", reg: Reg("R11")}, + {name: "rdseed long low", code: []byte{0x0f, 0xc7, 0xfa}, want: "RDSEEDL", reg: DX}, + {name: "rdseed long extended", code: []byte{0x41, 0x0f, 0xc7, 0xfb}, want: "RDSEEDL", reg: Reg("R11")}, + {name: "rdseed quad low", code: []byte{0x48, 0x0f, 0xc7, 0xfa}, want: "RDSEEDQ", reg: DX}, + {name: "rdseed quad extended", code: []byte{0x49, 0x0f, 0xc7, 0xfb}, want: "RDSEEDQ", reg: Reg("R11")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want || len(got.Instrs[0].Args) != 1 || got.Instrs[0].Args[0].Reg != test.reg { + t.Fatalf("decoded %#x as %#v, want %s %s", test.code, got.Instrs, test.want, test.reg) + } + }) + } +} diff --git a/amd64_lower_range.go b/amd64_lower_range.go new file mode 100644 index 000000000..7952c9462 --- /dev/null +++ b/amd64_lower_range.go @@ -0,0 +1,281 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64FloatingRangeSpec struct { + laneBits int + scalar bool +} + +// amd64FloatingRangeSpecs is the complete Go 1.27 VRANGE grammar. The +// immediate supplies the comparison and sign modes, so those orthogonal +// dimensions stay data instead of expanding into opcode-specific handlers. +var amd64FloatingRangeSpecs = map[Op]amd64FloatingRangeSpec{ + "VRANGEPS": {laneBits: 32}, + "VRANGEPD": {laneBits: 64}, + "VRANGESS": {laneBits: 32, scalar: true}, + "VRANGESD": {laneBits: 64, scalar: true}, +} + +func (c *amd64Ctx) lowerFloatingRange(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64FloatingRangeSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if c.goarch == "386" { + return true, false, fmt.Errorf("386 %s exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.rounding != "" { + return true, false, fmt.Errorf("%s %s has a suffix absent from Go 1.27's range optab: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar && properties.broadcast { + return true, false, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 4 && len(ins.Args) != 5 { + return true, false, fmt.Errorf("%s %s expects immediate, source2, source1, [K mask,] destination: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("%s %s immediate must be an unsigned byte: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 5 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[3]) { + return true, false, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + first, second := ins.Args[1], ins.Args[2] + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if spec.scalar { + if byteWidth != 16 { + return true, false, fmt.Errorf("%s %s scalar destination must be an X register: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoEVEXVectorRegister(destination, byteWidth) || !c.isGoEVEXVectorRegister(second, byteWidth) { + return true, false, fmt.Errorf("%s %s source1 and destination must be same-width Go vector registers: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae && !spec.scalar && byteWidth != 64 { + return true, false, fmt.Errorf("%s %s packed SAE is available only for Z registers: %q", c.goarch, baseOp, ins.Raw) + } + if properties.broadcast { + if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source2: %q", c.goarch, baseOp, ins.Raw) + } + } else if first.Kind == OpReg { + if !c.isGoEVEXVectorRegister(first, byteWidth) { + return true, false, fmt.Errorf("%s %s source2 register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("%s %s source2 must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae && first.Kind != OpReg { + return true, false, fmt.Errorf("%s %s.SAE requires a register source2: %q", c.goarch, baseOp, ins.Raw) + } + + mask := "" + if masked { + mask, err = c.loadK(ins.Args[3].Reg) + if err != nil { + return true, false, err + } + } + if spec.scalar { + return c.lowerScalarFloatingRange(spec, properties, ins, first, second, destination, mask) + } + return c.lowerPackedFloatingRange(spec, properties, ins, first, second, destination, byteWidth, mask) +} + +func (c *amd64Ctx) lowerPackedFloatingRange(spec amd64FloatingRangeSpec, properties amd64BinaryFloatingSuffix, ins Instr, first, second, destination Operand, byteWidth int, mask string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + source2, err := c.loadPackedCompareLanes(first, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + source1, err := c.loadPackedCompareLanes(second, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + result := c.emitFloatingRangeBits(lanes, spec.laneBits, source1, source2, uint8(ins.Args[0].Imm)) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarFloatingRange(spec amd64FloatingRangeSpec, properties amd64BinaryFloatingSuffix, ins Instr, first, second, destination Operand, mask string) (bool, bool, error) { + source2, err := c.loadFloatingScalarBits(first, spec.laneBits) + if err != nil { + return true, false, err + } + source1, err := c.loadFloatingScalarBits(second, spec.laneBits) + if err != nil { + return true, false, err + } + source2Vector := c.newTmp() + source1Vector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", source2Vector, spec.laneBits, spec.laneBits, source2) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", source1Vector, spec.laneBits, spec.laneBits, source1) + result := c.emitFloatingRangeBits(1, spec.laneBits, "%"+source1Vector, "%"+source2Vector, uint8(ins.Args[0].Imm)) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + secondBytes, err := c.loadX(second.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, secondBytes) + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) loadFloatingScalarBits(operand Operand, laneBits int) (string, error) { + value, err := c.loadFloatingScalarOperand(operand, laneBits) + if err != nil { + return "", err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, amd64FloatingScalarType(laneBits), value, laneBits) + return "%" + bits, nil +} + +// emitFloatingRangeBits follows Intel's range operation in integer bit space +// around one ordered floating comparison. This preserves NaN payloads, quiets +// signaling NaNs, and handles signed zero and equal-magnitude opposite signs. +func (c *amd64Ctx) emitFloatingRangeBits(lanes, laneBits int, source1, source2 string, immediate uint8) string { + integerType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + floatType := amd64FloatingVectorType(lanes, laneBits) + predicateType := fmt.Sprintf("<%d x i1>", lanes) + signBit := uint64(1) << (laneBits - 1) + absMask := signBit - 1 + quietBit := uint64(0x00400000) + infinity := uint64(0x7f800000) + if laneBits == 64 { + quietBit = uint64(0x0008000000000000) + infinity = uint64(0x7ff0000000000000) + } + signVector := llvmSplatInteger(lanes, laneBits, signBit) + absVector := llvmSplatInteger(lanes, laneBits, absMask) + quietVector := llvmSplatInteger(lanes, laneBits, quietBit) + infinityVector := llvmSplatInteger(lanes, laneBits, infinity) + + abs1, abs2 := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", abs1, integerType, source1, absVector) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", abs2, integerType, source2, absVector) + + compareLeft, compareRight := source1, source2 + if immediate&3 >= 2 { + compareLeft, compareRight = "%"+abs1, "%"+abs2 + } + leftFloat, rightFloat := c.newTmp(), c.newTmp() + ordered := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", leftFloat, integerType, compareLeft, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to %s\n", rightFloat, integerType, compareRight, floatType) + fmt.Fprintf(c.b, " %%%s = fcmp ole %s %%%s, %%%s\n", ordered, floatType, leftFloat, rightFloat) + selected := c.newTmp() + if immediate&1 == 0 { + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", selected, predicateType, ordered, integerType, source1, integerType, source2) + } else { + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", selected, predicateType, ordered, integerType, source2, integerType, source1) + } + result := "%" + selected + + signXor, signDifference, oppositeSigns := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor %s %s, %s\n", signXor, integerType, source1, source2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %s\n", signDifference, integerType, signXor, signVector) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, zeroinitializer\n", oppositeSigns, integerType, signDifference) + if immediate&3 < 2 { + zero1, zero2, bothZero, oppositeZero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", zero1, integerType, abs1) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", zero2, integerType, abs2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", bothZero, predicateType, zero1, zero2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", oppositeZero, predicateType, bothZero, oppositeSigns) + zeroResult := "zeroinitializer" + if immediate&1 == 0 { + zeroResult = signVector + } + overridden := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", overridden, predicateType, oppositeZero, integerType, zeroResult, integerType, result) + result = "%" + overridden + } else { + equalMagnitude, equalOpposite := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %%%s\n", equalMagnitude, integerType, abs1, abs2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", equalOpposite, predicateType, equalMagnitude, oppositeSigns) + tieResult := "%" + abs1 + if immediate&1 == 0 { + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", negative, integerType, abs1, signVector) + tieResult = "%" + negative + } + overridden := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", overridden, predicateType, equalOpposite, integerType, tieResult, integerType, result) + result = "%" + overridden + } + + isNaN1, isNaN2 := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", isNaN1, integerType, abs1, infinityVector) + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %s\n", isNaN2, integerType, abs2, infinityVector) + withoutNaN1 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", withoutNaN1, predicateType, isNaN1, integerType, source2, integerType, result) + withoutQuietNaNs := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %%%s\n", withoutQuietNaNs, predicateType, isNaN2, integerType, source1, integerType, withoutNaN1) + result = "%" + withoutQuietNaNs + + signMode := immediate >> 2 & 3 + if signMode != 1 { + magnitude := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", magnitude, integerType, result, absVector) + switch signMode { + case 0: + sourceSign := c.newTmp() + signed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", sourceSign, integerType, source1, signVector) + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", signed, integerType, magnitude, sourceSign) + result = "%" + signed + case 2: + result = "%" + magnitude + case 3: + signed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %s\n", signed, integerType, magnitude, signVector) + result = "%" + signed + } + } + + quiet1, quiet2 := c.newTmp(), c.newTmp() + quietField1, quietField2 := c.newTmp(), c.newTmp() + quietClear1, quietClear2 := c.newTmp(), c.newTmp() + signaling1, signaling2 := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %s, %s\n", quiet1, integerType, source1, quietVector) + fmt.Fprintf(c.b, " %%%s = or %s %s, %s\n", quiet2, integerType, source2, quietVector) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", quietField1, integerType, source1, quietVector) + fmt.Fprintf(c.b, " %%%s = and %s %s, %s\n", quietField2, integerType, source2, quietVector) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", quietClear1, integerType, quietField1) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, zeroinitializer\n", quietClear2, integerType, quietField2) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", signaling1, predicateType, isNaN1, quietClear1) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %%%s\n", signaling2, predicateType, isNaN2, quietClear2) + withSecondSignaling := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %%%s, %s %s\n", withSecondSignaling, predicateType, signaling2, integerType, quiet2, integerType, result) + withFirstSignaling := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %%%s, %s %%%s\n", withFirstSignaling, predicateType, signaling1, integerType, quiet1, integerType, withSecondSignaling) + return "%" + withFirstSignaling +} diff --git a/amd64_lower_range_test.go b/amd64_lower_range_test.go new file mode 100644 index 000000000..001bad7e1 --- /dev/null +++ b/amd64_lower_range_test.go @@ -0,0 +1,373 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64FloatingRangeGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64FloatingRangeSpec{ + "VRANGEPS": {laneBits: 32}, + "VRANGEPD": {laneBits: 64}, + "VRANGESS": {laneBits: 32, scalar: true}, + "VRANGESD": {laneBits: 64, scalar: true}, + } + if len(amd64FloatingRangeSpecs) != len(expected) { + t.Fatalf("floating-range grammar has %d entries, want %d", len(amd64FloatingRangeSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64FloatingRangeSpecs[op]; !ok { + t.Errorf("floating-range grammar omitted %s", op) + } else if got != want { + t.Errorf("floating-range grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86FloatingRangeCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT floatingrangeforms(SB),$0-0\n") + for _, op := range []string{"VRANGEPS", "VRANGEPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, %s1, %s2, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST $2, 16(AX), %s2, %s%d\n", op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $3, %s1, %s2, K1, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s.Z $4, 24(AX), %s2, K2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z $5, 32(AX), %s2, K3, %s%d\n", op, width, width, last) + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE $6, Z1, Z2, Z%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z $7, Z1, Z2, K4, Z%d\n", op, last) + } + } + } + } + for _, op := range []string{"VRANGESS", "VRANGESD"} { + last := 31 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $8, X1, X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s $9, 40(AX), X2, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE $10, X1, X2, X%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $11, X1, X2, K5, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.Z $12, 48(AX), X2, K6, X%d\n", op, last) + fmt.Fprintf(&source, "\t%s.SAE.Z $13, X1, X2, K7, X%d\n", op, last) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"floatingrangeforms": {Name: "floatingrangeforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "floating-range-"+target.name+".ll", "floating-range-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86FloatingRangeRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VRANGEPS $-1, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS $256, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS $0, X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VRANGEPS.BCST $0, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS.SAE $0, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGESD.SAE $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS.RN_SAE $0, Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VRANGEPS.Z $0, X0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGEPS $0, X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VRANGESS $0, Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRANGESS.BCST $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRANGESD $0, X0, X1, Y2"}, + {goarch: "386", instruction: "VRANGEPS $0, X0, X1, X2"}, + {goarch: "386", instruction: "VRANGEPS $0, X0, X1, K1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's floating-range table", test.instruction) + } + }) + } +} + +func TestAMD64FloatingRangeRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + source.WriteString(`TEXT floatingrangesemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ source2+8(FP), BX + MOVQ source1+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), R8 + KMOVQ R8, K1 + VMOVDQU32 0(BX), Z0 + VMOVDQU32 0(CX), Z1 + STC +`) + for immediate := 0; immediate < 16; immediate++ { + fmt.Fprintf(&source, "\tVRANGEPS $%d, Z0, Z1, Z2\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU32 Z2, %d(AX)\n", immediate*64) + } + source.WriteString(` VMOVDQU32 0(DX), Z2 + VRANGEPS $2, Z0, Z1, K1, Z2 + VMOVDQU32 Z2, 1024(AX) + VRANGEPS.Z $3, Z0, Z1, K1, Z2 + VMOVDQU32 Z2, 1088(AX) + VRANGEPS.BCST $2, 0(BX), Z1, Z2 + VMOVDQU32 Z2, 1152(AX) + SETCS 1216(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"floatingrangesemantics": { + Name: "floatingrangesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatingrangesemantics(uint8_t *, const uint32_t *, const uint32_t *, const uint32_t *, uint64_t); +static float f32(uint32_t u) { float f; memcpy(&f,&u,4); return f; } +static uint32_t range32(uint32_t a, uint32_t b, unsigned imm) { + const uint32_t sign=UINT32_C(0x80000000), absMask=UINT32_C(0x7fffffff); + const uint32_t quiet=UINT32_C(0x00400000), inf=UINT32_C(0x7f800000); + uint32_t aa=a&absMask, ab=b&absMask, tmp; + int nanA=aa>inf, nanB=ab>inf, snanA=nanA && !(a&quiet), snanB=nanB && !(b&quiet); + unsigned op=imm&3, signs=(imm>>2)&3; + if (nanB) tmp=a; + else if (nanA) tmp=b; + else if (((a^b)&sign) && aa==0 && ab==0) tmp=(op&1)?0:sign; + else if (((a^b)&sign) && aa==ab && op>=2) tmp=(op&1)?aa:(aa|sign); + else { + int le=op>=2 ? f32(aa)<=f32(ab) : f32(a)<=f32(b); + tmp=(op&1) ? (le?b:a) : (le?a:b); + } + if (signs==0) tmp=(tmp&absMask)|(a&sign); + else if (signs==2) tmp&=absMask; + else if (signs==3) tmp|=sign; + if (snanB) tmp=b|quiet; + if (snanA) tmp=a|quiet; + return tmp; +} +int main(void) { + const uint32_t source1[16]={ + 0x00000000,0x80000000,0x3f800000,0xbf800000, + 0x40000000,0xc0000000,0x40400000,0xc0800000, + 0x7f800000,0xff800000,0x7fc12345,0xffc23456, + 0x7f812345,0xff823456,0x3f000000,0xbf400000}; + const uint32_t source2[16]={ + 0x80000000,0x00000000,0xbf800000,0x3f800000, + 0xc0400000,0x3f000000,0xc0400000,0x40a00000, + 0x7f000000,0xff000000,0x3f800000,0x7fc34567, + 0x40000000,0x7f834567,0xbf000000,0x3fc00000}; + uint32_t old[16], out[305]; + const uint64_t mask=UINT64_C(0xa55a); + for (int i=0;i<16;i++) old[i]=UINT32_C(0x41000000)+(uint32_t)i; + memset(out,0,sizeof(out)); + floatingrangesemantics((uint8_t*)out,source2,source1,old,mask); + for (unsigned imm=0;imm<16;imm++) for (int i=0;i<16;i++) + if (out[imm*16+i]!=range32(source1[i],source2[i],imm)) return 10+(int)imm; + for (int i=0;i<16;i++) { + uint32_t r2=range32(source1[i],source2[i],2); + uint32_t r3=range32(source1[i],source2[i],3); + if (out[256+i]!=(((mask>>i)&1)?r2:old[i])) return 40; + if (out[272+i]!=(((mask>>i)&1)?r3:0)) return 41; + if (out[288+i]!=range32(source1[i],source2[0],2)) return 42; + } + if (((uint8_t*)out)[1216]!=1) return 43; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "floating_range_semantics", triple, ir, mainC, runPrefix) +} + +func TestAMD64FloatingRangeDoubleAndScalarRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + source.WriteString(`TEXT floatingrangedoublesemantics(SB),$0-40 + MOVQ out+0(FP), AX + MOVQ source2+8(FP), BX + MOVQ source1+16(FP), CX + MOVQ old+24(FP), DX + MOVQ mask+32(FP), R8 + KMOVQ R8, K1 + VMOVDQU64 0(BX), Z0 + VMOVDQU64 0(CX), Z1 + STC +`) + for immediate := 0; immediate < 16; immediate++ { + fmt.Fprintf(&source, "\tVRANGEPD $%d, Z0, Z1, Z2\n", immediate) + fmt.Fprintf(&source, "\tVMOVDQU64 Z2, %d(AX)\n", immediate*64) + } + source.WriteString(` VMOVDQU64 0(BX), X0 + VMOVDQU64 0(CX), X1 + VMOVDQU64 0(DX), X2 + VRANGESD $2, X0, X1, X2 + VMOVDQU64 X2, 1024(AX) + VMOVDQU64 0(DX), X2 + VRANGESD $2, X0, X1, K1, X2 + VMOVDQU64 X2, 1040(AX) + VRANGESD.Z $2, X0, X1, K1, X2 + VMOVDQU64 X2, 1056(AX) + SETCS 1072(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"floatingrangedoublesemantics": { + Name: "floatingrangedoublesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void floatingrangedoublesemantics(uint8_t *, const uint64_t *, const uint64_t *, const uint64_t *, uint64_t); +static double f64(uint64_t u) { double f; memcpy(&f,&u,8); return f; } +static uint64_t range64(uint64_t a, uint64_t b, unsigned imm) { + const uint64_t sign=UINT64_C(0x8000000000000000), absMask=UINT64_C(0x7fffffffffffffff); + const uint64_t quiet=UINT64_C(0x0008000000000000), inf=UINT64_C(0x7ff0000000000000); + uint64_t aa=a&absMask, ab=b&absMask, tmp; + int nanA=aa>inf, nanB=ab>inf, snanA=nanA && !(a&quiet), snanB=nanB && !(b&quiet); + unsigned op=imm&3, signs=(imm>>2)&3; + if (nanB) tmp=a; + else if (nanA) tmp=b; + else if (((a^b)&sign) && aa==0 && ab==0) tmp=(op&1)?0:sign; + else if (((a^b)&sign) && aa==ab && op>=2) tmp=(op&1)?aa:(aa|sign); + else { + int le=op>=2 ? f64(aa)<=f64(ab) : f64(a)<=f64(b); + tmp=(op&1) ? (le?b:a) : (le?a:b); + } + if (signs==0) tmp=(tmp&absMask)|(a&sign); + else if (signs==2) tmp&=absMask; + else if (signs==3) tmp|=sign; + if (snanB) tmp=b|quiet; + if (snanA) tmp=a|quiet; + return tmp; +} +int main(void) { + const uint64_t source1[8]={ + UINT64_C(0),UINT64_C(0x8000000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0xfff823456789abcd),UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff023456789abcd)}; + const uint64_t source2[8]={ + UINT64_C(0x8000000000000000),UINT64_C(0),UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000000), + UINT64_C(0x4000000000000000),UINT64_C(0x7ff83456789abcde),UINT64_C(0x4008000000000000),UINT64_C(0x7ff0456789abcdef)}; + const uint64_t old[8]={UINT64_C(0x4020000000000000),UINT64_C(0x4030000000000000),0,0,0,0,0,0}; + uint64_t out[135]; + memset(out,0,sizeof(out)); + floatingrangedoublesemantics((uint8_t*)out,source2,source1,old,0); + for (unsigned imm=0;imm<16;imm++) for (int i=0;i<8;i++) + if (out[imm*8+i]!=range64(source1[i],source2[i],imm)) return 10+(int)imm; + if (out[128]!=range64(source1[0],source2[0],2)) return 40; + if (out[129]!=source1[1]) return 44; + if (out[130]!=old[0] || out[131]!=source1[1]) return 41; + if (out[132]!=0 || out[133]!=source1[1]) return 42; + if (((uint8_t*)out)[1072]!=1) return 43; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "floating_range_double_scalar_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_rdpid_test.go b/amd64_lower_rdpid_test.go new file mode 100644 index 000000000..e30ad61dd --- /dev/null +++ b/amd64_lower_rdpid_test.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86RDPIDCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + registers []string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}}, + {goarch: "386", triple: "i686-pc-windows-msvc", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}}, + {goarch: "amd64", triple: "x86_64-apple-darwin", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc", registers: []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"}}, + } { + t.Run(target.goarch+"/"+target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rdpidforms(SB),$0-0\n") + for _, register := range target.registers { + fmt.Fprintf(&source, "\tRDPID %s\n", register) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "rdpidforms": {Name: "rdpidforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.x86.rdpid()", `"target-features"="+rdpid"`} { + if !strings.Contains(ll, want) { + t.Fatalf("RDPID lowering omitted %q:\n%s", want, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-rdpid.ll", "x86-rdpid.o", ll) + }) + } +} + +func TestTranslateX86RDPIDRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "386", instruction: "RDPID AL"}, + {goarch: "386", instruction: "RDPID 0(AX)"}, + {goarch: "386", instruction: "RDPID R8"}, + {goarch: "amd64", instruction: "RDPID AL"}, + {goarch: "amd64", instruction: "RDPID 0(AX)"}, + {goarch: "amd64", instruction: "RDPID AX, DX"}, + {goarch: "amd64", instruction: "RDPID.Z AX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", ".", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT badrdpid(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{ + "badrdpid": {Name: "badrdpid", Ret: Void}, + }, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go's RDPID optab", test.instruction) + } + }) + } +} diff --git a/amd64_lower_reciprocal.go b/amd64_lower_reciprocal.go new file mode 100644 index 000000000..64e84fd67 --- /dev/null +++ b/amd64_lower_reciprocal.go @@ -0,0 +1,367 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64Reciprocal14Spec struct { + laneBits int + scalar bool + rsqr bool +} + +// amd64Reciprocal14Specs covers every opcode backed by Go 1.27's +// _yvexpandpd and _yvgetexpsd reciprocal-estimate tables. +var amd64Reciprocal14Specs = map[Op]amd64Reciprocal14Spec{ + "VRCP14PS": {laneBits: 32}, + "VRCP14PD": {laneBits: 64}, + "VRCP14SS": {laneBits: 32, scalar: true}, + "VRCP14SD": {laneBits: 64, scalar: true}, + "VRSQRT14PS": {laneBits: 32, rsqr: true}, + "VRSQRT14PD": {laneBits: 64, rsqr: true}, + "VRSQRT14SS": {laneBits: 32, scalar: true, rsqr: true}, + "VRSQRT14SD": {laneBits: 64, scalar: true, rsqr: true}, +} + +// lowerReciprocal14 implements the complete Go 1.27 RCP14/RSQRT14 family. +// Packed PS/PD forms accept X/Y/Z register or memory sources, scalar-memory +// broadcast, and K1-K7 merge/.Z masking. Scalar SS/SD forms accept an X or +// scalar-memory source, an X passthrough, and the same masking on amd64; the +// four-operand masked scalar form exceeds the 386 assembler's operand limit. +func (c *amd64Ctx) lowerReciprocal14(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, found := amd64Reciprocal14Specs[Op(baseOp)] + if !found { + return false, false, nil + } + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.sae || properties.rounding != "" || spec.scalar && properties.broadcast { + return true, false, fmt.Errorf("%s %s suffix is absent from its Go 1.27 encoding: %q", c.goarch, baseOp, ins.Raw) + } + if spec.scalar { + return c.lowerScalarReciprocal14(baseOp, spec, properties, ins) + } + return c.lowerPackedReciprocal14(baseOp, spec, properties, ins) +} + +func (c *amd64Ctx) lowerPackedReciprocal14(baseOp string, spec amd64Reciprocal14Spec, properties amd64BinaryFloatingSuffix, ins Instr) (bool, bool, error) { + if len(ins.Args) != 2 && len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects source, [K1-K7,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 3 + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be X, Y, or Z: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's EVEX vector class: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source register must match its destination: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := byteWidth * 8 / spec.laneBits + llvmType := amd64FloatingVectorType(lanes, spec.laneBits) + var sourceValue string + if properties.broadcast { + scalar, err := c.loadFloatingScalarOperand(source, spec.laneBits) + if err != nil { + return true, false, err + } + sourceValue = c.splatFloatingScalar(scalar, lanes, spec.laneBits) + } else { + var err error + sourceValue, err = c.loadFloatingVectorOperand(source, byteWidth, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + } + + passthrough := "zeroinitializer" + mask := "-1" + maskBits := 8 + if spec.laneBits == 32 && byteWidth == 64 { + maskBits = 16 + } + if masked { + loadedMask, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + mask = c.truncateReciprocal14Mask(loadedMask, maskBits) + if !properties.zeroing { + passthrough, err = c.loadFloatingVectorOperand(destination, byteWidth, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + } + } + family := "rcp14" + if spec.rsqr { + family = "rsqrt14" + } + intrinsic := fmt.Sprintf("llvm.x86.avx512.%s.%s.%d", family, map[int]string{32: "ps", 64: "pd"}[spec.laneBits], byteWidth*8) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %s, i%d %s)\n", result, llvmType, intrinsic, llvmType, sourceValue, llvmType, passthrough, maskBits, mask) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", out, llvmType, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarReciprocal14(baseOp string, spec amd64Reciprocal14Spec, properties amd64BinaryFloatingSuffix, ins Instr) (bool, bool, error) { + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("%s %s expects source, passthrough, [K1-K7,] destination: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if c.goarch == "386" && masked { + return true, false, fmt.Errorf("386 %s mask forms exceed the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return true, false, fmt.Errorf("%s %s.Z requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + if masked && !amd64NonzeroKOperand(ins.Args[2]) { + return true, false, fmt.Errorf("%s %s masked form requires K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + passthroughArg := ins.Args[1] + if !c.isGoEVEXVectorRegister(destination, 16) || !c.isGoEVEXVectorRegister(passthroughArg, 16) { + return true, false, fmt.Errorf("%s %s passthrough and destination must be X registers: %q", c.goarch, baseOp, ins.Raw) + } + source := ins.Args[0] + if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, 16) { + return true, false, fmt.Errorf("%s %s source must be an X register or scalar memory: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be an X register or scalar memory: %q", c.goarch, baseOp, ins.Raw) + } + + lanes := 128 / spec.laneBits + llvmType := amd64FloatingVectorType(lanes, spec.laneBits) + scalar, err := c.loadFloatingScalarOperand(source, spec.laneBits) + if err != nil { + return true, false, err + } + sourceValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s zeroinitializer, %s %s, i32 0\n", sourceValue, llvmType, amd64FloatingScalarType(spec.laneBits), scalar) + passthrough, err := c.loadFloatingVectorOperand(passthroughArg, 16, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + merge := "zeroinitializer" + mask := "-1" + if masked { + loadedMask, err := c.loadK(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + mask = c.truncateReciprocal14Mask(loadedMask, 8) + if !properties.zeroing { + merge, err = c.loadFloatingVectorOperand(destination, 16, lanes, spec.laneBits) + if err != nil { + return true, false, err + } + } + } + family := "rcp14" + if spec.rsqr { + family = "rsqrt14" + } + intrinsic := fmt.Sprintf("llvm.x86.avx512.%s.%s", family, map[int]string{32: "ss", 64: "sd"}[spec.laneBits]) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s, %s %%%s, %s %s, i8 %s)\n", result, llvmType, intrinsic, llvmType, passthrough, llvmType, sourceValue, llvmType, merge, mask) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, llvmType, result) + return true, false, c.storeVectorBytes(destination.Reg, 16, "%"+out) +} + +func (c *amd64Ctx) truncateReciprocal14Mask(mask string, bits int) string { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i%d\n", truncated, mask, bits) + return "%" + truncated +} + +// lowerReciprocalEstimate implements the complete Go 1.27 legacy and VEX +// RCP/RSQRT estimate family: packed RCPPS/RSQRTPS/VRCPPS/VRSQRTPS plus scalar +// RCPSS/RSQRTSS/VRCPSS/VRSQRTSS. LLVM's target intrinsics preserve the +// instructions' approximate semantics instead of replacing them with exact +// generic floating-point operations. +func (c *amd64Ctx) lowerReciprocalEstimate(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + raw := strings.ToUpper(string(rawOp)) + base := raw + if dot := strings.IndexByte(raw, '.'); dot >= 0 { + base = raw[:dot] + } + switch base { + case "RCPPS", "RSQRTPS", "VRCPPS", "VRSQRTPS", + "RCPSS", "RSQRTSS", "VRCPSS", "VRSQRTSS": + default: + return false, false, nil + } + if raw != base { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, base, ins.Raw) + } + if strings.HasSuffix(base, "SS") { + return c.lowerScalarReciprocalEstimate(base, ins) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects vector-or-memory source and vector destination: %q", c.goarch, base, ins.Raw) + } + + source, destination := ins.Args[0], ins.Args[1] + byteWidth := amd64VectorByteWidth(destination.Reg) + legacy := base == "RCPPS" || base == "RSQRTPS" + if legacy { + if destination.Kind != OpReg || byteWidth != 16 || !c.isGoLegacyXReg(destination.Reg) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's legacy X register class: %q", c.goarch, base, ins.Raw) + } + if source.Kind == OpReg && !c.isGoLegacyXReg(source.Reg) { + return true, false, fmt.Errorf("%s %s source must be an in-range X register or memory: %q", c.goarch, base, ins.Raw) + } + } else { + if destination.Kind != OpReg || byteWidth != 16 && byteWidth != 32 || !amd64VEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's VEX X/Y register classes: %q", c.goarch, base, ins.Raw) + } + if source.Kind == OpReg && !amd64VEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must match the destination width: %q", c.goarch, base, ins.Raw) + } + } + if source.Kind != OpReg && !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, base, ins.Raw) + } + + lanes := byteWidth / 4 + value, err := c.loadPackedReciprocalSource(source, byteWidth, lanes) + if err != nil { + return true, false, err + } + intrinsic := "llvm.x86.sse.rcp.ps" + if strings.Contains(base, "RSQRT") { + intrinsic = "llvm.x86.sse.rsqrt.ps" + } + if byteWidth == 32 { + intrinsic = "llvm.x86.avx.rcp.ps.256" + if strings.Contains(base, "RSQRT") { + intrinsic = "llvm.x86.avx.rsqrt.ps.256" + } + } + vectorType := fmt.Sprintf("<%d x float>", lanes) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(%s %s)\n", result, vectorType, intrinsic, vectorType, value) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytesValue, vectorType, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+bytesValue) +} + +func (c *amd64Ctx) lowerScalarReciprocalEstimate(base string, ins Instr) (bool, bool, error) { + legacy := base == "RCPSS" || base == "RSQRTSS" + wantArgs := 3 + if legacy { + wantArgs = 2 + } + if len(ins.Args) != wantArgs { + return true, false, fmt.Errorf("%s %s expects %d operands from its Go 1.27 table: %q", c.goarch, base, wantArgs, ins.Raw) + } + + source := ins.Args[0] + destination := ins.Args[len(ins.Args)-1] + passthrough := destination + if !legacy { + passthrough = ins.Args[1] + } + if legacy { + if destination.Kind != OpReg || !c.isGoLegacyXReg(destination.Reg) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's legacy X register class: %q", c.goarch, base, ins.Raw) + } + if source.Kind == OpReg && !c.isGoLegacyXReg(source.Reg) { + return true, false, fmt.Errorf("%s %s source must be an in-range X register or scalar memory: %q", c.goarch, base, ins.Raw) + } + if source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s source uses an out-of-range legacy address register: %q", c.goarch, base, ins.Raw) + } + } else { + if !amd64VEXVectorRegister(passthrough, 16) || !amd64VEXVectorRegister(destination, 16) { + return true, false, fmt.Errorf("%s %s passthrough and destination must be VEX X registers: %q", c.goarch, base, ins.Raw) + } + if source.Kind == OpReg && !amd64VEXVectorRegister(source, 16) { + return true, false, fmt.Errorf("%s %s source must be a VEX X register or scalar memory: %q", c.goarch, base, ins.Raw) + } + } + if source.Kind != OpReg && !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be an X register or scalar memory: %q", c.goarch, base, ins.Raw) + } + + scalar, err := c.loadScalarReciprocalSource(source) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x float> zeroinitializer, float %s, i32 0\n", input, scalar) + intrinsic := "llvm.x86.sse.rcp.ps" + if strings.Contains(base, "RSQRT") { + intrinsic = "llvm.x86.sse.rsqrt.ps" + } + estimates := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <4 x float> @%s(<4 x float> %%%s)\n", estimates, intrinsic, input) + estimate := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x float> %%%s, i32 0\n", estimate, estimates) + upper, err := c.loadFloatingVectorOperand(passthrough, 16, 4, 32) + if err != nil { + return true, false, err + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <4 x float> %s, float %%%s, i32 0\n", result, upper, estimate) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytesValue, result) + return true, false, c.storeVectorBytes(destination.Reg, 16, "%"+bytesValue) +} + +func (c *amd64Ctx) loadScalarReciprocalSource(source Operand) (string, error) { + if source.Kind == OpSym { + // A bare integer is absolute memory in the Go assembler; '$' is the + // immediate spelling and is intentionally rejected by this family. + if address, err := parseInt(source.Sym); err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load float, ptr %s, align 1\n", value, c.ptrFromAddrI64(fmt.Sprintf("%d", address))) + return "%" + value, nil + } + } + return c.loadFloatingScalarOperand(source, 32) +} + +func (c *amd64Ctx) loadPackedReciprocalSource(source Operand, byteWidth, lanes int) (string, error) { + if source.Kind == OpSym { + // As in Go's x86 assembler, a bare integer is an absolute memory + // operand. '$' is required for an immediate. + if address, parseErr := parseInt(source.Sym); parseErr == nil { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load <%d x i8>, ptr %s, align 1\n", bytesValue, byteWidth, c.ptrFromAddrI64(fmt.Sprintf("%d", address))) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %%%s to <%d x float>\n", value, byteWidth, bytesValue, lanes) + return "%" + value, nil + } + } + return c.loadFloatingVectorOperand(source, byteWidth, lanes, 32) +} diff --git a/amd64_lower_reciprocal14_test.go b/amd64_lower_reciprocal14_test.go new file mode 100644 index 000000000..97349dead --- /dev/null +++ b/amd64_lower_reciprocal14_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86Reciprocal14CompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's _yvexpandpd table accepts EVEX X/Y/Z register or memory + // sources, optional scalar-memory broadcast, and optional K1-K7 merge/.Z + // masking. _yvgetexpsd accepts scalar memory/X sources, an X passthrough, + // and the same optional mask. The 386 frontend accepts the unmasked EVEX + // forms but not mask operands, and limits Z registers to Z0-Z7. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 31 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT reciprocal14forms(SB),$0-0\n") + for _, op := range []string{"VRCP14PS", "VRCP14PD", "VRSQRT14PS", "VRSQRT14PD"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s1, %s20\n", op, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s20\n", op, width) + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s20\n", op, width) + } + fmt.Fprintf(&source, "\t%s Z1, Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s 24(AX), Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s.BCST 32(AX), Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s X1, K1, X20\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), K2, Y20\n", op) + fmt.Fprintf(&source, "\t%s.BCST.Z 48(AX), K3, Z%d\n", op, lastZ) + } + for _, op := range []string{"VRCP14SS", "VRCP14SD", "VRSQRT14SS", "VRSQRT14SD"} { + fmt.Fprintf(&source, "\t%s X1, X20, X21\n", op) + fmt.Fprintf(&source, "\t%s 56(AX), X20, X21\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K4, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z 64(AX), X20, K5, X21\n", op) + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "reciprocal14forms": {Name: "reciprocal14forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, intrinsic := range []string{ + "@llvm.x86.avx512.rcp14.ps.128", "@llvm.x86.avx512.rcp14.ps.256", "@llvm.x86.avx512.rcp14.ps.512", + "@llvm.x86.avx512.rcp14.pd.128", "@llvm.x86.avx512.rcp14.pd.256", "@llvm.x86.avx512.rcp14.pd.512", + "@llvm.x86.avx512.rsqrt14.ps.128", "@llvm.x86.avx512.rsqrt14.ps.256", "@llvm.x86.avx512.rsqrt14.ps.512", + "@llvm.x86.avx512.rsqrt14.pd.128", "@llvm.x86.avx512.rsqrt14.pd.256", "@llvm.x86.avx512.rsqrt14.pd.512", + "@llvm.x86.avx512.rcp14.ss", "@llvm.x86.avx512.rcp14.sd", + "@llvm.x86.avx512.rsqrt14.ss", "@llvm.x86.avx512.rsqrt14.sd", + } { + if !strings.Contains(ll, intrinsic) { + t.Fatalf("translation omitted %s:\n%s", intrinsic, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "reciprocal14-"+target.name+".ll", "reciprocal14-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86Reciprocal14RejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VRCP14PS X0, Y1"}, + {goarch: "amd64", instruction: "VRSQRT14PD.Z Z0, Z1"}, + {goarch: "amd64", instruction: "VRCP14PS.BCST X0, X1"}, + {goarch: "amd64", instruction: "VRCP14PD.SAE Z0, Z1"}, + {goarch: "amd64", instruction: "VRSQRT14PS Z0, K0, Z1"}, + {goarch: "amd64", instruction: "VRCP14SS X0, X1"}, + {goarch: "amd64", instruction: "VRCP14SD Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRSQRT14SS.BCST 8(AX), X1, X2"}, + {goarch: "amd64", instruction: "VRSQRT14SD.SAE X0, X1, X2"}, + {goarch: "amd64", instruction: "VRCP14SS.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VRCP14SS X0, X1, K0, X2"}, + {goarch: "386", instruction: "VRCP14SS X0, X1, K1, X2"}, + {goarch: "386", instruction: "VRSQRT14PD Z0, Z8"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's reciprocal14 tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_reciprocal_scalar_test.go b/amd64_lower_reciprocal_scalar_test.go new file mode 100644 index 000000000..680548bb3 --- /dev/null +++ b/amd64_lower_reciprocal_scalar_test.go @@ -0,0 +1,230 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86ScalarReciprocalCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + const vexLast = 15 + var source strings.Builder + source.WriteString("DATA scalarreciprocaldata+0(SB)/4, $0\n") + source.WriteString("GLOBL scalarreciprocaldata(SB), $4\n") + source.WriteString("TEXT scalarreciprocalforms(SB),$0-4\n") + for _, op := range []string{"RCPSS", "RSQRTSS"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s scalarreciprocaldata(SB), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s arg+0(FP), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 0, X%d\n", op, legacyLast) + } + for _, op := range []string{"VRCPSS", "VRSQRTSS"} { + fmt.Fprintf(&source, "\t%s X1, X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 8(AX), X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s scalarreciprocaldata(SB), X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s arg+0(FP), X2, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 0, X2, X%d\n", op, vexLast) + if target.goarch == "386" { + // Go's VEX register classes retain their 16-register + // compatibility closure even in 32-bit mode. + fmt.Fprintf(&source, "\t%s 16(R9), X13, X14\n", op) + } + } + if target.goarch == "amd64" { + for _, op := range []string{"RCPSS", "RSQRTSS"} { + fmt.Fprintf(&source, "\t%s 16(R11)(R12*2), X13\n", op) + } + for _, op := range []string{"VRCPSS", "VRSQRTSS"} { + fmt.Fprintf(&source, "\t%s 16(R11)(R12*2), X13, X14\n", op) + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "scalarreciprocalforms": { + Name: "scalarreciprocalforms", Args: []LLVMType{LLVMType("float")}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: LLVMType("float"), Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "@llvm.x86.sse.rcp.ps", "@llvm.x86.sse.rsqrt.ps", + "extractelement <4 x float>", "insertelement <4 x float>", + `"target-features"="+avx,+sse"`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("scalar reciprocal lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "scalar-reciprocal-"+target.name+".ll", "scalar-reciprocal-"+target.name+".o", ir) + }) + } +} + +func TestAMD64ScalarReciprocalRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT scalarreciprocalsemantics(SB),$0-24 + MOVQ out+0(FP), DI + MOVQ source+8(FP), AX + MOVQ upper+16(FP), BX + + MOVUPS (BX), X0 + STC + RCPSS (AX), X0 + MOVUPS X0, 0(DI) + SETCS 64(DI) + + MOVUPS (BX), X1 + STC + RSQRTSS (AX), X1 + MOVUPS X1, 16(DI) + SETCS 65(DI) + + MOVUPS (BX), X2 + STC + VRCPSS (AX), X2, X3 + MOVUPS X3, 32(DI) + SETCS 66(DI) + + MOVUPS (BX), X4 + STC + VRSQRTSS (AX), X4, X5 + MOVUPS X5, 48(DI) + SETCS 67(DI) + VZEROUPPER + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "scalarreciprocalsemantics": { + Name: "scalarreciprocalsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void scalarreciprocalsemantics(uint8_t *, const float *, const float *); +static int close_enough(float got, float want) { + float delta = got - want; + if (delta < 0) delta = -delta; + return delta < 0.01f; +} +int main(void) { + float source[4] = {4.0f, 100.0f, 200.0f, 300.0f}; + float upper[4] = {99.0f, 10.0f, 20.0f, 30.0f}; + uint8_t out[68] = {0}; + scalarreciprocalsemantics(out, source, upper); + const float *values = (const float *)out; + for (int vector = 0; vector < 4; ++vector) { + float want = (vector & 1) ? 0.5f : 0.25f; + if (!close_enough(values[4*vector], want)) return 10+vector; + if (values[4*vector+1] != 10.0f || values[4*vector+2] != 20.0f || values[4*vector+3] != 30.0f) return 20+vector; + if (out[64+vector] != 1) return 30+vector; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_reciprocal_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86ScalarReciprocalRejectsFormsOutsideGo127Tables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "RCPSS X0"}, + {goarch: "amd64", instruction: "RSQRTSS X0, X1, X2"}, + {goarch: "amd64", instruction: "RCPSS Y0, X1"}, + {goarch: "amd64", instruction: "RCPSS X0, X16"}, + {goarch: "amd64", instruction: "VRCPSS X0, X1"}, + {goarch: "amd64", instruction: "VRSQRTSS X0, X1, Y2"}, + {goarch: "amd64", instruction: "VRCPSS Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRCPSS.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VRSQRTSS X0, X1, K1, X2"}, + {goarch: "386", instruction: "RCPSS X0, X8"}, + {goarch: "386", instruction: "RSQRTSS 8(R9), X0"}, + {goarch: "386", instruction: "VRCPSS X16, X1, X2"}, + {goarch: "386", instruction: "VRSQRTSS X0, X1, X16"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar reciprocal tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_reciprocal_test.go b/amd64_lower_reciprocal_test.go new file mode 100644 index 000000000..55831835c --- /dev/null +++ b/amd64_lower_reciprocal_test.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedReciprocalCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's legacy yxm table accepts X/m128 -> X, while the VEX + // _yvptest table accepts both X/m128 -> X and Y/m256 -> Y. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + const vexLast = 15 + var source strings.Builder + source.WriteString("TEXT packedreciprocalforms(SB),$0-0\n") + for _, op := range []string{"RCPPS", "RSQRTPS"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 0, X%d\n", op, legacyLast) + } + for _, op := range []string{"VRCPPS", "VRSQRTPS"} { + fmt.Fprintf(&source, "\t%s X1, X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s Y1, Y%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 16(AX), Y%d\n", op, vexLast) + fmt.Fprintf(&source, "\t%s 0, Y%d\n", op, vexLast) + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedreciprocalforms": {Name: "packedreciprocalforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, intrinsic := range []string{"@llvm.x86.sse.rcp.ps", "@llvm.x86.sse.rsqrt.ps", "@llvm.x86.avx.rcp.ps.256", "@llvm.x86.avx.rsqrt.ps.256"} { + if !strings.Contains(ll, intrinsic) { + t.Fatalf("translation omitted %s:\n%s", intrinsic, ll) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-reciprocal-"+target.name+".ll", "packed-reciprocal-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedReciprocalRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "RCPPS Y0, X1"}, + {goarch: "amd64", instruction: "RSQRTPS X0, X16"}, + {goarch: "amd64", instruction: "VRCPPS X16, X1"}, + {goarch: "amd64", instruction: "VRCPPS Y0, X1"}, + {goarch: "amd64", instruction: "VRSQRTPS Z0, Z1"}, + {goarch: "amd64", instruction: "VRCPPS.Z X0, X1"}, + {goarch: "amd64", instruction: "VRSQRTPS X0, K1, X1"}, + {goarch: "386", instruction: "RCPPS X0, X8"}, + {goarch: "386", instruction: "VRCPPS X16, X0"}, + {goarch: "386", instruction: "VRCPPS Y16, Y0"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's reciprocal tables for %s", test.instruction, test.goarch) + } + }) + } +} diff --git a/amd64_lower_rorx_test.go b/amd64_lower_rorx_test.go new file mode 100644 index 000000000..6387c143b --- /dev/null +++ b/amd64_lower_rorx_test.go @@ -0,0 +1,228 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawRORXReportedEncoding(t *testing.T) { + const source = ` +TEXT rawRORX(SB),$0-0 + BYTE $0xc4 + BYTE $0xe3 + BYTE $0x7b + BYTE $0xf0 + BYTE $0xe8 + BYTE $0x19 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "386", triple: "i686-pc-windows-msvc"}, + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawRORX": {Name: "rawRORX", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `"target-features"="+bmi2"`) { + t.Fatalf("raw RORX did not infer BMI2 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-rorx.ll", "raw-rorx.o", ir) + }) + } +} + +func TestX86RORXGo127CompleteNamedForms(t *testing.T) { + const source = ` +TEXT rorxforms(SB),$0-0 + RORXL $7, (BX), DX + RORXL $31, R11, R12 + RORXQ $7, 16(BX)(SI*2), DX + RORXQ $63, R11, R12 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rorxforms": {Name: "rorxforms", Ret: Void}}, + }); err != nil { + t.Fatal(err) + } +} + +func TestX86RORXCompleteGo127FormsAcrossTargets(t *testing.T) { + registers386 := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} + registersAMD64 := append(append([]string{}, registers386...), "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15") + targets := []struct { + name string + goarch string + triple string + registers []string + }{ + {name: "386-linux", goarch: "386", triple: "i386-unknown-linux-gnu", registers: registers386}, + {name: "386-windows", goarch: "386", triple: "i686-pc-windows-msvc", registers: registers386}, + {name: "amd64-darwin", goarch: "amd64", triple: "x86_64-apple-darwin", registers: registersAMD64}, + {name: "amd64-linux", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registers: registersAMD64}, + {name: "amd64-windows", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registers: registersAMD64}, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rorxall(SB),$0-0\n") + for _, width := range []string{"L", "Q"} { + for index, sourceRegister := range target.registers { + destination := target.registers[(index+3)%len(target.registers)] + fmt.Fprintf(&source, "\tRORX%s $%d, %s, %s\n", width, index*17&255, sourceRegister, destination) + } + fmt.Fprintf(&source, "\tRORX%s $-1, 8(AX), DX\n", width) + fmt.Fprintf(&source, "\tRORX%s $63, 16(AX)(CX*4), BX\n", width) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rorxall": {Name: "rorxall", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `"target-features"="+bmi2"`) { + t.Fatalf("RORX family did not infer BMI2 target feature:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "rorx-family.ll", "rorx-family.o", ir) + }) + } +} + +func TestX86RawRORXCompleteRegisterFields(t *testing.T) { + for _, mode := range []int{32, 64} { + registerCount := 8 + if mode == 64 { + registerCount = 16 + } + for _, width64 := range []bool{false, true} { + for source := 0; source < registerCount; source++ { + for destination := 0; destination < registerCount; destination++ { + for _, immediate := range []byte{0, 1, 31, 32, 63, 255} { + p0 := byte((^destination>>3)&1)<<7 | 1<<6 | byte((^source>>3)&1)<<5 | 3 + p1 := byte(0x7b) + if width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + encoding := []byte{0xc4, p0, p1, 0xf0, modRM, immediate} + instruction, length, ok, err := decodedX86RORXInstruction(encoding, mode) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("mode=%d width64=%v source=%d destination=%d immediate=%d: ok=%v length=%d err=%v", mode, width64, source, destination, immediate, ok, length, err) + } + wantOp := Op("RORXL") + if width64 && mode == 64 { + wantOp = "RORXQ" + } + wantSource, _ := decodedX86GeneralRegister(source) + wantDestination, _ := decodedX86GeneralRegister(destination) + if instruction.Op != wantOp || len(instruction.Args) != 3 || instruction.Args[0].Imm != int64(immediate) || instruction.Args[1].Reg != wantSource || instruction.Args[2].Reg != wantDestination { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + } + } + } + } + } +} + +func TestX86RawRORXMemoryForms(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + }{ + {name: "direct", encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0x13, 0x07}}, + {name: "extended base", encoding: []byte{0xc4, 0xc3, 0x7b, 0xf0, 0x13, 0x07}}, + {name: "sib disp8", encoding: []byte{0xc4, 0x83, 0xfb, 0xf0, 0x54, 0x88, 0x10, 0xff}}, + {name: "base less", encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0x14, 0x25, 0x78, 0x56, 0x34, 0x12, 0x1f}}, + {name: "segment", encoding: []byte{0x64, 0xc4, 0xe3, 0x7b, 0xf0, 0x54, 0x48, 0x08, 0x1f}}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86RORXInstruction(test.encoding, 64) + if !ok || err != nil || length != len(test.encoding) || len(instruction.Args) != 3 || instruction.Args[1].Kind != OpMem { + t.Fatalf("encoding %x: instruction=%#v ok=%v length=%d err=%v", test.encoding, instruction, ok, length, err) + } + }) + } +} + +func TestX86RawRORXRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex l", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7f, 0xf0, 0xd2, 0x07}}, + {name: "vex pp", mode: 64, encoding: []byte{0xc4, 0xe3, 0x79, 0xf0, 0xd2, 0x07}}, + {name: "vex vvvv", mode: 64, encoding: []byte{0xc4, 0xe3, 0x73, 0xf0, 0xd2, 0x07}}, + {name: "missing modrm", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0}}, + {name: "missing immediate", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0xd2}}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe3, 0x7b, 0xf0, 0xd2, 0x07}}, + {name: "rip relative", mode: 64, encoding: []byte{0xc4, 0xe3, 0x7b, 0xf0, 0x15, 0, 0, 0, 0, 0x07}}, + {name: "extended source 386", mode: 32, encoding: []byte{0xc4, 0xc3, 0x7b, 0xf0, 0xc0, 0x07}}, + {name: "extended destination 386", mode: 32, encoding: []byte{0xc4, 0x63, 0x7b, 0xf0, 0xc0, 0x07}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86RORXInstruction(test.encoding, test.mode) + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} + +func TestX86RORXGo127ModeOracle(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + want bool + }{ + {goarch: "386", instruction: "RORXL $7, (BX), DX", want: true}, + {goarch: "386", instruction: "RORXQ $7, (BX), DX", want: true}, + {goarch: "386", instruction: "RORXL $7, R8, DX", want: false}, + {goarch: "amd64", instruction: "RORXQ $7, R8, R15", want: true}, + } { + t.Run(fmt.Sprintf("%s/%s", test.goarch, test.instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, test.goarch, "TEXT oracle(SB),$0-0\n\t"+test.instruction+"\n\tRET\n", test.want) + }) + } +} diff --git a/amd64_lower_round.go b/amd64_lower_round.go new file mode 100644 index 000000000..dea3e4125 --- /dev/null +++ b/amd64_lower_round.go @@ -0,0 +1,153 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64RoundSpec struct { + laneBits int + scalar bool + vector bool +} + +// amd64RoundSpecs mirrors Go 1.27's yxshuf, _yvroundpd, and _yvdppd tables. +// The legacy family is X-only, the packed VEX family is X/Y, and the scalar +// VEX family has an explicit upper-lane source. +var amd64RoundSpecs = map[Op]amd64RoundSpec{ + "ROUNDPS": {laneBits: 32}, + "ROUNDPD": {laneBits: 64}, + "ROUNDSS": {laneBits: 32, scalar: true}, + "ROUNDSD": {laneBits: 64, scalar: true}, + "VROUNDPS": {laneBits: 32, vector: true}, + "VROUNDPD": {laneBits: 64, vector: true}, + "VROUNDSS": {laneBits: 32, scalar: true, vector: true}, + "VROUNDSD": {laneBits: 64, scalar: true, vector: true}, +} + +func (c *amd64Ctx) lowerRound(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + raw := strings.ToUpper(string(rawOp)) + if strings.Contains(raw, ".") { + base := Op(strings.SplitN(raw, ".", 2)[0]) + if _, recognized := amd64RoundSpecs[base]; recognized { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, base, ins.Raw) + } + return false, false, nil + } + spec, recognized := amd64RoundSpecs[Op(raw)] + if !recognized { + return false, false, nil + } + wantArgs := 3 + if spec.vector && spec.scalar { + wantArgs = 4 + } + if len(ins.Args) != wantArgs { + return true, false, fmt.Errorf("%s %s expects immediate, source, %sdestination: %q", c.goarch, raw, map[bool]string{true: "upper-lane source, "}[spec.vector && spec.scalar], ins.Raw) + } + if ins.Args[0].Kind != OpImm { + return true, false, fmt.Errorf("%s %s expects an 8-bit immediate: %q", c.goarch, raw, ins.Raw) + } + imm := ins.Args[0].Imm + if (!spec.vector && (imm < 0 || imm > 255)) || (spec.vector && (imm < -128 || imm > 255)) { + return true, false, fmt.Errorf("%s %s immediate is outside Go 1.27's table: %q", c.goarch, raw, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return true, false, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, raw, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + if spec.scalar { + if byteWidth != 16 || !c.isGoVEXVectorRegister(destination, 16, false) { + return true, false, fmt.Errorf("%s %s destination must be an X register in the Go VEX class: %q", c.goarch, raw, ins.Raw) + } + } else if (!spec.vector && byteWidth != 16) || (spec.vector && byteWidth != 16 && byteWidth != 32) || !c.isGoVEXVectorRegister(destination, byteWidth, false) { + return true, false, fmt.Errorf("%s %s destination has a width outside Go 1.27's table: %q", c.goarch, raw, ins.Raw) + } + source := ins.Args[1] + if !c.isGoVEXVectorRegister(source, byteWidth, true) { + return true, false, fmt.Errorf("%s %s source must be same-width X/Y or memory: %q", c.goarch, raw, ins.Raw) + } + if spec.vector && spec.scalar { + upper := ins.Args[2] + if !c.isGoVEXVectorRegister(upper, 16, false) { + return true, false, fmt.Errorf("%s %s upper-lane source must be an X register: %q", c.goarch, raw, ins.Raw) + } + } + intrinsic := amd64RoundIntrinsic(uint8(imm)) + if spec.scalar { + return c.lowerScalarRound(spec, ins, destination, intrinsic) + } + return c.lowerPackedRound(spec, source, destination, byteWidth, intrinsic) +} + +func amd64RoundIntrinsic(imm uint8) string { + if imm&4 != 0 { + return "rint" + } + switch imm & 3 { + case 0: + return "roundeven" + case 1: + return "floor" + case 2: + return "ceil" + default: + return "trunc" + } +} + +func (c *amd64Ctx) lowerScalarRound(spec amd64RoundSpec, ins Instr, destination Operand, intrinsic string) (bool, bool, error) { + value, err := c.evalScalarFloatForInteger(ins.Args[1], spec.laneBits) + if err != nil { + return true, false, err + } + rounded := c.emitScalarFloatRound(value, spec.laneBits, intrinsic) + bits := c.newTmp() + floatingType := amd64FloatingTypeForBits(spec.laneBits) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to i%d\n", bits, floatingType, rounded, spec.laneBits) + baseReg := destination.Reg + if spec.vector { + baseReg = ins.Args[2].Reg + } + baseBytes, err := c.loadX(baseReg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, baseBytes) + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+bits, base, "", false) +} + +func (c *amd64Ctx) lowerPackedRound(spec amd64RoundSpec, source, destination Operand, byteWidth int, intrinsic string) (bool, bool, error) { + bytesValue, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / spec.laneBits + floatingType := fmt.Sprintf("<%d x %s>", lanes, amd64FloatingTypeForBits(spec.laneBits)) + floating := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", floating, byteWidth, bytesValue, floatingType) + result := "poison" + for lane := 0; lane < lanes; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", value, floatingType, floating, lane) + rounded := c.emitScalarFloatRound("%"+value, spec.laneBits, intrinsic) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, %s %s, i32 %d\n", inserted, floatingType, result, amd64FloatingTypeForBits(spec.laneBits), rounded, lane) + result = "%" + inserted + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, floatingType, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitScalarFloatRound(value string, bits int, intrinsic string) string { + typeName := amd64FloatingTypeForBits(bits) + intrinsicSuffix := "f64" + if bits == 32 { + intrinsicSuffix = "f32" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @llvm.%s.%s(%s %s)\n", result, typeName, intrinsic, intrinsicSuffix, typeName, value) + return "%" + result +} diff --git a/amd64_lower_round_test.go b/amd64_lower_round_test.go new file mode 100644 index 000000000..18a77c76f --- /dev/null +++ b/amd64_lower_round_test.go @@ -0,0 +1,209 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86RoundCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT roundforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"ROUNDPS", "ROUNDPD", "ROUNDSS", "ROUNDSD"} { + fmt.Fprintf(&source, "\t%s $0, X0, X1\n", op) + fmt.Fprintf(&source, "\t%s $255, 8(BX), X2\n", op) + } + for _, op := range []string{"VROUNDPS", "VROUNDPD"} { + fmt.Fprintf(&source, "\t%s $0, X0, X1\n", op) + fmt.Fprintf(&source, "\t%s $-1, 16(BX), X2\n", op) + fmt.Fprintf(&source, "\t%s $255, Y0, Y1\n", op) + fmt.Fprintf(&source, "\t%s $2, 32(BX), Y2\n", op) + } + for _, op := range []string{"VROUNDSS", "VROUNDSD"} { + fmt.Fprintf(&source, "\t%s $0, X0, X1, X2\n", op) + fmt.Fprintf(&source, "\t%s $-1, 8(BX), X1, X2\n", op) + fmt.Fprintf(&source, "\t%s $255, X0, X1, X2\n", op) + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"roundforms": {Name: "roundforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "round-"+target.name+".ll", "round-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86RoundRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDSD $-1, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDSS $256, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDPD $0, Y0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDPS $0, X0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "ROUNDSS.Z $0, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDPS $-129, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDPD $256, Y0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDPS $0, X0, Y1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDSS $0, X0, X1"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDSD $0, X0, Y1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VROUNDSS $0, X0, X1, Y2"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "ROUNDSS $0, X8, X0"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "VROUNDPS $0, Y8, Y0"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "VROUNDSD $0, X0, X8, X1"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+test.instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ROUND tables for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64RoundRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT roundsemantics(SB),NOSPLIT,$0-40 + MOVQ out+0(FP), AX + MOVQ a32+8(FP), BX + MOVQ a64+16(FP), CX + MOVQ upper32+24(FP), DX + MOVQ upper64+32(FP), SI + ROUNDPS $0, (BX), X0 + MOVUPS X0, 0(AX) + ROUNDPS $1, (BX), X0 + MOVUPS X0, 16(AX) + ROUNDPS $2, (BX), X0 + MOVUPS X0, 32(AX) + ROUNDPS $3, (BX), X0 + MOVUPS X0, 48(AX) + ROUNDPS $4, (BX), X0 + MOVUPS X0, 64(AX) + MOVUPS (DX), X1 + VROUNDSS $1, (BX), X1, X2 + MOVUPS X2, 80(AX) + ROUNDPD $0, (CX), X0 + MOVUPD X0, 96(AX) + ROUNDPD $1, (CX), X0 + MOVUPD X0, 112(AX) + ROUNDPD $2, (CX), X0 + MOVUPD X0, 128(AX) + ROUNDPD $3, (CX), X0 + MOVUPD X0, 144(AX) + ROUNDPD $4, (CX), X0 + MOVUPD X0, 160(AX) + MOVUPD (SI), X1 + VROUNDSD $2, (CX), X1, X2 + MOVUPD X2, 176(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "roundsemantics": {Name: "roundsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void roundsemantics(uint8_t *, const float *, const double *, const float *, const double *); +static int check32(const float *got, const float *want, int base) { + for (int i = 0; i < 4; i++) if (got[i] != want[i]) return base+i; + return 0; +} +static int check64(const double *got, const double *want, int base) { + for (int i = 0; i < 2; i++) if (got[i] != want[i]) return base+i; + return 0; +} +int main(void) { + float a32[4] = {1.5f, -1.5f, 2.75f, -2.75f}; + double a64[2] = {2.5, -2.5}; + float upper32[4] = {99.0f, 10.0f, 20.0f, 30.0f}; + double upper64[2] = {99.0, 40.0}; + float want32[6][4] = { + {2, -2, 3, -3}, {1, -2, 2, -3}, {2, -1, 3, -2}, + {1, -1, 2, -2}, {2, -2, 3, -3}, {1, 10, 20, 30}, + }; + double want64[6][2] = { + {2, -2}, {2, -3}, {3, -2}, {2, -2}, {2, -2}, {3, 40}, + }; + uint8_t out[192] = {0}; + roundsemantics(out, a32, a64, upper32, upper64); + for (int i = 0; i < 6; i++) { + int failed = check32((const float *)(out + 16*i), want32[i], 10+4*i); + if (failed) return failed; + } + for (int i = 0; i < 6; i++) { + int failed = check64((const double *)(out + 96 + 16*i), want64[i], 40+2*i); + if (failed) return failed; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "round_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_rtm.go b/amd64_lower_rtm.go new file mode 100644 index 000000000..4456e61c0 --- /dev/null +++ b/amd64_lower_rtm.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64RTMKind uint8 + +const ( + amd64RTMBegin amd64RTMKind = iota + amd64RTMAbort + amd64RTMEnd + amd64RTMTest +) + +type amd64RTMOperand uint8 + +const ( + amd64RTMNoOperand amd64RTMOperand = iota + amd64RTMBranch + amd64RTMUnsignedByte +) + +type amd64RTMSpec struct { + kind amd64RTMKind + operand amd64RTMOperand + outputAX bool +} + +// amd64RTMSpecs is the complete Go 1.27 Restricted Transactional Memory +// grammar. XBEGIN is included even though Go's positive encoder corpus keeps +// its cases as TODOs: the optab accepts one Ybr target and writes abort status +// to EAX. XABORT accepts the complete Yu8 immediate domain. +var amd64RTMSpecs = map[Op]amd64RTMSpec{ + "XBEGIN": {kind: amd64RTMBegin, operand: amd64RTMBranch, outputAX: true}, + "XABORT": {kind: amd64RTMAbort, operand: amd64RTMUnsignedByte}, + "XEND": {kind: amd64RTMEnd}, + "XTEST": {kind: amd64RTMTest}, +} + +type amd64RTMForm struct { + branch Operand + immediate uint8 +} + +func (c *amd64Ctx) lowerRTM(bi int, ii int, op Op, ins Instr, emitCondBr amd64EmitCondBr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64RTMSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + form, err := c.parseRTMForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + return c.emitRTM(bi, ii, baseOp, spec, form, emitCondBr) +} + +func (c *amd64Ctx) parseRTMForm(op string, spec amd64RTMSpec, ins Instr) (amd64RTMForm, error) { + var form amd64RTMForm + switch spec.operand { + case amd64RTMNoOperand: + if len(ins.Args) != 0 { + return form, fmt.Errorf("%s %s takes no operands: %q", c.goarch, op, ins.Raw) + } + case amd64RTMBranch: + if len(ins.Args) != 1 { + return form, fmt.Errorf("%s %s expects one Go 1.27 Ybr target: %q", c.goarch, op, ins.Raw) + } + form.branch = ins.Args[0] + case amd64RTMUnsignedByte: + if len(ins.Args) != 1 || ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return form, fmt.Errorf("%s %s expects one Go 1.27 Yu8 immediate: %q", c.goarch, op, ins.Raw) + } + form.immediate = uint8(ins.Args[0].Imm) + default: + panic("unknown amd64 RTM operand kind") + } + return form, nil +} + +func (c *amd64Ctx) rtmBranchTarget(bi int, ii int, operand Operand) (string, error) { + knownBlock := func(name string) bool { + for _, block := range c.blocks { + if block.name == name { + return true + } + } + return false + } + var target string + switch operand.Kind { + case OpIdent: + target = operand.Ident + case OpReg: + target = string(operand.Reg) + if !knownBlock(target) { + return "", fmt.Errorf("%s XBEGIN register is not a local branch target: %q", c.goarch, target) + } + case OpSym: + target = strings.TrimSpace(operand.Sym) + if strings.HasSuffix(target, "(SB)") { + return "", fmt.Errorf("%s XBEGIN target must be local: %q", c.goarch, target) + } + target = strings.TrimSuffix(target, "<>") + case OpMem: + if !strings.EqualFold(string(operand.Mem.Base), "PC") { + return "", fmt.Errorf("%s XBEGIN expects a label or PC-relative target", c.goarch) + } + current := c.blockBase[bi] + ii + index := current + int(operand.Mem.Off) + targetBlock, ok := c.blockByIdx[index] + if !ok || targetBlock < 0 || targetBlock >= len(c.blocks) { + return "", fmt.Errorf("%s XBEGIN invalid PC-relative target %d(PC)", c.goarch, operand.Mem.Off) + } + target = c.blocks[targetBlock].name + default: + return "", fmt.Errorf("%s XBEGIN expects one Go 1.27 Ybr target", c.goarch) + } + if target == "" || !knownBlock(target) { + return "", fmt.Errorf("%s XBEGIN unknown local target %q", c.goarch, target) + } + return target, nil +} + +func (c *amd64Ctx) emitRTM(bi int, ii int, op string, spec amd64RTMSpec, form amd64RTMForm, emitCondBr amd64EmitCondBr) (bool, bool, error) { + switch spec.kind { + case amd64RTMBegin: + if bi+1 >= len(c.blocks) { + return true, false, fmt.Errorf("%s XBEGIN has no success fallthrough", c.goarch) + } + target, err := c.rtmBranchTarget(bi, ii, form.branch) + if err != nil { + return true, false, err + } + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.xbegin()\n", status) + if err := c.storeRegSized(AX, I32, "%"+status); err != nil { + return true, false, err + } + success := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, -1\n", success, status) + if err := emitCondBr("%"+success, c.blocks[bi+1].name, target); err != nil { + return true, false, err + } + return true, true, nil + case amd64RTMAbort: + fmt.Fprintf(c.b, " call void @llvm.x86.xabort(i8 %d)\n", form.immediate) + case amd64RTMEnd: + c.b.WriteString(" call void @llvm.x86.xend()\n") + case amd64RTMTest: + active := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.xtest()\n", active) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, 0\n", zero, active) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + c.clearX86StatusFlags(x86StatusAll &^ x86StatusZF) + default: + panic("unknown amd64 RTM kind") + } + return true, false, nil +} diff --git a/amd64_lower_rtm_test.go b/amd64_lower_rtm_test.go new file mode 100644 index 000000000..82ff3d134 --- /dev/null +++ b/amd64_lower_rtm_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64RTMGrammarIsComplete(t *testing.T) { + want := map[Op]amd64RTMSpec{ + "XBEGIN": {kind: amd64RTMBegin, operand: amd64RTMBranch, outputAX: true}, + "XABORT": {kind: amd64RTMAbort, operand: amd64RTMUnsignedByte}, + "XEND": {kind: amd64RTMEnd}, + "XTEST": {kind: amd64RTMTest}, + } + if !reflect.DeepEqual(amd64RTMSpecs, want) { + t.Fatalf("RTM grammar = %+v, want %+v", amd64RTMSpecs, want) + } +} + +func TestTranslateX86RTMCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT rtmforms(SB),$0-0 + XBEGIN fallback + XTEST + XEND + RET +fallback: + XABORT $0 + XABORT $255 + RET +TEXT rtmpcrel(SB),$0-0 + XBEGIN 2(PC) + XEND + RET +` + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "rtmforms": {Name: "rtmforms", Ret: Void}, + "rtmpcrel": {Name: "rtmpcrel", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `"target-features"="+rtm"`, + `call i32 @llvm.x86.xbegin()`, + `call i32 @llvm.x86.xtest()`, + `call void @llvm.x86.xend()`, + `call void @llvm.x86.xabort(i8 0)`, + `call void @llvm.x86.xabort(i8 -1)`, + `icmp eq i32`, + `store i1 %`, + `br i1 %`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "rtm-"+target.name+".ll", "rtm-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86RTMRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "XBEGIN AX"}, + {goarch: "amd64", instruction: "XBEGIN (BX)"}, + {goarch: "amd64", instruction: "XABORT"}, + {goarch: "amd64", instruction: "XABORT $-1"}, + {goarch: "amd64", instruction: "XABORT $256"}, + {goarch: "amd64", instruction: "XABORT AX"}, + {goarch: "amd64", instruction: "XEND AX"}, + {goarch: "amd64", instruction: "XTEST.P"}, + {goarch: "386", instruction: "XABORT $256"}, + {goarch: "386", instruction: "XTEST AX"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's RTM tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_sad.go b/amd64_lower_sad.go new file mode 100644 index 000000000..3cddf3d36 --- /dev/null +++ b/amd64_lower_sad.go @@ -0,0 +1,254 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedSADKind uint8 + +const ( + amd64PackedSADBlockSum amd64PackedSADKind = iota + amd64PackedSADMultiple + amd64PackedSADDoubleBlock +) + +type amd64PackedSADForm uint8 + +const ( + amd64PackedSADLegacyX amd64PackedSADForm = iota + amd64PackedSADVEXXY + amd64PackedSADEVEXXYZ +) + +type amd64PackedSADSpec struct { + kind amd64PackedSADKind + form amd64PackedSADForm +} + +// amd64PackedSADSpecs is the complete Go 1.27 byte-SAD grammar. PSADBW uses +// fixed eight-byte block sums, MPSADBW uses one immediate-selected four-byte +// block and sliding windows, and VDBPSADBW independently selects four dwords +// before its sliding SAD. Their vector spellings intentionally differ: +// VPSADBW has EVEX X/Y/Z rows, VMPSADBW is VEX X/Y only, and VDBPSADBW adds +// EVEX writemasking to X/Y/Z. +var amd64PackedSADSpecs = map[Op]amd64PackedSADSpec{ + "PSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADLegacyX}, + "VPSADBW": {kind: amd64PackedSADBlockSum, form: amd64PackedSADEVEXXYZ}, + "MPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADLegacyX}, + "VMPSADBW": {kind: amd64PackedSADMultiple, form: amd64PackedSADVEXXY}, + "VDBPSADBW": {kind: amd64PackedSADDoubleBlock, form: amd64PackedSADEVEXXYZ}, +} + +// One ordered grammar governs both lowering and original-source effects. +func (c *amd64Ctx) lowerPackedSumAbsoluteDifferences(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64PackedSADSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("packed SAD dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, _, err := x86PackedSADProperties(ins, c.goarch) + if err != nil { + return true, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: %s m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext, ins.Op) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + b, err := c.loadPackedCompareBytes(second, form.byteWidth) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 { + // Keep the precise legacy opcode/read/fault behavior. A separately + // synthesized align-1 or MOVDQA load is not an equivalent contract. + result, err = c.emitPackedSADNativeMemory(first, b, form) + } else { + a, loadErr := c.loadPackedUnpackSource(first, x86PackedUnpackForm{ + x86PackedBinaryForm: form.x86PackedBinaryForm, spec: x86PackedUnpackSpec{laneBits: 8}, + }) + if loadErr != nil { + return true, false, loadErr + } + switch form.spec.kind { + case amd64PackedSADBlockSum: + result = c.emitPackedSumAbsoluteByteDifferences(form.byteWidth, a, b) + case amd64PackedSADMultiple: + result = c.emitMultipleSumAbsoluteDifferences(form.byteWidth, a, b, form.immediate) + case amd64PackedSADDoubleBlock: + words := c.emitDoubleBlockSumAbsoluteDifferences(form.byteWidth, b, a, form.immediate) + if form.mask >= 0 { + mask, loadErr := c.loadK(ins.Args[form.mask].Reg) + if loadErr != nil { + return true, false, loadErr + } + old := "zeroinitializer" + if !form.zeroing { + old, loadErr = c.loadPackedCompareLanes(destination, form.byteWidth, 16, false) + if loadErr != nil { + return true, false, loadErr + } + } + words = amd64ApplyIntegerLaneMask(c, form.byteWidth/2, 16, words, old, mask, form.zeroing) + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %s to <%d x i8>\n", bytes, form.byteWidth/2, words, form.byteWidth) + result = "%" + bytes + } + } + if err != nil { + return true, false, err + } + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, result) +} + +func (c *amd64Ctx) emitPackedSADNativeMemory(source Operand, data string, form x86PackedSADForm) (string, error) { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + switch pointerType { + case "ptr addrspace(256)": + segment = "%gs:" + case "ptr addrspace(257)": + segment = "%fs:" + } + opcode := "psadbw " + if form.spec.kind == amd64PackedSADMultiple { + opcode = fmt.Sprintf("mpsadbw $$%d, ", form.immediate) + } + carrier, value, bytes := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", carrier, data) + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, %q(<4 x i32> %%%s, %s %s)\n", + value, opcode+segment+"($2), $0", "=x,0,r,~{memory}", carrier, pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bytes, value) + return "%" + bytes, nil +} + +func (c *amd64Ctx) emitDoubleBlockSumAbsoluteDifferences(byteWidth int, source1, source2 string, immediate uint8) string { + shuffled := "poison" + for laneBase := 0; laneBase < byteWidth; laneBase += 16 { + for destinationDWord := 0; destinationDWord < 4; destinationDWord++ { + sourceDWord := int(immediate>>(2*destinationDWord)) & 3 + for byteInDWord := 0; byteInDWord < 4; byteInDWord++ { + sourceIndex := laneBase + sourceDWord*4 + byteInDWord + destinationIndex := laneBase + destinationDWord*4 + byteInDWord + value, inserted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", value, byteWidth, source2, sourceIndex) + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i8> %s, i8 %%%s, i32 %d\n", inserted, byteWidth, shuffled, value, destinationIndex) + shuffled = "%" + inserted + } + } + } + + lanes := byteWidth / 2 + result := "poison" + for blockBase := 0; blockBase < byteWidth; blockBase += 8 { + for outputInBlock := 0; outputInBlock < 4; outputInBlock++ { + source1Base := blockBase + if outputInBlock >= 2 { + source1Base += 4 + } + tmpBase := blockBase + outputInBlock + sum := "" + for byteInDWord := 0; byteInDWord < 4; byteInDWord++ { + firstByte, secondByte := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", firstByte, byteWidth, source1, source1Base+byteInDWord) + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", secondByte, byteWidth, shuffled, tmpBase+byteInDWord) + firstWide, secondWide := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i16\n", firstWide, firstByte) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i16\n", secondWide, secondByte) + firstMinusSecond, secondMinusFirst, firstGreater, difference := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i16 %%%s, %%%s\n", firstMinusSecond, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = sub i16 %%%s, %%%s\n", secondMinusFirst, secondWide, firstWide) + fmt.Fprintf(c.b, " %%%s = icmp uge i16 %%%s, %%%s\n", firstGreater, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %%%s, i16 %%%s\n", difference, firstGreater, firstMinusSecond, secondMinusFirst) + if sum == "" { + sum = "%" + difference + } else { + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i16 %s, %%%s\n", added, sum, difference) + sum = "%" + added + } + } + outputLane := blockBase/2 + outputInBlock + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i16> %s, i16 %s, i32 %d\n", inserted, lanes, result, sum, outputLane) + result = "%" + inserted + } + } + return result +} + +func (c *amd64Ctx) emitMultipleSumAbsoluteDifferences(byteWidth int, first, second string, immediate uint8) string { + intrinsic := "llvm.x86.sse41.mpsadbw" + lanes := 8 + if byteWidth == 32 { + intrinsic = "llvm.x86.avx2.mpsadbw" + lanes = 16 + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <%d x i16> @%s(<%d x i8> %s, <%d x i8> %s, i8 %d)\n", result, lanes, intrinsic, byteWidth, second, byteWidth, first, immediate) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i16> %%%s to <%d x i8>\n", bytesValue, lanes, result, byteWidth) + return "%" + bytesValue +} + +func (c *amd64Ctx) emitPackedSumAbsoluteByteDifferences(byteWidth int, first, second string) string { + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext <%d x i8> %s to <%d x i16>\n", firstWide, byteWidth, first, byteWidth) + fmt.Fprintf(c.b, " %%%s = zext <%d x i8> %s to <%d x i16>\n", secondWide, byteWidth, second, byteWidth) + firstMinusSecond := c.newTmp() + secondMinusFirst := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub <%d x i16> %%%s, %%%s\n", firstMinusSecond, byteWidth, firstWide, secondWide) + fmt.Fprintf(c.b, " %%%s = sub <%d x i16> %%%s, %%%s\n", secondMinusFirst, byteWidth, secondWide, firstWide) + firstGreater := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge <%d x i16> %%%s, %%%s\n", firstGreater, byteWidth, firstWide, secondWide) + differences := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, <%d x i16> %%%s, <%d x i16> %%%s\n", differences, byteWidth, firstGreater, byteWidth, firstMinusSecond, byteWidth, secondMinusFirst) + + groups := byteWidth / 8 + result := "zeroinitializer" + for group := 0; group < groups; group++ { + sum := "" + for lane := group * 8; lane < (group+1)*8; lane++ { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i16> %%%s, i32 %d\n", value, byteWidth, differences, lane) + if sum == "" { + sum = "%" + value + continue + } + added := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i16 %s, %%%s\n", added, sum, value) + sum = "%" + added + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i16 %s to i64\n", wide, sum) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i64> %s, i64 %%%s, i32 %d\n", inserted, groups, result, wide, group) + result = "%" + inserted + } + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i64> %s to <%d x i8>\n", bytesValue, groups, result, byteWidth) + return "%" + bytesValue +} diff --git a/amd64_lower_scalar_float_to_int.go b/amd64_lower_scalar_float_to_int.go new file mode 100644 index 000000000..a0e16e39a --- /dev/null +++ b/amd64_lower_scalar_float_to_int.go @@ -0,0 +1,81 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ScalarFloatToIntSpec struct { + inputBits int + outputBits int + truncating bool + unsigned bool + vector bool +} + +var amd64ScalarFloatToIntSpecs = map[Op]amd64ScalarFloatToIntSpec{ + "CVTSS2SL": {inputBits: 32, outputBits: 32}, + "CVTSS2SQ": {inputBits: 32, outputBits: 64}, + "CVTSD2SL": {inputBits: 64, outputBits: 32}, + "CVTSD2SQ": {inputBits: 64, outputBits: 64}, + "CVTTSS2SL": {inputBits: 32, outputBits: 32, truncating: true}, + "CVTTSS2SQ": {inputBits: 32, outputBits: 64, truncating: true}, + "CVTTSD2SL": {inputBits: 64, outputBits: 32, truncating: true}, + "CVTTSD2SQ": {inputBits: 64, outputBits: 64, truncating: true}, + + "VCVTSS2SI": {inputBits: 32, outputBits: 32, vector: true}, + "VCVTSS2SIQ": {inputBits: 32, outputBits: 64, vector: true}, + "VCVTSD2SI": {inputBits: 64, outputBits: 32, vector: true}, + "VCVTSD2SIQ": {inputBits: 64, outputBits: 64, vector: true}, + "VCVTTSS2SI": {inputBits: 32, outputBits: 32, truncating: true, vector: true}, + "VCVTTSS2SIQ": {inputBits: 32, outputBits: 64, truncating: true, vector: true}, + "VCVTTSD2SI": {inputBits: 64, outputBits: 32, truncating: true, vector: true}, + "VCVTTSD2SIQ": {inputBits: 64, outputBits: 64, truncating: true, vector: true}, + + "VCVTSS2USIL": {inputBits: 32, outputBits: 32, unsigned: true, vector: true}, + "VCVTSS2USIQ": {inputBits: 32, outputBits: 64, unsigned: true, vector: true}, + "VCVTSD2USIL": {inputBits: 64, outputBits: 32, unsigned: true, vector: true}, + "VCVTSD2USIQ": {inputBits: 64, outputBits: 64, unsigned: true, vector: true}, + "VCVTTSS2USIL": {inputBits: 32, outputBits: 32, truncating: true, unsigned: true, vector: true}, + "VCVTTSS2USIQ": {inputBits: 32, outputBits: 64, truncating: true, unsigned: true, vector: true}, + "VCVTTSD2USIL": {inputBits: 64, outputBits: 32, truncating: true, unsigned: true, vector: true}, + "VCVTTSD2USIQ": {inputBits: 64, outputBits: 64, truncating: true, unsigned: true, vector: true}, +} + +// lowerScalarFloatToInteger implements Go 1.27's complete signed legacy/VEX/ +// EVEX scalar float-to-integer family and its EVEX unsigned counterparts. +func (c *amd64Ctx) lowerScalarFloatToInteger(rawOp Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(rawOp)), ".") + if _, known := amd64ScalarFloatToIntSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(rawOp), string(ins.Op)) { + return true, false, fmt.Errorf("scalar conversion dispatch/source mismatch: %s/%s", rawOp, ins.Op) + } + ins.Op = rawOp + f, known, err := x86ScalarFloatIntegerProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: scalar conversion lacks native MXCSR target", ErrProbeNeedsContext) + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride { + return true, false, fmt.Errorf("%w: scalar conversion needs its original native address size", ErrProbeNeedsContext) + } + return true, false, c.emitNativeScalarFloatInteger(ins, f) +} + +func (c *amd64Ctx) evalScalarFloatForInteger(source Operand, bits int) (string, error) { + if bits == 32 { + return c.evalF32(source) + } + return c.evalF64(source) +} + +func amd64FloatingTypeForBits(bits int) LLVMType { + if bits == 32 { + return LLVMType("float") + } + return LLVMType("double") +} diff --git a/amd64_lower_scalar_float_to_int_test.go b/amd64_lower_scalar_float_to_int_test.go new file mode 100644 index 000000000..b38531742 --- /dev/null +++ b/amd64_lower_scalar_float_to_int_test.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ScalarFloatToIntegerCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + xreg := "X16" + if target.goarch == "386" { + xreg = "X7" // A high named bank has no physical mode32 identity. + } + var source strings.Builder + source.WriteString("TEXT scalarfloattointforms(SB),NOSPLIT,$0-0\n") + for _, precision := range []string{"SS", "SD"} { + for _, truncating := range []bool{false, true} { + prefix := "CVT" + if truncating { + prefix = "CVTT" + } + for _, width := range []string{"L", "Q"} { + if target.goarch == "386" && width == "Q" { + continue + } + op := prefix + precision + "2S" + width + fmt.Fprintf(&source, "\t%s X0, AX\n", op) + fmt.Fprintf(&source, "\t%s 8(BX), DI\n", op) + vop := "V" + prefix + precision + "2SI" + if width == "Q" { + vop += "Q" + } + fmt.Fprintf(&source, "\t%s X0, AX\n", vop) + fmt.Fprintf(&source, "\t%s 16(BX), DI\n", vop) + if truncating { + fmt.Fprintf(&source, "\t%s.SAE %s, AX\n", vop, xreg) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s %s, AX\n", vop, rounding, xreg) + } + } + } + } + for _, truncating := range []bool{false, true} { + prefix := "VCVT" + if truncating { + prefix = "VCVTT" + } + for _, width := range []string{"L", "Q"} { + if target.goarch == "386" && width == "Q" { + continue + } + op := prefix + precision + "2USI" + width + fmt.Fprintf(&source, "\t%s %s, AX\n", op, xreg) + fmt.Fprintf(&source, "\t%s 24(BX), DI\n", op) + if truncating { + fmt.Fprintf(&source, "\t%s.SAE %s, AX\n", op, xreg) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s %s, AX\n", op, rounding, xreg) + } + } + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"scalarfloattointforms": {Name: "scalarfloattointforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scalar-float-to-int-"+target.name+".ll", "scalar-float-to-int-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ScalarFloatToIntegerRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "CVTSS2SL Y0, AX", + "CVTTSD2SQ X16, AX", + "CVTSS2SL X0, X1", + "CVTSS2SL.Z X0, AX", + "VCVTSS2SI Y0, AX", + "VCVTSS2SI X0, X1", + "VCVTSS2SI.SAE X0, AX", + "VCVTTSS2SI.RN_SAE X0, AX", + "VCVTSD2SI.RD_SAE 8(BX), AX", + "VCVTTSD2SI.SAE 8(BX), AX", + "VCVTSS2USIL X0, X1", + "VCVTSS2USIL.Z X0, AX", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ScalarFloatToIntegerRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "CVTSS2SQ X0, AX", + "CVTTSD2SQ 8(BX), AX", + "VCVTTSD2SIQ.SAE X16, AX", + "VCVTSS2USIQ X16, AX", + "VCVTTSD2USIQ.SAE X16, AX", + "CVTSS2SL X8, AX", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ScalarFloatToIntegerRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86ScalarFloatToIntegerRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar float-to-integer tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_scalar_integer_float.go b/amd64_lower_scalar_integer_float.go new file mode 100644 index 000000000..8072c2c4f --- /dev/null +++ b/amd64_lower_scalar_integer_float.go @@ -0,0 +1,240 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ScalarIntegerFloatSpec struct { + sourceBits int + resultBits int + unsigned bool + rounding bool +} + +// This is the complete Go 1.27 scalar integer-to-floating-point family from +// _yvcvtsi2sdl and _yvcvtusi2sdl. It is also the source of truth used by the +// supported-opcode extractor. +var amd64ScalarIntegerFloatOps = map[string]amd64ScalarIntegerFloatSpec{ + "VCVTSI2SDL": {sourceBits: 32, resultBits: 64}, + "VCVTSI2SDQ": {sourceBits: 64, resultBits: 64, rounding: true}, + "VCVTSI2SSL": {sourceBits: 32, resultBits: 32, rounding: true}, + "VCVTSI2SSQ": {sourceBits: 64, resultBits: 32, rounding: true}, + "VCVTUSI2SDL": {sourceBits: 32, resultBits: 64, unsigned: true}, + "VCVTUSI2SDQ": {sourceBits: 64, resultBits: 64, unsigned: true, rounding: true}, + "VCVTUSI2SSL": {sourceBits: 32, resultBits: 32, unsigned: true, rounding: true}, + "VCVTUSI2SSQ": {sourceBits: 64, resultBits: 32, unsigned: true, rounding: true}, +} + +func (c *amd64Ctx) lowerScalarIntegerToFloat(op Op, ins Instr) (ok bool, terminated bool, err error) { + if ok, term, err := c.lowerSignedIntegerScalarFloat(op, ins); ok { + return ok, term, err + } + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64ScalarIntegerFloatOps[baseOp] + if !recognized { + return false, false, nil + } + rounding := "" + switch suffix { + case "": + case "RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE": + if !spec.rounding { + return true, false, fmt.Errorf("%s %s does not enable explicit rounding: %q", c.goarch, baseOp, ins.Raw) + } + rounding = suffix + default: + return true, false, fmt.Errorf("%s %s suffix is absent from Go 1.27's scalar integer-to-float tables: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects integer source, X passthrough, X destination: %q", c.goarch, baseOp, ins.Raw) + } + source, passthrough, destination := ins.Args[0], ins.Args[1], ins.Args[2] + if source.Kind == OpReg { + if !c.isGoYrlRegister(source) { + return true, false, fmt.Errorf("%s %s integer source is outside Go 1.27's GP-register class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s integer source must be a GP register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if rounding != "" && source.Kind != OpReg { + return true, false, fmt.Errorf("%s %s explicit rounding requires a GP-register source: %q", c.goarch, baseOp, ins.Raw) + } + if !c.isGoEVEXVectorRegister(passthrough, 16) || !c.isGoEVEXVectorRegister(destination, 16) { + return true, false, fmt.Errorf("%s %s passthrough and destination must be Go 1.27 X registers: %q", c.goarch, baseOp, ins.Raw) + } + + integerType := amd64IntegerTypeForBits(spec.sourceBits) + integer, err := c.evalIntSized(source, integerType) + if err != nil { + return true, false, err + } + floatType := LLVMType("float") + if spec.resultBits == 64 { + floatType = LLVMType("double") + } + conversion := "sitofp" + if spec.unsigned { + conversion = "uitofp" + } + var low string + if rounding == "" { + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", converted, conversion, integerType, integer, floatType) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to i%d\n", bits, floatType, converted, spec.resultBits) + low = "%" + bits + } else { + low = c.emitRoundedScalarIntegerToFloatBits(integer, spec, rounding) + } + passthroughBytes, err := c.loadX(passthrough.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.resultBits, spec.resultBits, passthroughBytes) + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.resultBits, low, base, "", false) +} + +// emitRoundedScalarIntegerToFloatBits converts through an explicit integer +// significand so embedded rounding remains correct even when LLVM 22 lowers a +// constrained conversion to an instruction controlled by the ambient MXCSR. +func (c *amd64Ctx) emitRoundedScalarIntegerToFloatBits(integer string, spec amd64ScalarIntegerFloatSpec, rounding string) string { + integerType := amd64IntegerTypeForBits(spec.sourceBits) + precision := 24 + bias := 127 + if spec.resultBits == 64 { + precision = 53 + bias = 1023 + } + + negative := "false" + magnitude := integer + if !spec.unsigned { + isNegative := c.newTmp() + negated := c.newTmp() + absolute := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, 0\n", isNegative, integerType, integer) + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", negated, integerType, integer) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %s\n", absolute, isNegative, integerType, negated, integerType, integer) + negative = "%" + isNegative + magnitude = "%" + absolute + } + + isZero := c.newTmp() + safeMagnitude := c.newTmp() + leadingZeros := c.newTmp() + msb := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", isZero, integerType, magnitude) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s 1, %s %s\n", safeMagnitude, isZero, integerType, integerType, magnitude) + fmt.Fprintf(c.b, " %%%s = call %s @llvm.ctlz.i%d(%s %%%s, i1 false)\n", leadingZeros, integerType, spec.sourceBits, integerType, safeMagnitude) + fmt.Fprintf(c.b, " %%%s = sub %s %d, %%%s\n", msb, integerType, spec.sourceBits-1, leadingZeros) + + needsRightShift := c.newTmp() + rawRightShift := c.newTmp() + rightShift := c.newTmp() + rawLeftShift := c.newTmp() + leftShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge %s %%%s, %d\n", needsRightShift, integerType, msb, precision-1) + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, %d\n", rawRightShift, integerType, msb, precision-1) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", rightShift, needsRightShift, integerType, rawRightShift, integerType) + fmt.Fprintf(c.b, " %%%s = sub %s %d, %%%s\n", rawLeftShift, integerType, precision-1, msb) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s 0, %s %%%s\n", leftShift, needsRightShift, integerType, integerType, rawLeftShift) + + shiftedRight := c.newTmp() + shiftedLeft := c.newTmp() + significand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", shiftedRight, integerType, magnitude, rightShift) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", shiftedLeft, integerType, magnitude, leftShift) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %%%s\n", significand, needsRightShift, integerType, shiftedRight, integerType, shiftedLeft) + + oneAtShift := c.newTmp() + remainderMask := c.newTmp() + remainder := c.newTmp() + inexact := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", oneAtShift, integerType, rightShift) + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, 1\n", remainderMask, integerType, oneAtShift) + fmt.Fprintf(c.b, " %%%s = and %s %s, %%%s\n", remainder, integerType, magnitude, remainderMask) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", inexact, integerType, remainder) + + increment := "false" + switch rounding { + case "RN_SAE": + hasShift := c.newTmp() + rawHalfShift := c.newTmp() + halfShift := c.newTmp() + half := c.newTmp() + aboveHalf := c.newTmp() + equalHalf := c.newTmp() + lowBit := c.newTmp() + odd := c.newTmp() + tieAndOdd := c.newTmp() + roundNearest := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", hasShift, integerType, rightShift) + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, 1\n", rawHalfShift, integerType, rightShift) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", halfShift, hasShift, integerType, rawHalfShift, integerType) + fmt.Fprintf(c.b, " %%%s = shl %s 1, %%%s\n", half, integerType, halfShift) + fmt.Fprintf(c.b, " %%%s = icmp ugt %s %%%s, %%%s\n", aboveHalf, integerType, remainder, half) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %%%s\n", equalHalf, integerType, remainder, half) + fmt.Fprintf(c.b, " %%%s = and %s %%%s, 1\n", lowBit, integerType, significand) + fmt.Fprintf(c.b, " %%%s = icmp ne %s %%%s, 0\n", odd, integerType, lowBit) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", tieAndOdd, equalHalf, odd) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", roundNearest, aboveHalf, tieAndOdd) + increment = "%" + roundNearest + case "RD_SAE": + if !spec.unsigned { + roundDown := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %s, %%%s\n", roundDown, negative, inexact) + increment = "%" + roundDown + } + case "RU_SAE": + if spec.unsigned { + increment = "%" + inexact + } else { + positive := c.newTmp() + roundUp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", positive, negative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", roundUp, positive, inexact) + increment = "%" + roundUp + } + } + + incrementInteger := c.newTmp() + roundedSignificand := c.newTmp() + overflow := c.newTmp() + normalizedOverflow := c.newTmp() + normalized := c.newTmp() + exponentIncrement := c.newTmp() + exponent := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", incrementInteger, increment, integerType) + fmt.Fprintf(c.b, " %%%s = add %s %%%s, %%%s\n", roundedSignificand, integerType, significand, incrementInteger) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %d\n", overflow, integerType, roundedSignificand, uint64(1)< +#include +extern void scalarintegerfloat(uint8_t *, const int32_t *, const int64_t *, const uint32_t *, const uint64_t *, const float *, const double *); +int main(void) { + const int32_t signed32 = -16777217; + const int64_t signed64 = -16777217; + const uint32_t unsigned32 = UINT32_MAX; + const uint64_t unsigned64 = UINT64_MAX; + const float passthrough32[4] = {200.0f, 201.0f, 202.0f, 203.0f}; + const double passthrough64[2] = {100.0, 101.0}; + union { uint8_t bytes[128]; double d64[16]; float f32[32]; } out = {0}; + scalarintegerfloat(out.bytes, &signed32, &signed64, &unsigned32, &unsigned64, passthrough32, passthrough64); + if (out.d64[0] != -16777217.0 || out.d64[1] != 101.0) return 1; + if (out.d64[2] != -16777217.0 || out.d64[3] != 101.0) return 2; + if (out.f32[8] != -16777218.0f || out.f32[9] != 201.0f || out.f32[10] != 202.0f || out.f32[11] != 203.0f) return 3; + if (out.f32[12] != -16777216.0f || out.f32[13] != 201.0f || out.f32[14] != 202.0f || out.f32[15] != 203.0f) return 4; + if (out.d64[8] != 4294967295.0 || out.d64[9] != 101.0) return 5; + if (out.d64[10] != 18446744073709549568.0 || out.d64[11] != 101.0) return 6; + if (out.f32[24] != 4294967040.0f || out.f32[25] != 201.0f || out.f32[26] != 202.0f || out.f32[27] != 203.0f) { + fprintf(stderr, "unsigned32 RD result=%a upper={%a,%a,%a}\n", out.f32[24], out.f32[25], out.f32[26], out.f32[27]); + return 7; + } + if (out.f32[28] != 18446744073709551616.0f || out.f32[29] != 201.0f || out.f32[30] != 202.0f || out.f32[31] != 203.0f) return 8; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_integer_float", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_scaled_round.go b/amd64_lower_scaled_round.go new file mode 100644 index 000000000..dfc3b3a20 --- /dev/null +++ b/amd64_lower_scaled_round.go @@ -0,0 +1,277 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ScaledRoundMode uint8 + +const ( + amd64RoundScale amd64ScaledRoundMode = iota + amd64Reduce +) + +type amd64ScaledRoundSpec struct { + laneBits int + scalar bool + mode amd64ScaledRoundMode +} + +var amd64ScaledRoundSpecs = map[Op]amd64ScaledRoundSpec{ + "VRNDSCALEPS": {laneBits: 32}, + "VRNDSCALEPD": {laneBits: 64}, + "VRNDSCALESS": {laneBits: 32, scalar: true}, + "VRNDSCALESD": {laneBits: 64, scalar: true}, + "VREDUCEPS": {laneBits: 32, mode: amd64Reduce}, + "VREDUCEPD": {laneBits: 64, mode: amd64Reduce}, + "VREDUCESS": {laneBits: 32, scalar: true, mode: amd64Reduce}, + "VREDUCESD": {laneBits: 64, scalar: true, mode: amd64Reduce}, +} + +func (c *amd64Ctx) lowerScaledRound(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp, suffix := rawOp, "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + spec, recognized := amd64ScaledRoundSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + shape := amd64UnaryImmediateFloatingShape{laneBits: spec.laneBits, scalar: spec.scalar} + form, err := c.parseUnaryImmediateFloatingForm(baseOp, suffix, shape, ins) + if err != nil { + return true, false, err + } + mxcsr := c.loadMXCSR() + dazBits, daz := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i32 %s, 64\n", dazBits, mxcsr) + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", daz, dazBits) + roundingMode := fmt.Sprintf("%d", form.immediate&3) + if form.immediate&4 != 0 { + shifted, extracted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i32 %s, 13\n", shifted, mxcsr) + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 3\n", extracted, shifted) + roundingMode = "%" + extracted + } + if spec.scalar { + return c.lowerScalarScaledRound(spec, form.properties, form.source, form.passthrough, form.destination, form.mask, "%"+daz, roundingMode, form.immediate) + } + return c.lowerPackedScaledRound(spec, form.properties, form.source, form.destination, form.byteWidth, form.mask, "%"+daz, roundingMode, form.immediate) +} + +func (c *amd64Ctx) lowerPackedScaledRound(spec amd64ScaledRoundSpec, properties amd64BinaryFloatingSuffix, source, destination Operand, byteWidth int, mask, daz, roundingMode string, immediate uint8) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + input, err := c.loadPackedCompareLanes(source, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + result := c.emitScaledRoundBits(lanes, spec.laneBits, input, daz, roundingMode, immediate>>4, spec.mode) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarScaledRound(spec amd64ScaledRoundSpec, properties amd64BinaryFloatingSuffix, source, passthrough, destination Operand, mask, daz, roundingMode string, immediate uint8) (bool, bool, error) { + sourceBits, err := c.loadFloatingScalarBits(source, spec.laneBits) + if err != nil { + return true, false, err + } + input := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", input, spec.laneBits, spec.laneBits, sourceBits) + result := c.emitScaledRoundBits(1, spec.laneBits, "%"+input, daz, roundingMode, immediate>>4, spec.mode) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(passthrough.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitScaledRoundBits(lanes, laneBits int, input, daz, roundingMode string, scale uint8, mode amd64ScaledRoundMode) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + mantissaBits, bias := 23, 127 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + if laneBits == 64 { + mantissaBits, bias = 52, 1023 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + } + implicitBit := uint64(1) << mantissaBits + alreadyThreshold := bias + mantissaBits - int(scale) + regularThreshold := bias - int(scale) + stepBits := uint64(bias-int(scale)) << mantissaBits + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + bits, exponent, mantissa, sign := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", bits, vectorType, input, lane) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", exponent, laneBits, bits, exponentMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", mantissa, laneBits, bits, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", sign, laneBits, bits, signBit) + exponentZero, exponentOnes, mantissaZero := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponent) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", exponentOnes, laneBits, exponent, exponentMask) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", mantissaZero, laneBits, mantissa) + mantissaNonzero, negative := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", mantissaNonzero, mantissaZero) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", negative, laneBits, sign) + isNaN, isInfinity := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, exponentOnes, mantissaNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, exponentOnes, mantissaZero) + effectiveZeroPart, effectiveZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %s\n", effectiveZeroPart, mantissaZero, daz) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", effectiveZero, exponentZero, effectiveZeroPart) + + exponentField := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", exponentField, laneBits, exponent, mantissaBits) + already, regularLower, regularUpper, regular := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp uge i%d %%%s, %d\n", already, laneBits, exponentField, alreadyThreshold) + fmt.Fprintf(c.b, " %%%s = icmp uge i%d %%%s, %d\n", regularLower, laneBits, exponentField, regularThreshold) + fmt.Fprintf(c.b, " %%%s = icmp ult i%d %%%s, %d\n", regularUpper, laneBits, exponentField, alreadyThreshold) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", regular, regularLower, regularUpper) + half := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %d\n", half, laneBits, exponentField, regularThreshold-1) + + safeExponent, drop := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %d\n", safeExponent, regular, laneBits, exponentField, laneBits, alreadyThreshold-1) + fmt.Fprintf(c.b, " %%%s = sub i%d %d, %%%s\n", drop, laneBits, mantissaBits+bias-int(scale), safeExponent) + significand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", significand, laneBits, mantissa, implicitBit) + quotient, oneAtDrop, remainderMask, remainder := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %%%s\n", quotient, laneBits, significand, drop) + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %%%s\n", oneAtDrop, laneBits, drop) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, 1\n", remainderMask, laneBits, oneAtDrop) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %%%s\n", remainder, laneBits, significand, remainderMask) + halfRemainder, aboveHalf, equalHalf := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, 1\n", halfRemainder, laneBits, oneAtDrop) + fmt.Fprintf(c.b, " %%%s = icmp ugt i%d %%%s, %%%s\n", aboveHalf, laneBits, remainder, halfRemainder) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %%%s\n", equalHalf, laneBits, remainder, halfRemainder) + quotientOddValue, quotientOdd, tiedOdd := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 1\n", quotientOddValue, laneBits, quotient) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", quotientOdd, laneBits, quotientOddValue) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", tiedOdd, equalHalf, quotientOdd) + nearestRegular := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", nearestRegular, aboveHalf, tiedOdd) + remainderNonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", remainderNonzero, laneBits, remainder) + incrementRegular := c.emitScaledRoundIncrement(roundingMode, "%"+negative, "%"+remainderNonzero, "%"+nearestRegular) + incrementValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i%d\n", incrementValue, incrementRegular, laneBits) + roundedQuotient, roundedSignificand := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %%%s\n", roundedQuotient, laneBits, quotient, incrementValue) + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %%%s\n", roundedSignificand, laneBits, roundedQuotient, drop) + carryValue, carry, carryInteger := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", carryValue, laneBits, roundedSignificand, implicitBit<<1) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", carry, laneBits, carryValue) + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i%d\n", carryInteger, carry, laneBits) + roundedExponentField, roundedExponent := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %%%s\n", roundedExponentField, laneBits, exponentField, carryInteger) + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %d\n", roundedExponent, laneBits, roundedExponentField, mantissaBits) + roundedFraction, roundedMagnitude, roundedRegular := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", roundedFraction, laneBits, roundedSignificand, mantissaMask) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", roundedMagnitude, laneBits, roundedExponent, roundedFraction) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", roundedRegular, laneBits, roundedMagnitude, sign) + + halfAbove := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ugt i%d %%%s, %d\n", halfAbove, laneBits, significand, implicitBit) + nonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", nonzero, effectiveZero) + incrementHalf := c.emitScaledRoundIncrement(roundingMode, "%"+negative, "%"+nonzero, "%"+halfAbove) + incrementSmall := c.emitScaledRoundIncrement(roundingMode, "%"+negative, "%"+nonzero, "false") + incrementNonRegular := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i1 %s, i1 %s\n", incrementNonRegular, half, incrementHalf, incrementSmall) + signedStep, stepOrZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", signedStep, laneBits, sign, stepBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", stepOrZero, incrementNonRegular, laneBits, signedStep, laneBits, sign) + roundedNeeded, roundedFinite, zeroAdjusted := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", roundedNeeded, regular, laneBits, roundedRegular, laneBits, stepOrZero) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", roundedFinite, already, laneBits, bits, laneBits, roundedNeeded) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", zeroAdjusted, effectiveZero, laneBits, sign, laneBits, roundedFinite) + withInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", withInfinity, isInfinity, laneBits, bits, laneBits, zeroAdjusted) + quietNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %d\n", quietNaN, laneBits, bits, quietBit) + roundedFinal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", roundedFinal, isNaN, laneBits, quietNaN, laneBits, withInfinity) + laneResult := "%" + roundedFinal + if mode == amd64Reduce { + effectiveSource := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", effectiveSource, effectiveZero, laneBits, sign, laneBits, bits) + laneResult = c.emitReduceResult(laneBits, "%"+effectiveSource, "%"+zeroAdjusted, "%"+isNaN, "%"+isInfinity, roundingMode, quietBit, signBit) + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, laneBits, laneResult, lane) + result = "%" + inserted + } + return result +} + +func (c *amd64Ctx) emitScaledRoundIncrement(roundingMode, negative, nonzero, nearest string) string { + isNearest, isDown, isUp := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 0\n", isNearest, roundingMode) + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 1\n", isDown, roundingMode) + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 2\n", isUp, roundingMode) + nearestIncrement, downIncrement := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", nearestIncrement, isNearest, nearest) + fmt.Fprintf(c.b, " %%%s = and i1 %s, %s\n", downIncrement, negative, nonzero) + downSelected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", downSelected, isDown, downIncrement) + nonnegative, upIncrement, upSelected := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", nonnegative, negative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", upIncrement, nonnegative, nonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", upSelected, isUp, upIncrement) + combined, result := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", combined, nearestIncrement, downSelected) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", result, combined, upSelected) + return "%" + result +} + +func (c *amd64Ctx) emitReduceResult(laneBits int, sourceBits, finiteRoundedBits, isNaN, isInfinity, roundingMode string, quietBit, signBit uint64) string { + floatType := "float" + if laneBits == 64 { + floatType = "double" + } + special := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %s, %s\n", special, isNaN, isInfinity) + safeSourceBits, safeRoundedBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %s\n", safeSourceBits, special, laneBits, laneBits, sourceBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d 0, i%d %s\n", safeRoundedBits, special, laneBits, laneBits, finiteRoundedBits) + sourceFloat, roundedFloat := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to %s\n", sourceFloat, laneBits, safeSourceBits, floatType) + fmt.Fprintf(c.b, " %%%s = bitcast i%d %%%s to %s\n", roundedFloat, laneBits, safeRoundedBits, floatType) + reducedFloat, reducedBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = fsub %s %%%s, %%%s\n", reducedFloat, floatType, sourceFloat, roundedFloat) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to i%d\n", reducedBits, floatType, reducedFloat, laneBits) + reducedMagnitude, reducedZero := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", reducedMagnitude, laneBits, reducedBits, signBit-1) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", reducedZero, laneBits, reducedMagnitude) + isDown := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 1\n", isDown, roundingMode) + zeroBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %d, i%d 0\n", zeroBits, isDown, laneBits, signBit, laneBits) + zeroAdjusted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", zeroAdjusted, reducedZero, laneBits, zeroBits, laneBits, reducedBits) + withInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d 0, i%d %%%s\n", withInfinity, isInfinity, laneBits, laneBits, zeroAdjusted) + quietNaNBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", quietNaNBits, laneBits, sourceBits, quietBit) + final := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", final, isNaN, laneBits, quietNaNBits, laneBits, withInfinity) + return "%" + final +} diff --git a/amd64_lower_scaled_round_test.go b/amd64_lower_scaled_round_test.go new file mode 100644 index 000000000..c315d2685 --- /dev/null +++ b/amd64_lower_scaled_round_test.go @@ -0,0 +1,305 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateAMD64ScaledRoundRepresentativeForms(t *testing.T) { + const source = `TEXT scaledroundforms(SB),$0-0 + VREDUCEPS $3, Z1, K2, Z3 + VRNDSCALESD $20, X4, X5, K6, X7 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"scaledroundforms": {Name: "scaledroundforms", Ret: Void}}, + }); err != nil { + t.Fatal(err) + } +} + +func TestAMD64ScaledRoundGrammarCoversCompleteGoFamilies(t *testing.T) { + expected := map[Op]amd64ScaledRoundSpec{ + "VRNDSCALEPS": {laneBits: 32}, + "VRNDSCALEPD": {laneBits: 64}, + "VRNDSCALESS": {laneBits: 32, scalar: true}, + "VRNDSCALESD": {laneBits: 64, scalar: true}, + "VREDUCEPS": {laneBits: 32, mode: amd64Reduce}, + "VREDUCEPD": {laneBits: 64, mode: amd64Reduce}, + "VREDUCESS": {laneBits: 32, scalar: true, mode: amd64Reduce}, + "VREDUCESD": {laneBits: 64, scalar: true, mode: amd64Reduce}, + } + if len(amd64ScaledRoundSpecs) != len(expected) { + t.Fatalf("scaled-round grammar has %d entries, want %d", len(amd64ScaledRoundSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64ScaledRoundSpecs[op]; !ok { + t.Errorf("scaled-round grammar omitted %s", op) + } else if got != want { + t.Errorf("scaled-round grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86ScaledRoundCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scaledroundallforms(SB),$0-0\n") + for _, op := range []string{"VRNDSCALEPS", "VRNDSCALEPD", "VREDUCEPS", "VREDUCEPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" && width == "Z" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, %s1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s $255, 8(AX), %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST $1, 16(AX), %s%d\n", op, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $2, %s1, K1, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.Z $3, 24(AX), K2, %s%d\n", op, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z $4, 32(AX), K3, %s%d\n", op, width, last) + } + if width == "Z" { + fmt.Fprintf(&source, "\t%s.SAE $5, Z1, Z%d\n", op, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.SAE.Z $6, Z1, K4, Z%d\n", op, last) + } + } + } + } + if target.goarch == "amd64" { + for _, op := range []string{"VRNDSCALESS", "VRNDSCALESD", "VREDUCESS", "VREDUCESD"} { + fmt.Fprintf(&source, "\t%s $0, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $255, 40(AX), X2, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE $1, X1, X2, X31\n", op) + fmt.Fprintf(&source, "\t%s $2, X1, X2, K5, X31\n", op) + fmt.Fprintf(&source, "\t%s.Z $3, 48(AX), X2, K6, X31\n", op) + fmt.Fprintf(&source, "\t%s.SAE.Z $4, X1, X2, K7, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"scaledroundallforms": {Name: "scaledroundallforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scaled-round-"+target.name+".ll", "scaled-round-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScaledRoundRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VREDUCEPS $-1, X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPD $256, X0, X1"}, + {goarch: "amd64", instruction: "VREDUCEPS X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS $0, X0, Y1"}, + {goarch: "amd64", instruction: "VREDUCEPS.BCST $0, X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS.SAE $0, Z0, X1"}, + {goarch: "amd64", instruction: "VREDUCEPD.SAE $0, 0(AX), Z1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS.RN_SAE $0, Z0, Z1"}, + {goarch: "amd64", instruction: "VREDUCEPS.Z $0, X0, X1"}, + {goarch: "amd64", instruction: "VRNDSCALEPS $0, X0, K0, X1"}, + {goarch: "amd64", instruction: "VREDUCESS $0, Y0, X1, X2"}, + {goarch: "amd64", instruction: "VRNDSCALESD.BCST $0, 0(AX), X1, X2"}, + {goarch: "amd64", instruction: "VREDUCESD $0, X0, X1, Y2"}, + {goarch: "386", instruction: "VRNDSCALESS $0, X0, X1, X2"}, + {goarch: "386", instruction: "VREDUCEPS $0, X0, K1, X2"}, + {goarch: "386", instruction: "VRNDSCALEPS $0, Z8, Z0"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scaled-round tables", test.instruction) + } + }) + } +} + +func TestAMD64ScaledRoundRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + source.WriteString(`TEXT scaledroundsemantics(SB),$0-56 + MOVQ out+0(FP), AX + MOVQ source32+8(FP), BX + MOVQ source64+16(FP), CX + MOVQ old32+24(FP), DX + MOVQ old64+32(FP), R8 + MOVQ upper+40(FP), R9 + MOVQ mask+48(FP), R10 + KMOVQ R10, K1 + VMOVDQU64 0(R9), X4 + STC +`) + for scale := 0; scale < 16; scale++ { + for rounding := 0; rounding < 4; rounding++ { + index := scale*4 + rounding + immediate := scale<<4 | rounding + fmt.Fprintf(&source, "\tVRNDSCALESS $%d, %d(BX), X4, X5\n", immediate, index%16*4) + fmt.Fprintf(&source, "\tVMOVSS X5, %d(AX)\n", index*4) + fmt.Fprintf(&source, "\tVREDUCESS $%d, %d(BX), X4, X5\n", immediate, index%16*4) + fmt.Fprintf(&source, "\tVMOVSS X5, %d(AX)\n", 256+index*4) + fmt.Fprintf(&source, "\tVRNDSCALESD $%d, %d(CX), X4, X5\n", immediate, index%16*8) + fmt.Fprintf(&source, "\tVMOVSD X5, %d(AX)\n", 512+index*8) + fmt.Fprintf(&source, "\tVREDUCESD $%d, %d(CX), X4, X5\n", immediate, index%16*8) + fmt.Fprintf(&source, "\tVMOVSD X5, %d(AX)\n", 1024+index*8) + } + } + source.WriteString(` VMOVDQU32 0(DX), Z2 + VRNDSCALEPS $35, 0(BX), K1, Z2 + VMOVDQU32 Z2, 1536(AX) + VREDUCEPS.Z $35, 0(BX), K1, Z2 + VMOVDQU32 Z2, 1600(AX) + VRNDSCALEPS.BCST $35, 0(BX), Z2 + VMOVDQU32 Z2, 1664(AX) + VMOVDQU64 0(R8), Z3 + VRNDSCALEPD $35, 0(CX), K1, Z3 + VMOVDQU64 Z3, 1728(AX) + VREDUCEPD.Z $35, 0(CX), K1, Z3 + VMOVDQU64 Z3, 1792(AX) + VRNDSCALEPD.BCST $35, 0(CX), Z3 + VMOVDQU64 Z3, 1856(AX) + VRNDSCALESS $0, 0(BX), X4, X5 + VMOVDQU64 X5, 1920(AX) + VREDUCESD $0, 0(CX), X4, X6 + VMOVDQU64 X6, 1936(AX) + VRNDSCALESS $36, 0(BX), X4, X5 + VMOVSS X5, 1952(AX) + VREDUCESS $36, 8(BX), X4, X5 + VMOVSS X5, 1956(AX) + VRNDSCALESD $36, 0(CX), X4, X5 + VMOVSD X5, 1960(AX) + VREDUCESD $36, 16(CX), X4, X5 + VMOVSD X5, 1968(AX) + SETCS 1976(AX) + RET +`) + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"scaledroundsemantics": { + Name: "scaledroundsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: I64, Index: 6, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void scaledroundsemantics(uint8_t *, const uint32_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static int inc(unsigned rc,int neg,int nonzero,int nearest) { return rc==0?nearest:(rc==1?neg&&nonzero:(rc==2?!neg&&nonzero:0)); } +static uint32_t round32(uint32_t u,unsigned scale,unsigned rc,int daz) { + const uint32_t em=0x7f800000,fm=0x007fffff,qb=0x00400000,sb=0x80000000,ib=0x00800000; + uint32_t e=(u&em)>>23,f=u&fm,s=u&sb; if(e==255) return f?(u|qb):u; + if(e==0&&(f==0||daz)) return s; + unsigned already=127+23-scale, regular=127-scale; + if(e>=already) return u; + uint32_t sig=f|ib; + if(e>=regular) { unsigned drop=23+127-scale-e; uint32_t q=sig>>drop, rem=sig&((UINT32_C(1)<half||(rem==half&&(q&1)); q+=(uint32_t)inc(rc,s!=0,rem!=0,n); uint32_t rs=q<ib; return s|(inc(rc,s!=0,1,n)?((127-scale)<<23):0); +} +static uint64_t round64(uint64_t u,unsigned scale,unsigned rc,int daz) { + const uint64_t em=UINT64_C(0x7ff0000000000000),fm=UINT64_C(0x000fffffffffffff),qb=UINT64_C(0x0008000000000000),sb=UINT64_C(0x8000000000000000),ib=UINT64_C(0x0010000000000000); + uint64_t e=(u&em)>>52,f=u&fm,s=u&sb; if(e==2047) return f?(u|qb):u; + if(e==0&&(f==0||daz)) return s; + unsigned already=1023+52-scale, regular=1023-scale; + if(e>=already) return u; + uint64_t sig=f|ib; + if(e>=regular) { unsigned drop=52+1023-scale-(unsigned)e; uint64_t q=sig>>drop, rem=sig&((UINT64_C(1)<half||(rem==half&&(q&1)); q+=(uint64_t)inc(rc,s!=0,rem!=0,n); uint64_t rs=q<ib; return s|(inc(rc,s!=0,1,n)?((uint64_t)(1023-scale)<<52):0); +} +static uint32_t reduce32(uint32_t u,unsigned scale,unsigned rc,int daz) { uint32_t e=u&0x7f800000,f=u&0x007fffff,s=u&0x80000000; if(e==0x7f800000) return f?(u|0x00400000):0; uint32_t effective=(e==0&&(f==0||daz))?s:u, rounded=round32(u,scale,rc,daz), r; float a,b,c; memcpy(&a,&effective,4); memcpy(&b,&rounded,4); c=a-b; memcpy(&r,&c,4); return (r&0x7fffffff)?r:(rc==1?0x80000000:0); } +static uint64_t reduce64(uint64_t u,unsigned scale,unsigned rc,int daz) { uint64_t e=u&UINT64_C(0x7ff0000000000000),f=u&UINT64_C(0x000fffffffffffff),s=u&UINT64_C(0x8000000000000000); if(e==UINT64_C(0x7ff0000000000000)) return f?(u|UINT64_C(0x0008000000000000)):0; uint64_t effective=(e==0&&(f==0||daz))?s:u, rounded=round64(u,scale,rc,daz), r; double a,b,c; memcpy(&a,&effective,8); memcpy(&b,&rounded,8); c=a-b; memcpy(&r,&c,8); return (r&UINT64_C(0x7fffffffffffffff))?r:(rc==1?UINT64_C(0x8000000000000000):0); } +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + const uint32_t s32[16]={0x3fc00000,0x40200000,0xbfc00000,0xc0200000,0x3fb00000,0xbfb00000,0,0x80000000,0x7f800000,0xff800000,0x7fc12345,0xff812345,1,0x80000001,0x4e800000,0x3f800001}; + const uint64_t s64[16]={UINT64_C(0x3ff8000000000000),UINT64_C(0x4004000000000000),UINT64_C(0xbff8000000000000),UINT64_C(0xc004000000000000),UINT64_C(0x3ff6000000000000),UINT64_C(0xbff6000000000000),0,UINT64_C(0x8000000000000000),UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000),UINT64_C(0x7ff8123456789abc),UINT64_C(0xfff0123456789abc),1,UINT64_C(0x8000000000000001),UINT64_C(0x41d0000000000000),UINT64_C(0x3ff0000000000001)}; + uint32_t old32[16]; uint64_t old64[8]; const uint64_t upper[2]={UINT64_C(0x1122334455667788),UINT64_C(0x99aabbccddeeff00)}; const uint64_t mask=UINT64_C(0xa55a); + uint8_t out[1977]={0},outDaz[1977]={0},dynamic[4][1977]; for(int i=0;i<16;i++)old32[i]=UINT32_C(0x41000000)+i; for(int i=0;i<8;i++)old64[i]=UINT64_C(0x4020000000000000)+i; + unsigned csr=_mm_getcsr(); scaledroundsemantics(out,s32,s64,old32,old64,upper,mask); _mm_setcsr(csr|0x40); scaledroundsemantics(outDaz,s32,s64,old32,old64,upper,mask); _mm_setcsr(csr); + for(unsigned rc=0;rc<4;rc++){_mm_setcsr((csr&~(3u<<13))|(rc<<13));scaledroundsemantics(dynamic[rc],s32,s64,old32,old64,upper,mask);} _mm_setcsr(csr); + for(unsigned scale=0;scale<16;scale++)for(unsigned rc=0;rc<4;rc++){unsigned i=scale*4+rc,j=i&15;if(u32(out+i*4)!=round32(s32[j],scale,rc,0)||u32(out+256+i*4)!=reduce32(s32[j],scale,rc,0))return 10;if(u64(out+512+i*8)!=round64(s64[j],scale,rc,0)||u64(out+1024+i*8)!=reduce64(s64[j],scale,rc,0))return 11;if(u32(outDaz+i*4)!=round32(s32[j],scale,rc,1)||u32(outDaz+256+i*4)!=reduce32(s32[j],scale,rc,1))return 12;if(u64(outDaz+512+i*8)!=round64(s64[j],scale,rc,1)||u64(outDaz+1024+i*8)!=reduce64(s64[j],scale,rc,1))return 13;} + for(int i=0;i<16;i++){uint32_t r=round32(s32[i],2,3,0),d=reduce32(s32[i],2,3,0);if(u32(out+1536+i*4)!=(((mask>>i)&1)?r:old32[i]))return 20;if(u32(out+1600+i*4)!=(((mask>>i)&1)?d:0))return 21;if(u32(out+1664+i*4)!=round32(s32[0],2,3,0))return 22;} + for(int i=0;i<8;i++){uint64_t r=round64(s64[i],2,3,0),d=reduce64(s64[i],2,3,0);if(u64(out+1728+i*8)!=(((mask>>i)&1)?r:old64[i]))return 23;if(u64(out+1792+i*8)!=(((mask>>i)&1)?d:0))return 24;if(u64(out+1856+i*8)!=round64(s64[0],2,3,0))return 25;} + if(u32(out+1920)!=round32(s32[0],0,0,0)||memcmp(out+1924,(const uint8_t*)upper+4,12))return 30;if(u64(out+1936)!=reduce64(s64[0],0,0,0)||memcmp(out+1944,(const uint8_t*)upper+8,8))return 31;if(out[1976]!=1)return 32; + for(unsigned rc=0;rc<4;rc++){if(u32(dynamic[rc]+1952)!=round32(s32[0],2,rc,0)||u32(dynamic[rc]+1956)!=reduce32(s32[2],2,rc,0)||u64(dynamic[rc]+1960)!=round64(s64[0],2,rc,0)||u64(dynamic[rc]+1968)!=reduce64(s64[2],2,rc,0))return 40+rc;} + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scaled_round_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_scalef.go b/amd64_lower_scalef.go new file mode 100644 index 000000000..b9637367d --- /dev/null +++ b/amd64_lower_scalef.go @@ -0,0 +1,383 @@ +package plan9asm + +import "fmt" + +type amd64ScaleFControls struct { + daz string + ftz string + rounding string +} + +func (c *amd64Ctx) scaleFControls(rounding string) amd64ScaleFControls { + mxcsr := c.loadMXCSR() + dazBits, daz := c.newTmp(), c.newTmp() + ftzBits, ftz := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i32 %s, 64\n", dazBits, mxcsr) + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", daz, dazBits) + fmt.Fprintf(c.b, " %%%s = and i32 %s, 32768\n", ftzBits, mxcsr) + fmt.Fprintf(c.b, " %%%s = icmp ne i32 %%%s, 0\n", ftz, ftzBits) + mode := "" + switch rounding { + case "RN_SAE": + mode = "0" + case "RD_SAE": + mode = "1" + case "RU_SAE": + mode = "2" + case "RZ_SAE": + mode = "3" + default: + shifted, extracted := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i32 %s, 13\n", shifted, mxcsr) + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, 3\n", extracted, shifted) + mode = "%" + extracted + } + return amd64ScaleFControls{daz: "%" + daz, ftz: "%" + ftz, rounding: mode} +} + +func (c *amd64Ctx) lowerPackedScaleF(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, source2, source1, destination Operand, byteWidth int, mask string) (bool, bool, error) { + lanes := byteWidth * 8 / spec.laneBits + scale, err := c.loadPackedCompareLanes(source2, byteWidth, spec.laneBits, properties.broadcast) + if err != nil { + return true, false, err + } + value, err := c.loadPackedCompareLanes(source1, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + result := c.emitScaleFBits(lanes, spec.laneBits, value, scale, c.scaleFControls(properties.rounding)) + if mask != "" { + oldBytes, loadErr := c.loadPackedCompareBytes(destination, byteWidth) + if loadErr != nil { + return true, false, loadErr + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, properties.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) lowerScalarScaleF(spec amd64BinaryFloatingSpec, properties amd64BinaryFloatingSuffix, source2, source1, destination Operand, mask string) (bool, bool, error) { + scaleBits, err := c.loadFloatingScalarBits(source2, spec.laneBits) + if err != nil { + return true, false, err + } + valueBits, err := c.loadFloatingScalarBits(source1, spec.laneBits) + if err != nil { + return true, false, err + } + scale, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", scale, spec.laneBits, spec.laneBits, scaleBits) + fmt.Fprintf(c.b, " %%%s = insertelement <1 x i%d> poison, i%d %s, i32 0\n", value, spec.laneBits, spec.laneBits, valueBits) + result := c.emitScaleFBits(1, spec.laneBits, "%"+value, "%"+scale, c.scaleFControls(properties.rounding)) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <1 x i%d> %s, i32 0\n", low, spec.laneBits, result) + passthroughBytes, err := c.loadX(source1.Reg) + if err != nil { + return true, false, err + } + base := c.bitcastVectorBytesToIntegerLanes(16, 128/spec.laneBits, spec.laneBits, passthroughBytes) + return true, false, c.storeVectorScalarRegister(destination.Reg, spec.laneBits, "%"+low, base, mask, properties.zeroing) +} + +func (c *amd64Ctx) emitScaleFBits(lanes, laneBits int, source1, source2 string, controls amd64ScaleFControls) string { + vectorType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + mantissaBits, bias, exponentMax, scaleClamp := 23, 127, 255, 512 + exponentMask := uint64(0x7f800000) + mantissaMask := uint64(0x007fffff) + quietBit := uint64(0x00400000) + signBit := uint64(0x80000000) + indefinite := uint64(0xffc00000) + if laneBits == 64 { + mantissaBits, bias, exponentMax, scaleClamp = 52, 1023, 2047, 4096 + exponentMask = uint64(0x7ff0000000000000) + mantissaMask = uint64(0x000fffffffffffff) + quietBit = uint64(0x0008000000000000) + signBit = uint64(0x8000000000000000) + indefinite = uint64(0xfff8000000000000) + } + implicitBit := uint64(1) << mantissaBits + result := "zeroinitializer" + for lane := 0; lane < lanes; lane++ { + x, y := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", x, vectorType, source1, lane) + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", y, vectorType, source2, lane) + laneResult := c.emitScaleFLane(laneBits, mantissaBits, bias, exponentMax, scaleClamp, + exponentMask, mantissaMask, quietBit, signBit, indefinite, implicitBit, + "%"+x, "%"+y, controls) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, result, laneBits, laneResult, lane) + result = "%" + inserted + } + return result +} + +type amd64ScaleFClass struct { + bits, sign, fraction, exponent string + zero, nan, snan, infinity string + positiveInfinity, negativeInfinity string +} + +func (c *amd64Ctx) classifyScaleFValue(laneBits, mantissaBits, exponentMax int, exponentMask, mantissaMask, quietBit, signBit uint64, bits, daz string) amd64ScaleFClass { + sign, fraction, exponentField := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", sign, laneBits, bits, signBit) + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", fraction, laneBits, bits, mantissaMask) + fmt.Fprintf(c.b, " %%%s = and i%d %s, %d\n", exponentField, laneBits, bits, exponentMask) + exponentZero, fractionNonzero, denormal := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", exponentZero, laneBits, exponentField) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", fractionNonzero, laneBits, fraction) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", denormal, exponentZero, fractionNonzero) + flush := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %s\n", flush, denormal, daz) + effective := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %s\n", effective, flush, laneBits, sign, laneBits, bits) + + effectiveFraction, shiftedExponent, effectiveExponent := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", effectiveFraction, laneBits, effective, mantissaMask) + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %d\n", shiftedExponent, laneBits, effective, mantissaBits) + if laneBits == 32 { + fmt.Fprintf(c.b, " %%%s = and i32 %%%s, %d\n", effectiveExponent, shiftedExponent, exponentMax) + } else { + exponentWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, %d\n", exponentWide, shiftedExponent, exponentMax) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", effectiveExponent, exponentWide) + } + abs, isZero, isExponentMax, isFractionNonzero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", abs, laneBits, effective, signBit-1) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", isZero, laneBits, abs) + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, %d\n", isExponentMax, effectiveExponent, exponentMax) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", isFractionNonzero, laneBits, effectiveFraction) + isNaN, isInfinity := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isNaN, isExponentMax, isFractionNonzero) + fractionQuiet, quietClear, isSNaN := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", fractionQuiet, laneBits, effectiveFraction, quietBit) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", quietClear, laneBits, fractionQuiet) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isSNaN, isNaN, quietClear) + fractionZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, 0\n", fractionZero, laneBits, effectiveFraction) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isInfinity, isExponentMax, fractionZero) + isNegative, positiveInfinity, negativeInfinity := c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", isNegative, laneBits, sign) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", negativeInfinity, isInfinity, isNegative) + notNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notNegative, isNegative) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", positiveInfinity, isInfinity, notNegative) + return amd64ScaleFClass{ + bits: "%" + effective, sign: "%" + sign, fraction: "%" + effectiveFraction, exponent: "%" + effectiveExponent, + zero: "%" + isZero, nan: "%" + isNaN, snan: "%" + isSNaN, infinity: "%" + isInfinity, + positiveInfinity: "%" + positiveInfinity, negativeInfinity: "%" + negativeInfinity, + } +} + +func (c *amd64Ctx) emitScaleFLane(laneBits, mantissaBits, bias, exponentMax, scaleClamp int, exponentMask, mantissaMask, quietBit, signBit, indefinite, implicitBit uint64, source1, source2 string, controls amd64ScaleFControls) string { + x := c.classifyScaleFValue(laneBits, mantissaBits, exponentMax, exponentMask, mantissaMask, quietBit, signBit, source1, controls.daz) + y := c.classifyScaleFValue(laneBits, mantissaBits, exponentMax, exponentMask, mantissaMask, quietBit, signBit, source2, controls.daz) + + yNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %s, 0\n", yNegative, laneBits, y.sign) + yExponentNegative, yExponentLarge := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %s, %d\n", yExponentNegative, y.exponent, bias) + largeExponent := 9 + if laneBits == 64 { + largeExponent = 12 + } + fmt.Fprintf(c.b, " %%%s = icmp sge i32 %s, %d\n", yExponentLarge, y.exponent, bias+largeExponent) + yUnbiased, yShift := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %s, %d\n", yUnbiased, y.exponent, bias) + fmt.Fprintf(c.b, " %%%s = sub i32 %d, %%%s\n", yShift, mantissaBits, yUnbiased) + yExponentOutOfRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", yExponentOutOfRange, yExponentNegative, yExponentLarge) + safeYShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %%%s\n", safeYShift, yExponentOutOfRange, mantissaBits, yShift) + yShiftLane := "%" + safeYShift + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, safeYShift) + yShiftLane = "%" + wide + } + ySignificand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", ySignificand, laneBits, y.fraction, implicitBit) + yIntegerWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %s\n", yIntegerWide, laneBits, ySignificand, yShiftLane) + yInteger := "%" + yIntegerWide + if laneBits == 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", narrow, yIntegerWide) + yInteger = "%" + narrow + } + oneShifted, remainderMask, remainder, remainderNonzero := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %s\n", oneShifted, laneBits, yShiftLane) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, 1\n", remainderMask, laneBits, oneShifted) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %%%s\n", remainder, laneBits, ySignificand, remainderMask) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", remainderNonzero, laneBits, remainder) + negativeInteger, negativeFloor := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 0, %s\n", negativeInteger, yInteger) + remainderAdjust := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %%%s to i32\n", remainderAdjust, remainderNonzero) + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, %%%s\n", negativeFloor, negativeInteger, remainderAdjust) + signedInteger := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %s\n", signedInteger, yNegative, negativeFloor, yInteger) + smallScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -1, i32 0\n", smallScale, yNegative) + clampedScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 -%d, i32 %d\n", clampedScale, yNegative, scaleClamp, scaleClamp) + nonzeroScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %%%s\n", nonzeroScale, yExponentLarge, clampedScale, signedInteger) + finiteScale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %%%s\n", finiteScale, yExponentNegative, smallScale, nonzeroScale) + scale := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i32 0, i32 %%%s\n", scale, y.zero, finiteScale) + + xExponentZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 0\n", xExponentZero, x.exponent) + xSignificandNormal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", xSignificandNormal, laneBits, x.fraction, implicitBit) + xSignificand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %s, i%d %%%s\n", xSignificand, xExponentZero, laneBits, x.fraction, laneBits, xSignificandNormal) + leadingZeros := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i%d @llvm.ctlz.i%d(i%d %%%s, i1 false)\n", leadingZeros, laneBits, laneBits, laneBits, xSignificand) + leadingZeros32 := "%" + leadingZeros + if laneBits == 64 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i32\n", narrow, leadingZeros) + leadingZeros32 = "%" + narrow + } + normalizeShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %s, %d\n", normalizeShift, leadingZeros32, laneBits-1-mantissaBits) + normalizeShiftLane := "%" + normalizeShift + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, normalizeShift) + normalizeShiftLane = "%" + wide + } + normalizedSignificand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %%%s, %s\n", normalizedSignificand, laneBits, xSignificand, normalizeShiftLane) + xBaseExponentNormal, xBaseExponent := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %s, %d\n", xBaseExponentNormal, x.exponent, bias) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %%%s\n", xBaseExponent, xExponentZero, 1-bias, xBaseExponentNormal) + xNormalizedExponent := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, %%%s\n", xNormalizedExponent, xBaseExponent, normalizeShift) + targetExponent := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %%%s\n", targetExponent, xNormalizedExponent, scale) + + overflow, normal := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, %d\n", overflow, targetExponent, bias) + fmt.Fprintf(c.b, " %%%s = icmp sge i32 %%%s, %d\n", normal, targetExponent, 1-bias) + normalExponent32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i32 %%%s, %d\n", normalExponent32, targetExponent, bias) + normalExponent := "%" + normalExponent32 + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, normalExponent32) + normalExponent = "%" + wide + } + normalExponentBits, normalFraction, normalMagnitude, normalBits := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d %s, %d\n", normalExponentBits, laneBits, normalExponent, mantissaBits) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %d\n", normalFraction, laneBits, normalizedSignificand, mantissaMask) + fmt.Fprintf(c.b, " %%%s = or i%d %%%s, %%%s\n", normalMagnitude, laneBits, normalExponentBits, normalFraction) + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", normalBits, laneBits, x.sign, normalMagnitude) + + subnormalShiftRaw, subnormalShiftHigh := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %d, %%%s\n", subnormalShiftRaw, 1-bias, targetExponent) + fmt.Fprintf(c.b, " %%%s = icmp sgt i32 %%%s, %d\n", subnormalShiftHigh, subnormalShiftRaw, mantissaBits+2) + subnormalShiftMax := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %%%s\n", subnormalShiftMax, subnormalShiftHigh, mantissaBits+2, subnormalShiftRaw) + subnormalShiftLow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i32 %%%s, 1\n", subnormalShiftLow, subnormalShiftMax) + subnormalShift := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 1, i32 %%%s\n", subnormalShift, subnormalShiftLow, subnormalShiftMax) + subnormalShiftLane := "%" + subnormalShift + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, subnormalShift) + subnormalShiftLane = "%" + wide + } + quotient, subOneShifted, subRemainderMask, subRemainder := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr i%d %%%s, %s\n", quotient, laneBits, normalizedSignificand, subnormalShiftLane) + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %s\n", subOneShifted, laneBits, subnormalShiftLane) + fmt.Fprintf(c.b, " %%%s = sub i%d %%%s, 1\n", subRemainderMask, laneBits, subOneShifted) + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, %%%s\n", subRemainder, laneBits, normalizedSignificand, subRemainderMask) + subRemainderNonzero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", subRemainderNonzero, laneBits, subRemainder) + halfShift32 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, 1\n", halfShift32, subnormalShift) + halfShift := "%" + halfShift32 + if laneBits == 64 { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, halfShift32) + halfShift = "%" + wide + } + half, aboveHalf, equalHalf, quotientOdd, tieOdd, nearest := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl i%d 1, %s\n", half, laneBits, halfShift) + fmt.Fprintf(c.b, " %%%s = icmp ugt i%d %%%s, %%%s\n", aboveHalf, laneBits, subRemainder, half) + fmt.Fprintf(c.b, " %%%s = icmp eq i%d %%%s, %%%s\n", equalHalf, laneBits, subRemainder, half) + quotientLowBit := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i%d %%%s, 1\n", quotientLowBit, laneBits, quotient) + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %%%s, 0\n", quotientOdd, laneBits, quotientLowBit) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", tieOdd, equalHalf, quotientOdd) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", nearest, aboveHalf, tieOdd) + xNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i%d %s, 0\n", xNegative, laneBits, x.sign) + increment := c.emitScaledRoundIncrement(controls.rounding, "%"+xNegative, "%"+subRemainderNonzero, "%"+nearest) + incrementBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to i%d\n", incrementBits, increment, laneBits) + roundedSubnormal, subnormalBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i%d %%%s, %%%s\n", roundedSubnormal, laneBits, quotient, incrementBits) + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", subnormalBits, laneBits, x.sign, roundedSubnormal) + isSubnormal := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i%d %%%s, %d\n", isSubnormal, laneBits, roundedSubnormal, implicitBit) + flushInexact, flushSubnormal := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %s, %%%s\n", flushInexact, controls.ftz, subRemainderNonzero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", flushSubnormal, flushInexact, isSubnormal) + underflowBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %s, i%d %%%s\n", underflowBits, flushSubnormal, laneBits, x.sign, laneBits, subnormalBits) + + toInfinity := c.emitScaledRoundIncrement(controls.rounding, "%"+xNegative, "true", "true") + overflowInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %d\n", overflowInfinity, toInfinity, laneBits, exponentMask, laneBits, exponentMask-1) + overflowBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", overflowBits, laneBits, x.sign, overflowInfinity) + finiteNonOverflow, finiteBits := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", finiteNonOverflow, normal, laneBits, normalBits, laneBits, underflowBits) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", finiteBits, overflow, laneBits, overflowBits, laneBits, finiteNonOverflow) + + yInfinityResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %s\n", yInfinityResult, y.positiveInfinity, laneBits, exponentMask, laneBits, x.sign) + yInfinitySigned := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %%%s\n", yInfinitySigned, laneBits, x.sign, yInfinityResult) + general := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", general, y.infinity, laneBits, yInfinitySigned, laneBits, finiteBits) + xZeroInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %s\n", xZeroInfinity, y.positiveInfinity, laneBits, indefinite, laneBits, x.bits) + xZeroResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %s\n", xZeroResult, y.infinity, laneBits, xZeroInfinity, laneBits, x.bits) + withZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", withZero, x.zero, laneBits, xZeroResult, laneBits, general) + xInfinityResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d %s\n", xInfinityResult, y.negativeInfinity, laneBits, indefinite, laneBits, x.bits) + withInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", withInfinity, x.infinity, laneBits, xInfinityResult, laneBits, withZero) + quietY := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", quietY, laneBits, y.bits, quietBit) + withYNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", withYNaN, y.nan, laneBits, quietY, laneBits, withInfinity) + quietX := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i%d %s, %d\n", quietX, laneBits, x.bits, quietBit) + xQNaNInfinity := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %d, i%d 0\n", xQNaNInfinity, y.positiveInfinity, laneBits, exponentMask, laneBits) + xQNaNResult := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", xQNaNResult, y.infinity, laneBits, xQNaNInfinity, laneBits, quietX) + xQNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %s, true\n", xQNaN, x.nan) + notSNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", notSNaN, x.snan) + isQNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", isQNaN, xQNaN, notSNaN) + withXQNaN := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i%d %%%s, i%d %%%s\n", withXQNaN, isQNaN, laneBits, xQNaNResult, laneBits, withYNaN) + final := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %%%s\n", final, x.snan, laneBits, quietX, laneBits, withXQNaN) + return "%" + final +} diff --git a/amd64_lower_scalef_test.go b/amd64_lower_scalef_test.go new file mode 100644 index 000000000..b570aa953 --- /dev/null +++ b/amd64_lower_scalef_test.go @@ -0,0 +1,292 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64ScaleFGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64BinaryFloatingSpec{ + "VSCALEFPS": {laneBits: 32, mode: amd64BinaryFloatingScale}, + "VSCALEFPD": {laneBits: 64, mode: amd64BinaryFloatingScale}, + "VSCALEFSS": {laneBits: 32, mode: amd64BinaryFloatingScale, scalar: true}, + "VSCALEFSD": {laneBits: 64, mode: amd64BinaryFloatingScale, scalar: true}, + } + for op, want := range expected { + if got, ok := amd64BinaryFloatingSpecs[op]; !ok { + t.Errorf("SCALEF grammar omitted %s", op) + } else if got != want { + t.Errorf("SCALEF grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86ScaleFCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scalefforms(SB),$0-0\n") + for _, op := range []string{"VSCALEFPS", "VSCALEFPD"} { + for _, width := range []string{"X", "Y", "Z"} { + last := 31 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s %s1, %s2, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s 8(AX), %s2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST 16(AX), %s2, %s%d\n", op, width, width, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s1, %s2, K1, %s%d\n", op, width, width, width, last) + fmt.Fprintf(&source, "\t%s.Z 24(AX), %s2, K2, %s%d\n", op, width, width, last) + fmt.Fprintf(&source, "\t%s.BCST.Z 32(AX), %s2, K3, %s%d\n", op, width, width, last) + } + if width == "Z" { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s Z1, Z2, Z%d\n", op, rounding, last) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.%s.Z Z1, Z2, K4, Z%d\n", op, rounding, last) + } + } + } + } + } + for _, op := range []string{"VSCALEFSS", "VSCALEFSD"} { + fmt.Fprintf(&source, "\t%s X1, X2, X7\n", op) + fmt.Fprintf(&source, "\t%s 40(AX), X2, X7\n", op) + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s X1, X2, X7\n", op, rounding) + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X2, K5, X31\n", op) + fmt.Fprintf(&source, "\t%s.Z 48(AX), X2, K6, X31\n", op) + fmt.Fprintf(&source, "\t%s.RN_SAE.Z X1, X2, K7, X31\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"scalefforms": {Name: "scalefforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scalef-"+target.name+".ll", "scalef-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScaleFRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VSCALEFPS X0, X1"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, X1, X2, X3, X4"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, Y1, Y2"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, 8(AX), X2"}, + {goarch: "amd64", instruction: "VSCALEFPS X0, X1, K0, X2"}, + {goarch: "amd64", instruction: "VSCALEFPS.Z X0, X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFPS.BCST X0, X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFPS.RN_SAE 8(AX), Z1, Z2"}, + {goarch: "amd64", instruction: "VSCALEFPS.SAE Z0, Z1, Z2"}, + {goarch: "amd64", instruction: "VSCALEFPS.RN_SAE Y0, Y1, Y2"}, + {goarch: "amd64", instruction: "VSCALEFSS Y0, X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFSS.BCST 8(AX), X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFSS.RN_SAE 8(AX), X1, X2"}, + {goarch: "amd64", instruction: "VSCALEFSS X0, Y1, X2"}, + {goarch: "386", instruction: "VSCALEFPS X0, X1, K1, X2"}, + {goarch: "386", instruction: "VSCALEFPS Z8, Z1, Z2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's SCALEF tables", test.instruction) + } + }) + } +} + +func TestAMD64ScaleFRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT scalefsemantics(SB),$0-72 + MOVQ out+0(FP), AX + MOVQ value32+8(FP), BX + MOVQ scale32+16(FP), CX + MOVQ value64+24(FP), DX + MOVQ scale64+32(FP), R8 + MOVQ old32+40(FP), R9 + MOVQ old64+48(FP), R10 + MOVQ upper+56(FP), R11 + MOVQ mask+64(FP), R12 + KMOVQ R12, K1 + VMOVDQU32 0(BX), Z1 + VMOVDQU32 0(CX), Z2 + VSCALEFPS Z2, Z1, Z3 + VMOVDQU32 Z3, 0(AX) + VSCALEFPS.RN_SAE Z2, Z1, Z3 + VMOVDQU32 Z3, 64(AX) + VMOVDQU32 0(R9), Z3 + VSCALEFPS Z2, Z1, K1, Z3 + VMOVDQU32 Z3, 128(AX) + VSCALEFPS.BCST 0(CX), Z1, Z3 + VMOVDQU32 Z3, 192(AX) + VMOVDQU64 0(DX), Z4 + VMOVDQU64 0(R8), Z5 + VSCALEFPD Z5, Z4, Z6 + VMOVDQU64 Z6, 256(AX) + VSCALEFPD.RN_SAE Z5, Z4, Z6 + VMOVDQU64 Z6, 320(AX) + VMOVDQU64 0(R10), Z6 + VSCALEFPD.Z Z5, Z4, K1, Z6 + VMOVDQU64 Z6, 384(AX) + VSCALEFPD.BCST 0(R8), Z4, Z6 + VMOVDQU64 Z6, 448(AX) + VMOVDQU64 0(R11), X7 + VSCALEFSS 60(CX), X7, X8 + VMOVDQU64 X8, 512(AX) + VSCALEFSD 56(R8), X7, X9 + VMOVDQU64 X9, 528(AX) + STC + SETCS 544(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"scalefsemantics": { + Name: "scalefsemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: Ptr, Index: 7, Field: -1}, + {Offset: 64, Type: I64, Index: 8, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +#include +#include +extern void scalefsemantics(uint8_t *, const uint32_t *, const uint32_t *, const uint64_t *, const uint64_t *, const uint32_t *, const uint64_t *, const uint64_t *, uint64_t); +static uint32_t u32(const uint8_t *p) { uint32_t v; memcpy(&v,p,4); return v; } +static uint64_t u64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +static uint32_t scale32(uint32_t x, uint32_t y, int daz) { + const uint32_t em=UINT32_C(0x7f800000), fm=UINT32_C(0x007fffff), qb=UINT32_C(0x00400000), sb=UINT32_C(0x80000000), ind=UINT32_C(0xffc00000); + if(daz && !(x&em) && (x&fm)) x&=sb; if(daz && !(y&em) && (y&fm)) y&=sb; + uint32_t xf=x&fm,yf=y&fm; int xn=(x&em)==em&&xf, xs=xn&&!(xf&qb), xi=(x&~sb)==em, xz=(x&~sb)==0; + int yn=(y&em)==em&&yf, yi=(y&~sb)==em, yp=yi&&!(y&sb), ym=yi&&(y&sb); + if(xs) return x|qb; + if(xn) return yi?(yp?em:0):(x|qb); + if(yn) return y|qb; + if(xi) return ym?ind:x; + if(xz) return yp?ind:x; + if(yp) return (x&sb)|em; if(ym) return x&sb; + float a,b,r; memcpy(&a,&x,4); memcpy(&b,&y,4); int n=b>=512?512:(b<=-512?-512:(int)floorf(b)); r=scalbnf(a,n); memcpy(&x,&r,4); return x; +} +static uint64_t scale64(uint64_t x, uint64_t y, int daz) { + const uint64_t em=UINT64_C(0x7ff0000000000000), fm=UINT64_C(0x000fffffffffffff), qb=UINT64_C(0x0008000000000000), sb=UINT64_C(0x8000000000000000), ind=UINT64_C(0xfff8000000000000); + if(daz && !(x&em) && (x&fm)) x&=sb; if(daz && !(y&em) && (y&fm)) y&=sb; + uint64_t xf=x&fm,yf=y&fm; int xn=(x&em)==em&&xf, xs=xn&&!(xf&qb), xi=(x&~sb)==em, xz=(x&~sb)==0; + int yn=(y&em)==em&&yf, yi=(y&~sb)==em, yp=yi&&!(y&sb), ym=yi&&(y&sb); + if(xs) return x|qb; + if(xn) return yi?(yp?em:0):(x|qb); + if(yn) return y|qb; + if(xi) return ym?ind:x; + if(xz) return yp?ind:x; + if(yp) return (x&sb)|em; if(ym) return x&sb; + double a,b,r; memcpy(&a,&x,8); memcpy(&b,&y,8); int n=b>=4096?4096:(b<=-4096?-4096:(int)floor(b)); r=scalbn(a,n); memcpy(&x,&r,8); return x; +} +int main(void) { + const uint32_t v32[16]={0x3f800000,0xbf800000,0x7f7fffff,0xff7fffff,0x00800000,0x80800000,1,0x80000001,0,0x80000000,0x7f800000,0xff800000,0x7fc12345,0x7f812345,0x3fc00000,0xbfc00000}; + const uint32_t s32[16]={0x3ff33333,0xbf8ccccd,0x3f800000,0x3f800000,0xbf800000,0xbf800000,0x3f800000,0x3f800000,0x7f800000,0xff800000,0xff800000,0x7f800000,0x7f800000,0,0xc3160000,0xc3160000}; + const uint64_t v64[8]={UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff),UINT64_C(0x0010000000000000),1,UINT64_C(0x7ff8123456789abc),UINT64_C(0x7ff0123456789abc)}; + const uint64_t s64[8]={UINT64_C(0x3ffe666666666666),UINT64_C(0xbff199999999999a),UINT64_C(0x3ff0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0x7ff0000000000000),0}; + uint32_t old32[16]; uint64_t old64[8]; + uint64_t upper[2]={UINT64_C(0x112233443fc00000),UINT64_C(0x99aabbccddeeff00)}; + for(int i=0;i<16;i++)old32[i]=UINT32_C(0x41000000)+(uint32_t)i; for(int i=0;i<8;i++)old64[i]=UINT64_C(0x4020000000000000)+(uint64_t)i; + const uint64_t mask=UINT64_C(0xa55a); unsigned original=_mm_getcsr(); + const int fes[4]={FE_TONEAREST,FE_DOWNWARD,FE_UPWARD,FE_TOWARDZERO}; + for(unsigned rc=0;rc<4;rc++) for(unsigned daz=0;daz<2;daz++) { + uint8_t out[545]={0}; unsigned csr=(original&~UINT32_C(0xe040))|(rc<<13)|(daz?UINT32_C(0x40):0); _mm_setcsr(csr); fesetround(fes[rc]); + scalefsemantics(out,v32,s32,v64,s64,old32,old64,upper,mask); + for(int i=0;i<16;i++) { + uint32_t want=scale32(v32[i],s32[i],daz); if(u32(out+i*4)!=want)return 10+rc; + int saved=fegetround(); fesetround(FE_TONEAREST); uint32_t rn=scale32(v32[i],s32[i],daz); fesetround(saved); if(u32(out+64+i*4)!=rn)return 20+rc; + if(u32(out+128+i*4)!=(((mask>>i)&1)?want:old32[i]))return 30+rc; + if(u32(out+192+i*4)!=scale32(v32[i],s32[0],daz))return 40+rc; + } + for(int i=0;i<8;i++) { + uint64_t want=scale64(v64[i],s64[i],daz); if(u64(out+256+i*8)!=want)return 50+rc; + int saved=fegetround(); fesetround(FE_TONEAREST); uint64_t rn=scale64(v64[i],s64[i],daz); fesetround(saved); if(u64(out+320+i*8)!=rn)return 60+rc; + if(u64(out+384+i*8)!=(((mask>>i)&1)?want:0))return 70+rc; + if(u64(out+448+i*8)!=scale64(v64[i],s64[0],daz))return 80+rc; + } + if(u32(out+512)!=scale32((uint32_t)upper[0],s32[15],daz)||memcmp(out+516,(uint8_t*)upper+4,12))return 90+rc; + if(u64(out+528)!=scale64(upper[0],s64[7],daz)||u64(out+536)!=upper[1])return 100+rc; + if(out[544]!=1)return 110+rc; + } + _mm_setcsr(original); fesetround(FE_TONEAREST); return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalef_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_scatter.go b/amd64_lower_scatter.go new file mode 100644 index 000000000..9825f9d90 --- /dev/null +++ b/amd64_lower_scatter.go @@ -0,0 +1,127 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64ScatterSpecs = map[string]amd64GatherSpec{ + "VPSCATTERDD": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VPSCATTERQQ": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VSCATTERDPS": {table: amd64GatherDPS, elemBits: 32, indexBits: 32}, + "VSCATTERQPD": {table: amd64GatherDPS, elemBits: 64, indexBits: 64}, + "VPSCATTERDQ": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VSCATTERDPD": {table: amd64GatherDPD, elemBits: 64, indexBits: 32}, + "VPSCATTERQD": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, + "VSCATTERQPS": {table: amd64GatherQPS, elemBits: 32, indexBits: 64}, +} + +// lowerScatter implements all Go 1.27 packed integer and floating scatter +// spellings. Values are kept as integer bit patterns, which is exact for the +// floating variants as well. +func (c *amd64Ctx) lowerScatter(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + if strings.Contains(rawOp, ".") { + baseOp := strings.SplitN(rawOp, ".", 2)[0] + if _, known := amd64ScatterSpecs[baseOp]; known { + return true, false, fmt.Errorf("amd64 %s has no suffixes in its Go 1.27 table: %q", baseOp, ins.Raw) + } + return false, false, nil + } + spec, ok := amd64ScatterSpecs[rawOp] + if !ok { + return false, false, nil + } + if len(ins.Args) != 3 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpMem { + return true, false, fmt.Errorf("amd64 %s expects data vector, K1-K7, VSIB memory: %q", rawOp, ins.Raw) + } + data, mask, memory := ins.Args[0], ins.Args[1], ins.Args[2].Mem + maskIndex, validMask := amd64ParseKReg(mask.Reg) + if !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s requires K1-K7: %q", rawOp, ins.Raw) + } + + dataBytes := amd64VectorByteWidth(data.Reg) + indexBytes := amd64VectorByteWidth(memory.Index) + if dataBytes == 0 || indexBytes == 0 || !amd64GatherTableAllows(spec.table, true, dataBytes, indexBytes) { + return true, false, fmt.Errorf("amd64 %s data/index widths are outside its Go 1.27 table: %q", rawOp, ins.Raw) + } + if err := c.validateGatherMemory(memory, indexBytes, true); err != nil { + return true, false, fmt.Errorf("amd64 %s invalid VSIB memory: %q: %w", rawOp, ins.Raw, err) + } + if _, ok := amd64GatherRegisterIndex(data.Reg, dataBytes, true, c.goarch, false); !ok { + return true, false, fmt.Errorf("amd64 %s data register is outside its Go 1.27 class: %q", rawOp, ins.Raw) + } + if _, ok := amd64GatherRegisterIndex(memory.Index, indexBytes, true, c.goarch, true); !ok { + return true, false, fmt.Errorf("amd64 %s VSIB index is outside its Go 1.27 class: %q", rawOp, ins.Raw) + } + return true, false, c.emitScatter(spec, memory, mask.Reg, data.Reg, dataBytes, indexBytes) +} + +func (c *amd64Ctx) emitScatter(spec amd64GatherSpec, memory MemRef, mask, data Reg, dataBytes, indexBytes int) error { + dataValue, err := c.loadGatherVector(data, dataBytes) + if err != nil { + return err + } + dataLanes := dataBytes * 8 / spec.elemBits + dataType := fmt.Sprintf("<%d x i%d>", dataLanes, spec.elemBits) + dataValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", dataValues, dataBytes, dataValue, dataType) + + indexValue, err := c.loadGatherVector(memory.Index, indexBytes) + if err != nil { + return err + } + indexLanes := indexBytes * 8 / spec.indexBits + indexType := fmt.Sprintf("<%d x i%d>", indexLanes, spec.indexBits) + indexValues := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", indexValues, indexBytes, indexValue, indexType) + activeLanes := dataLanes + if indexLanes < activeLanes { + activeLanes = indexLanes + } + + maskValue, err := c.loadK(mask) + if err != nil { + return err + } + base, err := c.gatherBaseAddress(memory) + if err != nil { + return err + } + for lane := 0; lane < activeLanes; lane++ { + indexLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", indexLane, indexType, indexValues, lane) + wideIndex := "%" + indexLane + if spec.indexBits == 32 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sext i32 %%%s to i64\n", extended, indexLane) + wideIndex = "%" + extended + } + if memory.Scale != 0 && memory.Scale != 1 { + scaled := c.newTmp() + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", scaled, wideIndex, memory.Scale) + wideIndex = "%" + scaled + } + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %s\n", address, base, wideIndex) + pointer, pointerType := c.gatherPointer(memory, "%"+address) + + condition := amd64MaskBitI1(c, maskValue, lane) + dataLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %%%s, i32 %d\n", dataLane, dataType, dataValues, lane) + storeLabel := c.newTmp() + "_scatter_store" + skipLabel := c.newTmp() + "_scatter_skip" + joinLabel := c.newTmp() + "_scatter_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n", condition, storeLabel, skipLabel) + fmt.Fprintf(c.b, "%s:\n", storeLabel) + fmt.Fprintf(c.b, " store i%d %%%s, %s %s, align 1\n", spec.elemBits, dataLane, pointerType, pointer) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", skipLabel) + fmt.Fprintf(c.b, " br label %%%s\n", joinLabel) + fmt.Fprintf(c.b, "%s:\n", joinLabel) + } + // Like EVEX gather, scatter clears completed mask bits to support precise + // restart after a fault. A normally completed instruction leaves zero. + return c.storeK(mask, "0") +} diff --git a/amd64_lower_scatter_prefetch_test.go b/amd64_lower_scatter_prefetch_test.go new file mode 100644 index 000000000..252126a46 --- /dev/null +++ b/amd64_lower_scatter_prefetch_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "fmt" + "sort" + "strings" + "testing" +) + +func TestAMD64VSIBPrefetchGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64VSIBPrefetchSpecs)) + for op := range amd64VSIBPrefetchSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "VGATHERPF0DPD", "VGATHERPF0DPS", "VGATHERPF0QPD", "VGATHERPF0QPS", + "VGATHERPF1DPD", "VGATHERPF1DPS", "VGATHERPF1QPD", "VGATHERPF1QPS", + "VSCATTERPF0DPD", "VSCATTERPF0DPS", "VSCATTERPF0QPD", "VSCATTERPF0QPS", + "VSCATTERPF1DPD", "VSCATTERPF1DPS", "VSCATTERPF1QPD", "VSCATTERPF1QPS", + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("VSIB-prefetch grammar opcodes = %v, want %v", got, want) + } + for op, spec := range amd64VSIBPrefetchSpecs { + name := string(op) + wantDirection := amd64VSIBPrefetchGather + if strings.HasPrefix(name, "VSCATTER") { + wantDirection = amd64VSIBPrefetchScatter + } + wantHint := 0 + if strings.Contains(name, "PF1") { + wantHint = 1 + } + shape := name[strings.Index(name, "PF")+3:] + wantIndexBits := 32 + if shape[0] == 'Q' { + wantIndexBits = 64 + } + wantElemBits := 32 + if shape[2] == 'D' { + wantElemBits = 64 + } + wantIndexBytes := 64 + if shape == "DPD" { + wantIndexBytes = 32 + } + if spec.direction != wantDirection || spec.hint != wantHint || spec.indexBits != wantIndexBits || + spec.elemBits != wantElemBits || spec.indexBytes != wantIndexBytes { + t.Errorf("VSIB-prefetch grammar %s = %+v, want direction=%d hint=%d elem=%d index=%d vector=%d", + op, spec, wantDirection, wantHint, wantElemBits, wantIndexBits, wantIndexBytes) + } + } +} + +func TestTranslateX86ScatterPrefetchCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scatterprefetchforms(SB),$0-0\n") + for _, op := range []string{"VSCATTERPF0DPD", "VSCATTERPF1DPD"} { + fmt.Fprintf(&source, "\t%s K1, 8(SP)(Y2*8)\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s K7, 16(R11)(Y20*4)\n", op) + } + } + for _, op := range []string{ + "VSCATTERPF0DPS", "VSCATTERPF0QPD", "VSCATTERPF0QPS", + "VSCATTERPF1DPS", "VSCATTERPF1QPD", "VSCATTERPF1QPS", + } { + fmt.Fprintf(&source, "\t%s K2, 24(BX)(Z3*4)\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s K6, 32(SP)(Z20*8)\n", op) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"scatterprefetchforms": {Name: "scatterprefetchforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "scatter-prefetch-"+target.name+".ll", "scatter-prefetch-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScatterPrefetchRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VSCATTERPF0DPD K0, (AX)(Y2*8)"}, + {goarch: "amd64", instruction: "VSCATTERPF1DPD K1, (AX)(Z2*8)"}, + {goarch: "amd64", instruction: "VSCATTERPF0DPS K1, (AX)(Y2*4)"}, + {goarch: "amd64", instruction: "VSCATTERPF0QPD K1, (AX)"}, + {goarch: "amd64", instruction: "VSCATTERPF0QPS (AX)(Z2*4), K1"}, + {goarch: "amd64", instruction: "VSCATTERPF1DPS.Z K1, (AX)(Z2*4)"}, + {goarch: "amd64", instruction: "VSCATTERPF1QPD K1, (AX)(Z2*4), Z3"}, + {goarch: "amd64", instruction: "VSCATTERPF1QPS X1, (AX)(Z2*4)"}, + {goarch: "386", instruction: "VSCATTERPF0DPD K1, (R8)(Y2*8)"}, + {goarch: "386", instruction: "VSCATTERPF1DPS K1, (AX)(Z8*4)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scatter-prefetch tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_scatter_test.go b/amd64_lower_scatter_test.go new file mode 100644 index 000000000..9c4a0c4d2 --- /dev/null +++ b/amd64_lower_scatter_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86ScatterCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 exposes eight integer/floating spellings over three exact + // data/index width tables. Every form is data, K1-K7, VSIB memory. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT scatterforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VPSCATTERDD", "VPSCATTERQQ", "VSCATTERDPS", "VSCATTERQPD"} { + fmt.Fprintf(&source, "\t%s X20, K1, (AX)(X2*1)\n", op) + fmt.Fprintf(&source, "\t%s Y20, K2, 8(BX)(Y3*2)\n", op) + fmt.Fprintf(&source, "\t%s Z%d, K3, 16(CX)(Z4*4)\n", op, lastZ) + } + for _, op := range []string{"VPSCATTERDQ", "VSCATTERDPD"} { + fmt.Fprintf(&source, "\t%s X20, K4, 24(DX)(X5*8)\n", op) + fmt.Fprintf(&source, "\t%s Y20, K5, 32(SI)(X6*1)\n", op) + fmt.Fprintf(&source, "\t%s Z%d, K6, 40(DI)(Y7*2)\n", op, lastZ) + } + for _, op := range []string{"VPSCATTERQD", "VSCATTERQPS"} { + fmt.Fprintf(&source, "\t%s X20, K7, 48(AX)(X2*4)\n", op) + fmt.Fprintf(&source, "\t%s X20, K1, 56(BX)(Y3*8)\n", op) + fmt.Fprintf(&source, "\t%s Y20, K2, 64(CX)(Z4*1)\n", op) + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"scatterforms": {Name: "scatterforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scatter-"+target.name+".ll", "scatter-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ScatterRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPSCATTERDD X1, (AX)(X2*1)", + "VPSCATTERDD X1, K0, (AX)(X2*1)", + "VPSCATTERDD X1, K1, (AX)", + "VPSCATTERDD Y1, K1, (AX)(X2*1)", + "VPSCATTERDQ X1, K1, (AX)(Y2*1)", + "VPSCATTERQD Y1, K1, (AX)(Y2*1)", + "VPSCATTERDD X1, K1, (AX)(X2*3)", + "VPSCATTERDD.Z X1, K1, (AX)(X2*1)", + "VPSCATTERDD X1, K1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86ScatterRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86ScatterRejected(t, "386", "i386-unknown-linux-gnu", "VPSCATTERDD X1, K1, (R8)(X2*1)") + assertX86ScatterRejected(t, "386", "i386-unknown-linux-gnu", "VPSCATTERDD X1, K1, (AX)(X8*1)") + assertX86ScatterRejected(t, "386", "i386-unknown-linux-gnu", "VPSCATTERDD Z8, K1, (AX)(Z7*1)") +} + +func assertX86ScatterRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scatter forms", instruction) + } +} diff --git a/amd64_lower_segment_base.go b/amd64_lower_segment_base.go new file mode 100644 index 000000000..a56e91f8a --- /dev/null +++ b/amd64_lower_segment_base.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SegmentBaseDirection uint8 + +const ( + amd64SegmentBaseRead amd64SegmentBaseDirection = iota + amd64SegmentBaseWrite +) + +type amd64SegmentBaseSpec struct { + segment Reg + direction amd64SegmentBaseDirection + bits int +} + +// amd64SegmentBaseSpecs is the complete Go 1.27 FSGSBASE grammar. Segment, +// transfer direction, and width are data; all eight opcodes share one Yrl +// register parser and one exact inline-assembly emitter. +var amd64SegmentBaseSpecs = map[Op]amd64SegmentBaseSpec{ + "RDFSBASEL": {segment: FS, direction: amd64SegmentBaseRead, bits: 32}, + "RDFSBASEQ": {segment: FS, direction: amd64SegmentBaseRead, bits: 64}, + "RDGSBASEL": {segment: GS, direction: amd64SegmentBaseRead, bits: 32}, + "RDGSBASEQ": {segment: GS, direction: amd64SegmentBaseRead, bits: 64}, + "WRFSBASEL": {segment: FS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRFSBASEQ": {segment: FS, direction: amd64SegmentBaseWrite, bits: 64}, + "WRGSBASEL": {segment: GS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRGSBASEQ": {segment: GS, direction: amd64SegmentBaseWrite, bits: 64}, +} + +type amd64SegmentBaseForm struct { + register Reg +} + +func (c *amd64Ctx) lowerSegmentBase(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64SegmentBaseSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseSegmentBaseForm(baseOp, ins) + if err != nil { + return true, false, err + } + return c.emitSegmentBase(spec, form) +} + +func (c *amd64Ctx) parseSegmentBaseForm(op string, ins Instr) (amd64SegmentBaseForm, error) { + var form amd64SegmentBaseForm + if len(ins.Args) != 1 { + return form, fmt.Errorf("%s %s expects one Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + operand := ins.Args[0] + if operand.Kind != OpReg || !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return form, fmt.Errorf("%s %s expects a Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + return amd64SegmentBaseForm{register: operand.Reg}, nil +} + +func (spec amd64SegmentBaseSpec) mnemonic() string { + prefix := "rd" + if spec.direction == amd64SegmentBaseWrite { + prefix = "wr" + } + width := "l" + if spec.bits == 64 { + width = "q" + } + return prefix + strings.ToLower(string(spec.segment)) + "base" + width +} + +func (c *amd64Ctx) emitSegmentBase(spec amd64SegmentBaseSpec, form amd64SegmentBaseForm) (bool, bool, error) { + typ := amd64IntegerTypeForBits(spec.bits) + constraints := "~{dirflag},~{fpsr},~{flags}" + assembly := spec.mnemonic() + " $0" + registerConstraint := "r" + if c.goarch == "386" { + // Intel defines FSGSBASE only in 64-bit mode, but Go 1.27's 386 + // assembler accepts the L spellings and emits their F3 0F AE /r + // encodings. LLVM intentionally rejects the mnemonic for i386. Emit + // Go's exact EAX encoding and use a fixed-register constraint so the + // object remains byte-compatible without claiming runtime viability. + modRM := byte(0xc0) + if spec.segment == GS { + modRM += 8 + } + if spec.direction == amd64SegmentBaseWrite { + modRM += 16 + } + assembly = fmt.Sprintf(".byte 0xf3, 0x0f, 0xae, 0x%02x", modRM) + registerConstraint = "{ax}" + } + if spec.direction == amd64SegmentBaseRead { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q()\n", + result, typ, assembly, "="+registerConstraint+","+constraints) + if err := c.storeRegSized(form.register, typ, "%"+result); err != nil { + return true, false, err + } + return true, false, nil + } + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.register}, typ) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", + assembly, registerConstraint+","+constraints, typ, value) + return true, false, nil +} diff --git a/amd64_lower_segment_base_test.go b/amd64_lower_segment_base_test.go new file mode 100644 index 000000000..f17f24329 --- /dev/null +++ b/amd64_lower_segment_base_test.go @@ -0,0 +1,147 @@ +package plan9asm + +import ( + "reflect" + "sort" + "strings" + "testing" +) + +func TestAMD64SegmentBaseGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64SegmentBaseSpecs)) + for op := range amd64SegmentBaseSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "RDFSBASEL", "RDFSBASEQ", "RDGSBASEL", "RDGSBASEQ", + "WRFSBASEL", "WRFSBASEQ", "WRGSBASEL", "WRGSBASEQ", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("segment-base grammar opcodes = %v, want %v", got, want) + } + for op, want := range map[Op]amd64SegmentBaseSpec{ + "RDFSBASEL": {segment: FS, direction: amd64SegmentBaseRead, bits: 32}, + "RDFSBASEQ": {segment: FS, direction: amd64SegmentBaseRead, bits: 64}, + "RDGSBASEL": {segment: GS, direction: amd64SegmentBaseRead, bits: 32}, + "RDGSBASEQ": {segment: GS, direction: amd64SegmentBaseRead, bits: 64}, + "WRFSBASEL": {segment: FS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRFSBASEQ": {segment: FS, direction: amd64SegmentBaseWrite, bits: 64}, + "WRGSBASEL": {segment: GS, direction: amd64SegmentBaseWrite, bits: 32}, + "WRGSBASEQ": {segment: GS, direction: amd64SegmentBaseWrite, bits: 64}, + } { + if got := amd64SegmentBaseSpecs[op]; got != want { + t.Errorf("segment-base grammar %s = %+v, want %+v", op, got, want) + } + if got := want.mnemonic(); strings.ToUpper(got) != string(op) { + t.Errorf("segment-base grammar %s emits %q", op, got) + } + } +} + +func TestTranslateX86SegmentBaseCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT segmentbaseforms(SB),$0-0 + RDFSBASEL AX + RDGSBASEL SP + WRFSBASEL CX + WRGSBASEL SP +` + if target.goarch == "amd64" { + source += ` RDFSBASEQ R11 + RDGSBASEQ R12 + WRFSBASEQ R13 + WRGSBASEQ R14 +` + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"segmentbaseforms": {Name: "segmentbaseforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if want := `"target-features"="+fsgsbase"`; !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + if target.goarch == "386" { + for _, encoding := range []string{ + ".byte 0xf3, 0x0f, 0xae, 0xc0", + ".byte 0xf3, 0x0f, 0xae, 0xc8", + ".byte 0xf3, 0x0f, 0xae, 0xd0", + ".byte 0xf3, 0x0f, 0xae, 0xd8", + } { + if !strings.Contains(ir, encoding) { + t.Errorf("386 IR is missing Go-compatible encoding %q:\n%s", encoding, ir) + } + } + } + compileLLVMToObject(t, llc, target.triple, "segment-base-"+target.name+".ll", "segment-base-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SegmentBaseRejectsFormsOutsideGoTables(t *testing.T) { + tests := []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "RDFSBASEL $1"}, + {goarch: "amd64", instruction: "RDFSBASEQ (BX)"}, + {goarch: "amd64", instruction: "RDGSBASEL X0"}, + {goarch: "amd64", instruction: "RDGSBASEQ"}, + {goarch: "amd64", instruction: "WRFSBASEL $1"}, + {goarch: "amd64", instruction: "WRFSBASEQ (BX)"}, + {goarch: "amd64", instruction: "WRGSBASEL X0"}, + {goarch: "amd64", instruction: "WRGSBASEQ.Z AX"}, + } + for _, op := range []string{ + "RDFSBASEQ", "RDGSBASEQ", "WRFSBASEQ", "WRGSBASEQ", + } { + tests = append(tests, struct { + goarch string + instruction string + }{goarch: "386", instruction: op + " AX"}) + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-base tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_segment_query.go b/amd64_lower_segment_query.go new file mode 100644 index 000000000..d9f46ce3b --- /dev/null +++ b/amd64_lower_segment_query.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SegmentQueryKind uint8 + +const ( + amd64SegmentQueryAccessRights amd64SegmentQueryKind = iota + amd64SegmentQueryLimit + amd64SegmentQueryReadable + amd64SegmentQueryWritable +) + +type amd64SegmentQuerySpec struct { + kind amd64SegmentQueryKind + bits int +} + +// amd64SegmentQuerySpecs is the complete Go 1.27 LAR/LSL/VERR/VERW grammar. +// The table records only query kind and optional destination width; all four +// families share one selector-source parser and one exact lowering mechanism. +var amd64SegmentQuerySpecs = map[Op]amd64SegmentQuerySpec{ + "LARW": {kind: amd64SegmentQueryAccessRights, bits: 16}, + "LARL": {kind: amd64SegmentQueryAccessRights, bits: 32}, + "LARQ": {kind: amd64SegmentQueryAccessRights, bits: 64}, + "LSLW": {kind: amd64SegmentQueryLimit, bits: 16}, + "LSLL": {kind: amd64SegmentQueryLimit, bits: 32}, + "LSLQ": {kind: amd64SegmentQueryLimit, bits: 64}, + "VERR": {kind: amd64SegmentQueryReadable}, + "VERW": {kind: amd64SegmentQueryWritable}, +} + +type amd64SegmentQueryForm struct { + selector Operand + destination Reg +} + +func (c *amd64Ctx) lowerSegmentQuery(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64SegmentQuerySpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.bits == 64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseSegmentQueryForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + return c.emitSegmentQuery(spec, form) +} + +func (c *amd64Ctx) parseSegmentQueryForm(op string, spec amd64SegmentQuerySpec, ins Instr) (amd64SegmentQueryForm, error) { + var form amd64SegmentQueryForm + wantArgs := 2 + if spec.bits == 0 { + wantArgs = 1 + } + if len(ins.Args) != wantArgs { + if wantArgs == 1 { + return form, fmt.Errorf("%s %s expects one selector source: %q", c.goarch, op, ins.Raw) + } + return form, fmt.Errorf("%s %s expects selector source and GP destination: %q", c.goarch, op, ins.Raw) + } + selector := ins.Args[0] + if selector.Kind == OpReg { + if !isX86YrlRegisterForArch(selector.Reg, c.goarch) { + return form, fmt.Errorf("%s %s selector register is outside Go 1.27's Yml class: %q", c.goarch, op, ins.Raw) + } + } else if !isAMD64MemoryOperand(selector) { + return form, fmt.Errorf("%s %s selector must be a GP register or memory: %q", c.goarch, op, ins.Raw) + } + if selector.Kind == OpMem && !x86MemoryRegistersValidForArch(selector.Mem, c.goarch) { + return form, fmt.Errorf("%s %s selector address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + form.selector = selector + if wantArgs == 1 { + return form, nil + } + destination := ins.Args[1] + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return form, fmt.Errorf("%s %s destination must be a Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + form.destination = destination.Reg + return form, nil +} + +func (c *amd64Ctx) emitSegmentQuery(spec amd64SegmentQuerySpec, form amd64SegmentQueryForm) (bool, bool, error) { + selector, err := c.evalIntSized(form.selector, I16) + if err != nil { + return true, false, err + } + if spec.bits == 0 { + mnemonic := "verr" + if spec.kind == amd64SegmentQueryWritable { + mnemonic = "verw" + } + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call i8 asm sideeffect %q, %q(i16 %s)\n", + status, mnemonic+" $1; sete $0", "=q,r,~{dirflag},~{fpsr},~{flags}", selector) + c.storeSegmentQueryZF("%" + status) + return true, false, nil + } + + typ := amd64IntegerTypeForBits(spec.bits) + previousDestination, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.destination}, typ) + if err != nil { + return true, false, err + } + + // Intel specifies that a failed descriptor check clears ZF but leaves the + // destination unchanged. Constraint 0 therefore ties the old destination + // value to the result instead of declaring a write-only output. SETE does + // not alter flags, so the second result captures the instruction's ZF while + // the translator's other virtual flag slots remain unchanged. + result := c.newTmp() + mnemonic := "lar" + if spec.kind == amd64SegmentQueryLimit { + mnemonic = "lsl" + } + width := map[int]string{16: "w", 32: "l", 64: "q"}[spec.bits] + assembly := mnemonic + width + " $3, $0; sete $1" + fmt.Fprintf(c.b, " %%%s = call { %s, i8 } asm sideeffect %q, %q(%s %s, i16 %s)\n", + result, typ, assembly, "=r,=q,0,r,~{dirflag},~{fpsr},~{flags}", typ, previousDestination, selector) + value := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 0\n", value, typ, result) + fmt.Fprintf(c.b, " %%%s = extractvalue { %s, i8 } %%%s, 1\n", status, typ, result) + c.storeSegmentQueryZF("%" + status) + if err := c.storeRegSized(form.destination, typ, "%"+value); err != nil { + return true, false, err + } + return true, false, nil +} + +func (c *amd64Ctx) storeSegmentQueryZF(status string) { + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %s, 0\n", zero, status) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) +} diff --git a/amd64_lower_segment_query_test.go b/amd64_lower_segment_query_test.go new file mode 100644 index 000000000..0ef048896 --- /dev/null +++ b/amd64_lower_segment_query_test.go @@ -0,0 +1,290 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +func TestAMD64SegmentQueryGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64SegmentQuerySpecs)) + for op := range amd64SegmentQuerySpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{"LARL", "LARQ", "LARW", "LSLL", "LSLQ", "LSLW", "VERR", "VERW"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("segment-query grammar opcodes = %v, want %v", got, want) + } + for op, want := range map[Op]amd64SegmentQuerySpec{ + "LARW": {kind: amd64SegmentQueryAccessRights, bits: 16}, + "LARL": {kind: amd64SegmentQueryAccessRights, bits: 32}, + "LARQ": {kind: amd64SegmentQueryAccessRights, bits: 64}, + "LSLW": {kind: amd64SegmentQueryLimit, bits: 16}, + "LSLL": {kind: amd64SegmentQueryLimit, bits: 32}, + "LSLQ": {kind: amd64SegmentQueryLimit, bits: 64}, + "VERR": {kind: amd64SegmentQueryReadable}, + "VERW": {kind: amd64SegmentQueryWritable}, + } { + if got := amd64SegmentQuerySpecs[op]; got != want { + t.Errorf("segment-query grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86SegmentQueryCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT segmentqueryforms(SB),$0-0 + LARW AX, DX + LARW 2(BX), CX + LARL CX, AX + LARL 4(BX), DX + LSLW DX, CX + LSLW 6(BX), AX + LSLL AX, DX + LSLL 8(BX), CX +` + if target.goarch == "386" { + source += "\tLARW AX, SP\n\tLARL AX, SP\n\tLSLW AX, SP\n\tLSLL AX, SP\n" + } + if target.goarch == "amd64" { + source += ` LARW R11, R12 + LARW 10(R13), R14 + LARQ R11, R12 + LARQ 12(R13), R14 + LSLW R11, R12 + LSLW 14(R13), R14 + LSLQ R11, R12 + LSLQ 16(R13), R14 +` + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"segmentqueryforms": {Name: "segmentqueryforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "segment-query-"+target.name+".ll", "segment-query-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SegmentQueryRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "LARW $1, AX"}, + {goarch: "amd64", instruction: "LARL AX, (BX)"}, + {goarch: "amd64", instruction: "LSLQ X0, AX"}, + {goarch: "amd64", instruction: "LSLW AX, X0"}, + {goarch: "amd64", instruction: "LARW.Z AX, DX"}, + {goarch: "386", instruction: "LARQ AX, DX"}, + {goarch: "386", instruction: "LSLQ AX, DX"}, + {goarch: "386", instruction: "LARW R11, AX"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-query tables", test.instruction) + } + }) + } +} + +func TestTranslateX86SegmentValidationCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + high := "DI" + if target.goarch == "amd64" { + high = "R11" + } + source := "TEXT segmentvalidationforms(SB),$0-0\n" + + "\tVERR AX\n\tVERR (BX)\n\tVERR " + high + "\n" + + "\tVERW DX\n\tVERW 2(BX)\n\tVERW " + high + "\n\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"segmentvalidationforms": {Name: "segmentvalidationforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "segment-validation-"+target.name+".ll", "segment-validation-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SegmentValidationRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "VERR $1"}, + {goarch: "amd64", instruction: "VERW X0"}, + {goarch: "amd64", instruction: "VERR AX, DX"}, + {goarch: "amd64", instruction: "VERW.Z AX"}, + {goarch: "386", instruction: "VERR R11"}, + {goarch: "386", instruction: "VERW (R11)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-validation table", test.instruction) + } + }) + } +} + +func TestAMD64SegmentQueryRuntimeSemantics(t *testing.T) { + if runtime.GOARCH != "amd64" { + // Rosetta advertises amd64 execution but raises SIGILL for LAR/LSL. + // Object compilation remains covered above; execute the architectural + // descriptor checks only on native amd64 hardware. + t.Skip("LAR/LSL runtime semantics require native amd64 hardware") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT segmentquerysemantics(SB),NOSPLIT,$0-16 + MOVQ selector+0(FP), AX + MOVQ out+8(FP), DI + MOVQ $0x1122334455667788, CX + LARQ AX, CX + MOVQ CX, 0(DI) + SETEQ 8(DI) + MOVQ $0x2233445566778899, CX + XORQ AX, AX + LARQ AX, CX + MOVQ CX, 16(DI) + SETEQ 24(DI) + MOVQ selector+0(FP), AX + MOVQ $0x33445566778899aa, DX + LSLQ AX, DX + MOVQ DX, 32(DI) + SETEQ 40(DI) + MOVQ $0x445566778899aabb, DX + XORQ AX, AX + LSLQ AX, DX + MOVQ DX, 48(DI) + SETEQ 56(DI) + MOVQ selector+0(FP), AX + STC + VERR AX + SETEQ 57(DI) + SETCS 58(DI) + VERW AX + SETEQ 59(DI) + XORQ AX, AX + VERR AX + SETEQ 60(DI) + VERW AX + SETEQ 61(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"segmentquerysemantics": { + Name: "segmentquerysemantics", Args: []LLVMType{I64, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void segmentquerysemantics(uint64_t, uint8_t *); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v; } +int main(void) { + uint16_t cs; + uint8_t out[64] = {0}; + __asm__ volatile ("mov %%cs,%0" : "=r"(cs)); + segmentquerysemantics(cs, out); + if (out[8] != 1 || out[40] != 1) return 10; + if (out[24] != 0 || out[56] != 0) return 11; + if (load64(out + 16) != UINT64_C(0x2233445566778899)) return 12; + if (load64(out + 48) != UINT64_C(0x445566778899aabb)) return 13; + if (out[57] != 1 || out[58] != 1) return 14; + if (out[59] != 0 || out[60] != 0 || out[61] != 0) return 15; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "segment_query_semantics", triple, ir, mainC, nil) +} diff --git a/amd64_lower_segment_register.go b/amd64_lower_segment_register.go new file mode 100644 index 000000000..99c3db24a --- /dev/null +++ b/amd64_lower_segment_register.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerSegmentRegisterMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + if (baseOp != "MOVW" && baseOp != "MOVL" && baseOp != "MOVQ") || len(ins.Args) != 2 { + return false, false, nil + } + sourceSegment := x86SegmentRegisterOperand(ins.Args[0]) + destinationSegment := x86SegmentRegisterOperand(ins.Args[1]) + if !sourceSegment && !destinationSegment { + return false, false, nil + } + if baseOp != "MOVW" { + return true, false, fmt.Errorf("%s segment-register moves use MOVW in Go 1.27's ymovtab: %q", c.goarch, ins.Raw) + } + if !x86SegmentMoveSuffixAccepted(rawOp) { + return true, false, fmt.Errorf("%s segment-register MOVW has an invalid x86 suffix: %q", c.goarch, ins.Raw) + } + if sourceSegment == destinationSegment { + return true, false, fmt.Errorf("%s MOVW segment-register form expects exactly one segment register: %q", c.goarch, ins.Raw) + } + if destinationSegment { + source := ins.Args[0] + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, c.goarch) { + return true, false, fmt.Errorf("%s MOVW to a segment register expects a Yml source: %q", c.goarch, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s MOVW to a segment register expects a Yml source: %q", c.goarch, ins.Raw) + } + value, err := c.evalIntSized(source, I16) + if err != nil { + return true, false, err + } + return true, false, c.storeX86SegmentSelector(ins.Args[1].Reg, value) + } + + destination := ins.Args[1] + if destination.Kind == OpReg { + if !isX86YrlRegisterForArch(destination.Reg, c.goarch) { + return true, false, fmt.Errorf("%s MOVW from a segment register expects a Yml destination: %q", c.goarch, ins.Raw) + } + } else if !isAMD64MemoryOperand(destination) { + return true, false, fmt.Errorf("%s MOVW from a segment register expects a Yml destination: %q", c.goarch, ins.Raw) + } + call := c.loadX86SegmentSelector(ins.Args[0].Reg) + if destination.Kind == OpReg { + return true, false, c.storeRegSized(destination.Reg, I32, call) + } + selector := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i32 %s to i16\n", selector, call) + return true, false, c.storeScalarIntegerOperand(destination, I16, "%"+selector) +} + +func (c *amd64Ctx) loadX86SegmentSelector(segment Reg) string { + call := c.newTmp() + assembly := fmt.Sprintf("movl %%%s, ${0:k}", strings.ToLower(string(segment))) + fmt.Fprintf(c.b, " %%%s = call i32 asm sideeffect %q, %q()\n", + call, assembly, "=r,~{memory},~{dirflag},~{fpsr},~{flags}") + return "%" + call +} + +func (c *amd64Ctx) storeX86SegmentSelector(segment Reg, selector string) error { + assembly := fmt.Sprintf("movw $0, %%%s", strings.ToLower(string(segment))) + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i16 %s)\n", + assembly, "r,~{memory},~{dirflag},~{fpsr},~{flags}", selector) + return nil +} + +func x86SegmentMoveSuffixAccepted(rawOp string) bool { + if rawOp == "MOVW" { + return true + } + suffix := strings.TrimPrefix(rawOp, "MOVW.") + switch suffix { + case "Z", "SAE", "SAE.Z", + "RN_SAE", "RZ_SAE", "RD_SAE", "RU_SAE", + "RN_SAE.Z", "RZ_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", + "BCST", "BCST.Z": + // Go 1.27's ymovtab path accepts every syntactically valid x86 suffix + // here and emits the same 8C/8E bytes. Preserve that compatibility. + return true + default: + return false + } +} + +func x86SegmentRegisterOperand(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + switch operand.Reg { + case ES, CS, SS, DS, FS, GS: + return true + default: + return false + } +} diff --git a/amd64_lower_segment_register_test.go b/amd64_lower_segment_register_test.go new file mode 100644 index 000000000..bc933755b --- /dev/null +++ b/amd64_lower_segment_register_test.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86SegmentRegisterCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT segmentregisterforms(SB),$0-0\n") + for _, segment := range []string{"ES", "CS", "SS", "DS", "FS", "GS"} { + fmt.Fprintf(&source, "\tMOVW AX, %s\n", segment) + fmt.Fprintf(&source, "\tMOVW %s, AX\n", segment) + fmt.Fprintf(&source, "\tMOVW 8(BX), %s\n", segment) + fmt.Fprintf(&source, "\tMOVW %s, 16(BX)\n", segment) + } + for _, suffix := range []string{ + "Z", "SAE", "SAE.Z", + "RN_SAE", "RZ_SAE", "RD_SAE", "RU_SAE", + "RN_SAE.Z", "RZ_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", + "BCST", "BCST.Z", + } { + fmt.Fprintf(&source, "\tMOVW.%s AX, DS\n", suffix) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "segmentregisterforms": {Name: "segmentregisterforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "segment-register-"+target.name+".ll", "segment-register-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86SegmentRegisterRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "MOVL AX, DS", + "MOVQ AX, DS", + "MOVW AL, DS", + "MOVW X0, DS", + "MOVW DS, X0", + "MOVW DS, FS", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + requireX86GoAssemblerResult(t, goarch, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + triple := "i386-unknown-linux-gnu" + if goarch == "amd64" { + triple = "x86_64-unknown-linux-gnu" + } + assertX86SegmentRegisterRejected(t, goarch, triple, instruction) + } + }) + } +} + +func assertX86SegmentRegisterRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's segment-register MOVW table for %s", instruction, goarch) + } +} + +func TestX86SegmentRegistersDoNotShadowARM64Conditions(t *testing.T) { + file, err := Parse(ArchARM64, "TEXT select(SB),$0-0\n\tCSEL CS, R11, R3, R3\n\tRET\n") + if err != nil { + t.Fatal(err) + } + if len(file.Funcs) != 1 || len(file.Funcs[0].Instrs) < 2 || len(file.Funcs[0].Instrs[1].Args) != 4 { + t.Fatalf("unexpected ARM64 parse: %+v", file) + } + condition := file.Funcs[0].Instrs[1].Args[0] + if condition.Kind == OpReg { + t.Fatalf("ARM64 CS condition was parsed as an x86 segment register: %+v", condition) + } +} diff --git a/amd64_lower_set.go b/amd64_lower_set.go new file mode 100644 index 000000000..2400a9f27 --- /dev/null +++ b/amd64_lower_set.go @@ -0,0 +1,53 @@ +package plan9asm + +import ( + "fmt" +) + +// lowerConditionalSet implements the complete Go 1.27 SETcc family. All 16 +// names use yscond, whose single operand is the byte-register-or-memory Ymb +// class. +func (c *amd64Ctx) lowerConditionalSet(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, recognized, err := x86ConditionalSetProperties(op, ins, c.goarch) + if !recognized { + return false, false, nil + } + if err != nil { + return true, false, err + } + condition, err := c.x86Condition(form.condition) + if err != nil { + return true, false, err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i8 1, i8 0\n", value, condition) + if err := c.storeConditionalSetByte(form.destination, "%"+value); err != nil { + return true, false, err + } + return true, false, nil +} + +func (c *amd64Ctx) storeConditionalSetByte(dst Operand, value string) error { + switch dst.Kind { + case OpReg: + return c.storeRegSized(dst.Reg, I8, value) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i8 %s, %s %s, align 1\n", value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i8 %s, ptr %s, align 1\n", value, ptr) + return nil + case OpFP: + return c.storeFPResult(dst.FPOffset, I8, value) + default: + return fmt.Errorf("expected Ymb destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_shiftrotate.go b/amd64_lower_shiftrotate.go new file mode 100644 index 000000000..8a53f6800 --- /dev/null +++ b/amd64_lower_shiftrotate.go @@ -0,0 +1,358 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerScalarShiftRotate implements the complete Go 1.27 yshb/yshl family: +// RCL/RCR/ROL/ROR/SAR/SAL/SHL/SHR at B/W/L/Q widths, plus the special +// three-operand SHL/SHR W/L/Q double-shift table. +func (c *amd64Ctx) lowerScalarShiftRotate(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + stem, bits, recognized := amd64ScalarShiftRotateProperties(baseOp) + if !recognized { + return false, false, nil + } + if _, _, err := x86ScalarShiftProperties(op, ins, c.goarch); err != nil { + return true, false, err + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && bits == 64 { + return true, false, fmt.Errorf("386 %s is illegal in 32-bit mode: %q", baseOp, ins.Raw) + } + if len(ins.Args) == 3 { + if (stem != "SHL" && stem != "SHR") || bits == 8 { + return true, false, fmt.Errorf("%s %s has no three-operand form in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + return c.lowerScalarDoubleShift(baseOp, stem == "SHL", bits, ins) + } + if len(ins.Args) != 2 { + return true, false, fmt.Errorf("%s %s expects count, destination: %q", c.goarch, baseOp, ins.Raw) + } + count, err := c.scalarShiftCount(ins.Args[0], bits, false) + if err != nil { + return true, false, fmt.Errorf("%s %s count: %w", c.goarch, baseOp, err) + } + dst := ins.Args[1] + if !c.scalarShiftDestination(dst, bits) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Ymb/Yml class: %q", c.goarch, baseOp, ins.Raw) + } + ty := amd64IntegerTypeForBits(bits) + value, err := c.evalIntSized(dst, ty) + if err != nil { + return true, false, err + } + var result string + if stem == "RCL" || stem == "RCR" { + result = c.emitScalarCarryRotate(stem, bits, value, count) + } else { + result = c.emitScalarShiftRotate(stem, bits, value, count) + c.emitScalarShiftFlags(stem, bits, value, result, count, false) + } + if err := c.storeScalarIntegerOperand(dst, ty, result); err != nil { + return true, false, err + } + return true, false, nil +} + +func amd64ScalarShiftRotateProperties(op string) (stem string, bits int, ok bool) { + switch op { + case "RCLB", "RCRB", "ROLB", "RORB", "SARB", "SALB", "SHLB", "SHRB": + bits = 8 + case "RCLW", "RCRW", "ROLW", "RORW", "SARW", "SALW", "SHLW", "SHRW": + bits = 16 + case "RCLL", "RCRL", "ROLL", "RORL", "SARL", "SALL", "SHLL", "SHRL": + bits = 32 + case "RCLQ", "RCRQ", "ROLQ", "RORQ", "SARQ", "SALQ", "SHLQ", "SHRQ": + bits = 64 + default: + return "", 0, false + } + return op[:len(op)-1], bits, true +} + +func (c *amd64Ctx) emitScalarCarryRotate(stem string, bits int, value, count64 string) string { + // Model RCL/RCR as a rotate of the destination plus CF. Intel masks the + // count to five (six for Q) bits first; scalarShiftCount already did that. + // B/W then reduce it modulo 9/17, while L/Q values are already below 33/65. + totalBits := bits + 1 + effective64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = urem i64 %s, %d\n", effective64, count64, totalBits) + + wideType := LLVMType(fmt.Sprintf("i%d", totalBits)) + valueWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to %s\n", valueWide, amd64IntegerTypeForBits(bits), value, wideType) + oldCF := c.loadFlag(c.flagsCFSlot) + cfWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i1 %s to %s\n", cfWide, oldCF, wideType) + cfHigh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %d\n", cfHigh, wideType, cfWide, bits) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", combined, wideType, valueWide, cfHigh) + + toWideCount := func(value string) string { + converted := c.newTmp() + if totalBits <= 64 { + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", converted, value, wideType) + } else { + fmt.Fprintf(c.b, " %%%s = zext i64 %s to %s\n", converted, value, wideType) + } + return "%" + converted + } + effective := toWideCount("%" + effective64) + inverseRaw := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %d, %%%s\n", inverseRaw, totalBits, effective64) + inverse64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = urem i64 %%%s, %d\n", inverse64, inverseRaw, totalBits) + inverse := toWideCount("%" + inverse64) + + left := c.newTmp() + right := c.newTmp() + if stem == "RCL" { + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", left, wideType, combined, effective) + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %s\n", right, wideType, combined, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %s\n", right, wideType, combined, effective) + fmt.Fprintf(c.b, " %%%s = shl %s %%%s, %s\n", left, wideType, combined, inverse) + } + rotated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", rotated, wideType, left, right) + + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to %s\n", result, wideType, rotated, amd64IntegerTypeForBits(bits)) + cfShifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", cfShifted, wideType, rotated, bits) + newCF := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", newCF, wideType, cfShifted) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", newCF, c.flagsCFSlot) + + // OF uses the masked count, not the modulo-9/17 count: e.g. RCLB $10 + // rotates once but has undefined OF. Keep an old representative for those + // undefined cases; the byte-effect consumer independently kills availability. + msbShifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", msbShifted, amd64IntegerTypeForBits(bits), result, bits-1) + msb := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", msb, amd64IntegerTypeForBits(bits), msbShifted) + definedOF := c.newTmp() + if stem == "RCL" { + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, %%%s\n", definedOF, msb, newCF) + } else { + nextShifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = lshr %s %%%s, %d\n", nextShifted, amd64IntegerTypeForBits(bits), result, bits-2) + nextMSB := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %%%s to i1\n", nextMSB, amd64IntegerTypeForBits(bits), nextShifted) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, %%%s\n", definedOF, msb, nextMSB) + } + isOne := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 1\n", isOne, count64) + oldOF := c.loadFlag(c.flagsOFSlot) + newOF := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i1 %%%s, i1 %s\n", newOF, isOne, definedOF, oldOF) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", newOF, c.flagsOFSlot) + return "%" + result +} + +func (c *amd64Ctx) scalarShiftCount(count Operand, bits int, doubleShift bool) (string, error) { + mask := int64(31) + if bits == 64 { + mask = 63 + } + switch count.Kind { + case OpImm: + if count.ImmRaw != "" || count.ImmIsFloat { + return "", fmt.Errorf("count must be a resolved integer immediate") + } + if doubleShift { + if count.Imm < -128 || count.Imm > 127 { + return "", fmt.Errorf("immediate is outside Go 1.27's signed-imm8 class") + } + } else if count.Imm < 0 || count.Imm > 255 { + return "", fmt.Errorf("immediate is outside Go 1.27's unsigned-imm8 class") + } + return fmt.Sprintf("%d", count.Imm&mask), nil + case OpReg: + if count.Reg != CL && count.Reg != CX { + return "", fmt.Errorf("register count must be CL or CX") + } + value, err := c.loadReg(count.Reg) + if err != nil { + return "", err + } + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", masked, value, mask) + return "%" + masked, nil + default: + return "", fmt.Errorf("count must be an immediate, CL, or CX") + } +} + +func (c *amd64Ctx) scalarShiftDestination(dst Operand, bits int) bool { + if dst.Kind == OpReg { + if bits == 8 { + return isGoYmbRegisterForArch(dst.Reg, c.goarch) + } + return isX86YrlRegisterForArch(dst.Reg, c.goarch) + } + return isAMD64MemoryOperand(dst) +} + +func (c *amd64Ctx) emitScalarShiftRotate(stem string, bits int, value, count64 string) string { + ty := amd64IntegerTypeForBits(bits) + amount64 := count64 + if stem == "ROL" || stem == "ROR" { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", masked, count64, bits-1) + amount64 = "%" + masked + } + amount := amount64 + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", truncated, amount64, ty) + amount = "%" + truncated + } + + if stem == "ROL" || stem == "ROR" { + inverseNeg := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", inverseNeg, ty, amount) + inverse := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", inverse, ty, inverseNeg, bits-1) + left := c.newTmp() + right := c.newTmp() + if stem == "ROL" { + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", left, ty, value, amount) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", right, ty, value, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", right, ty, value, amount) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", left, ty, value, inverse) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", result, ty, left, right) + return "%" + result + } + + inRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %s, %d\n", inRange, count64, bits) + safe64 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %s, i64 %d\n", safe64, inRange, count64, bits-1) + safe := "%" + safe64 + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to %s\n", truncated, safe64, ty) + safe = "%" + truncated + } + shifted := c.newTmp() + switch stem { + case "SHL", "SAL": + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", shifted, ty, value, safe) + case "SHR": + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", shifted, ty, value, safe) + case "SAR": + fmt.Fprintf(c.b, " %%%s = ashr %s %s, %s\n", shifted, ty, value, safe) + return "%" + shifted + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s 0\n", result, inRange, ty, shifted, ty) + return "%" + result +} + +func (c *amd64Ctx) lowerScalarDoubleShift(baseOp string, left bool, bits int, ins Instr) (bool, bool, error) { + count, err := c.scalarShiftCount(ins.Args[0], bits, true) + if err != nil { + return true, false, fmt.Errorf("%s %s count: %w", c.goarch, baseOp, err) + } + if ins.Args[1].Kind != OpReg || !isX86YrlRegisterForArch(ins.Args[1].Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source must be a Yrl GP register: %q", c.goarch, baseOp, ins.Raw) + } + if !c.scalarShiftDestination(ins.Args[2], bits) { + return true, false, fmt.Errorf("%s %s destination must be a Yml GP register or memory: %q", c.goarch, baseOp, ins.Raw) + } + ty := amd64IntegerTypeForBits(bits) + source, err := c.evalIntSized(ins.Args[1], ty) + if err != nil { + return true, false, err + } + destination, err := c.evalIntSized(ins.Args[2], ty) + if err != nil { + return true, false, err + } + + // W double shifts mask the hardware count to five bits, leaving counts + // 17-31 architecturally undefined. Reduce that undefined region modulo the + // lane width so generated LLVM never shifts by the type width. + effective64 := count + if bits == 16 { + masked := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and i64 %s, 15\n", masked, count) + effective64 = "%" + masked + } + effective := effective64 + if bits != 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to %s\n", truncated, effective64, ty) + effective = "%" + truncated + } + inverseNeg := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s 0, %s\n", inverseNeg, ty, effective) + inverse := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and %s %%%s, %d\n", inverse, ty, inverseNeg, bits-1) + first := c.newTmp() + second := c.newTmp() + if left { + fmt.Fprintf(c.b, " %%%s = shl %s %s, %s\n", first, ty, destination, effective) + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %%%s\n", second, ty, source, inverse) + } else { + fmt.Fprintf(c.b, " %%%s = lshr %s %s, %s\n", first, ty, destination, effective) + fmt.Fprintf(c.b, " %%%s = shl %s %s, %%%s\n", second, ty, source, inverse) + } + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or %s %%%s, %%%s\n", merged, ty, first, second) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", zero, ty, effective) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %s, %s %%%s\n", result, zero, ty, destination, ty, merged) + value := "%" + result + if bits == 16 { + full := c.newTmp() + corrected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i64 %s, 16\n", full, count) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %s, i16 %s\n", corrected, full, source, value) + value = "%" + corrected + } + c.emitScalarShiftFlags(baseOp[:len(baseOp)-1], bits, destination, value, count, true) + if err := c.storeScalarIntegerOperand(ins.Args[2], ty, value); err != nil { + return true, false, err + } + return true, false, nil +} + +func (c *amd64Ctx) storeScalarIntegerOperand(dst Operand, ty LLVMType, value string) error { + switch dst.Kind { + case OpReg: + return c.storeRegSized(dst.Reg, ty, value) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", ty, value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", ty, value, ptr) + return nil + case OpFP: + return c.storeFPResult(dst.FPOffset, ty, value) + default: + return fmt.Errorf("expected GP register or memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_shuffle_blocks.go b/amd64_lower_shuffle_blocks.go new file mode 100644 index 000000000..56e51629b --- /dev/null +++ b/amd64_lower_shuffle_blocks.go @@ -0,0 +1,136 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var amd64Shuffle128BitBlockLaneBits = map[string]int{ + "VSHUFF32X4": 32, + "VSHUFI32X4": 32, + "VSHUFF64X2": 64, + "VSHUFI64X2": 64, +} + +// lowerShuffle128BitBlocks implements the complete Go 1.27 _yvshuff32x4 +// table. Floating and integer spellings have identical bit-level behavior. +func (c *amd64Ctx) lowerShuffle128BitBlocks(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + laneBits, ok := amd64Shuffle128BitBlockLaneBits[baseOp] + if !ok { + return false, false, nil + } + suffixes, err := amd64ParseTwoSourcePermuteSuffixes(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + if len(ins.Args) != 4 && len(ins.Args) != 5 { + return true, false, fmt.Errorf("amd64 %s expects $imm8, first source, second source, [K mask,] destination: %q", baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].ImmRaw != "" || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return true, false, fmt.Errorf("amd64 %s expects an unsigned-byte immediate: %q", baseOp, ins.Raw) + } + masked := len(ins.Args) == 5 + if masked && c.goarch == "386" { + return true, false, fmt.Errorf("386 %s masked form exceeds the Go assembler frontend's operand limit: %q", baseOp, ins.Raw) + } + if suffixes.zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s .Z requires a K1-K7 mask: %q", baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + byteWidth := 0 + if destination.Kind == OpReg { + byteWidth = amd64VectorByteWidth(destination.Reg) + } + if byteWidth != 32 && byteWidth != 64 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 %s expects an in-range Y or Z destination: %q", baseOp, ins.Raw) + } + second := ins.Args[2] + if !c.isGoPackedVectorMoveRegister(second, byteWidth) { + return true, false, fmt.Errorf("amd64 %s second source must match the destination width: %q", baseOp, ins.Raw) + } + first := ins.Args[1] + if first.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(first, byteWidth) { + return true, false, fmt.Errorf("amd64 %s first source register must match the destination width: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s first source must be a matching vector register or memory: %q", baseOp, ins.Raw) + } + if suffixes.broadcast && !isAMD64MemoryOperand(first) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory first source: %q", baseOp, ins.Raw) + } + + mask := "" + if masked { + maskArg := ins.Args[3] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + maskIndex, validMask := amd64ParseKReg(maskArg.Reg) + if !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + firstValue, err := c.loadPackedCompareLanes(first, byteWidth, laneBits, suffixes.broadcast) + if err != nil { + return true, false, err + } + secondValue, err := c.loadPackedCompareLanes(second, byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + lanes := byteWidth * 8 / laneBits + computed := c.emitShuffle128BitBlocks(lanes, laneBits, firstValue, secondValue, uint8(ins.Args[0].Imm)) + if masked { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + computed = amd64ApplyIntegerLaneMask(c, lanes, laneBits, computed, old, mask, suffixes.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, computed, byteWidth) + return true, false, c.storeVectorBytes(destination.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitShuffle128BitBlocks(lanes, laneBits int, firstPlan9, secondPlan9 string, immediate uint8) string { + blockLanes := 128 / laneBits + blocks := lanes / blockLanes + indices := make([]string, 0, lanes) + for outputBlock := 0; outputBlock < blocks; outputBlock++ { + var sourceBlock int + secondOperandOffset := 0 + if blocks == 2 { + sourceBlock = int(immediate>>outputBlock) & 1 + if outputBlock == 1 { + secondOperandOffset = lanes + } + } else { + sourceBlock = int(immediate>>(2*outputBlock)) & 3 + if outputBlock >= 2 { + secondOperandOffset = lanes + } + } + for lane := 0; lane < blockLanes; lane++ { + indices = append(indices, fmt.Sprintf("i32 %d", secondOperandOffset+sourceBlock*blockLanes+lane)) + } + } + shuffled := c.newTmp() + // The second Plan 9 source is Intel SRC1 and therefore LLVM operand 0; + // the first Plan 9 (r/m) source is Intel SRC2 / LLVM operand 1. + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> %s, <%d x i32> <%s>\n", + shuffled, lanes, laneBits, secondPlan9, lanes, laneBits, firstPlan9, lanes, strings.Join(indices, ", ")) + return "%" + shuffled +} diff --git a/amd64_lower_shuffle_blocks_test.go b/amd64_lower_shuffle_blocks_test.go new file mode 100644 index 000000000..0c87a9d4d --- /dev/null +++ b/amd64_lower_shuffle_blocks_test.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86Shuffle128BitBlocksCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 shares _yvshuff32x4 across the floating and integer D/Q + // spellings. It has Y/Z rows, an unsigned-byte immediate, optional K + // masking/.Z, and scalar broadcast from the first Plan 9 memory source. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ := 20 + if target.goarch == "386" { + lastZ = 7 + } + var source strings.Builder + source.WriteString("TEXT shuffle128bitblockforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"VSHUFF32X4", "VSHUFF64X2", "VSHUFI32X4", "VSHUFI64X2"} { + fmt.Fprintf(&source, "\t%s $0, Y1, Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s $2, 8(AX), Y20, Y21\n", op) + fmt.Fprintf(&source, "\t%s $255, Z1, Z2, Z%d\n", op, lastZ) + fmt.Fprintf(&source, "\t%s.BCST $3, 40(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s $4, Y1, Y20, K1, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z $5, Z1, Z20, K2, Z21\n", op) + fmt.Fprintf(&source, "\t%s.BCST.Z $6, 104(AX), Z20, K3, Z21\n", op) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "shuffle128bitblockforms": {Name: "shuffle128bitblockforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "shuffle-128-bit-blocks-"+target.name+".ll", "shuffle-128-bit-blocks-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86Shuffle128BitBlocksRejectsFormsOutsideGoAssemblerTable(t *testing.T) { + for _, instruction := range []string{ + "VSHUFF64X2 $1, X0, X1, X2", + "VSHUFF64X2 $1, Y0, Z1, Z2", + "VSHUFF64X2 $1, Y0, (AX), Y2", + "VSHUFF64X2 $1, Y0, Y1, K0, Y2", + "VSHUFF64X2 $1, Y0, Y1, AX", + "VSHUFF64X2.Z $1, Y0, Y1, Y2", + "VSHUFF64X2.BCST $1, Y0, Y1, Y2", + "VSHUFF64X2.Z.BCST $1, (AX), Y1, K1, Y2", + "VSHUFF64X2 $256, Y0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86Shuffle128BitBlocksRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86Shuffle128BitBlocksRejected(t, "386", "i386-unknown-linux-gnu", "VSHUFF64X2 $1, Y0, Y1, K1, Y2") + assertX86Shuffle128BitBlocksRejected(t, "386", "i386-unknown-linux-gnu", "VSHUFF64X2 $1, Z0, Z1, Z8") +} + +func assertX86Shuffle128BitBlocksRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VSHUFF/VSHUFI forms", instruction) + } +} diff --git a/amd64_lower_shuffle_bytes.go b/amd64_lower_shuffle_bytes.go new file mode 100644 index 000000000..2bf91202c --- /dev/null +++ b/amd64_lower_shuffle_bytes.go @@ -0,0 +1,115 @@ +package plan9asm + +import "fmt" + +// lowerPackedByteShuffle implements all Go 1.27 PSHUFB/VPSHUFB forms. +// VPSHUFB applies the byte shuffle independently to each 128-bit lane; EVEX +// writemasks operate at byte granularity. +func (c *amd64Ctx) lowerPackedByteShuffle(op Op, ins Instr) (ok bool, terminated bool, err error) { + form, known, err := x86ByteShuffleProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy byte shuffle m128 needs original native storage, not virtual FP/SP/BP", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + byteWidth := form.byteWidth + data, err := c.loadPackedCompareBytes(second, byteWidth) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 { + // Keep the original opcode, including m128 alignment and dead faults. + // The portable E4NF reader alone cannot reproduce this legacy effect. + result, err = c.emitPackedBinaryNativeMemory(first, 8, data, "pshufb") + } else if ins.x86RIPLiteral && len(ins.x86RIPLiteralData) == byteWidth { + result = c.emitPackedByteShuffleLiteral(byteWidth, data, ins.x86RIPLiteralData) + } else { + // E4NF: all control bytes are read, even with a dead or zero-masked + // result. Reuse the exact full-tuple reader, not the E4 lane reader. + control, err := c.loadPackedUnpackSource(first, x86PackedUnpackForm{ + x86PackedBinaryForm: form, spec: x86PackedUnpackSpec{laneBits: 8}, + }) + if err != nil { + return true, false, err + } + result = c.emitPackedByteShuffle(byteWidth, data, control) + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, byteWidth, 8, result, old, mask, form.zeroing) + } + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, byteWidth, result) +} + +func (c *amd64Ctx) emitPackedByteShuffleLiteral(byteWidth int, data string, control []byte) string { + indices := make([]int, byteWidth) + for index, selected := range control { + if selected&0x80 != 0 { + indices[index] = byteWidth // Select a zero from the second vector. + } else { + indices[index] = index/16*16 + int(selected&0x0f) + } + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> %s\n", + result, byteWidth, data, byteWidth, byteWidth, llvmI32Mask(indices)) + return "%" + result +} + +func (c *amd64Ctx) emitPackedByteShuffle(byteWidth int, data, control string) string { + if byteWidth == 16 { + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %s, <16 x i8> %s)\n", call, data, control) + return "%" + call + } + + parts := make([]string, 0, byteWidth/16) + for lane := 0; lane < byteWidth/16; lane++ { + dataLane := c.newTmp() + controlLane := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> poison, <16 x i32> %s\n", dataLane, byteWidth, data, byteWidth, llvmI32RangeMask(lane*16, 16)) + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> poison, <16 x i32> %s\n", controlLane, byteWidth, control, byteWidth, llvmI32RangeMask(lane*16, 16)) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %%%s, <16 x i8> %%%s)\n", call, dataLane, controlLane) + parts = append(parts, "%"+call) + } + partWidth := 16 + for len(parts) > 1 { + combined := make([]string, 0, len(parts)/2) + for i := 0; i < len(parts); i += 2 { + joined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> %s, <%d x i32> %s\n", joined, partWidth, parts[i], partWidth, parts[i+1], partWidth*2, llvmI32RangeMask(0, partWidth*2)) + combined = append(combined, "%"+joined) + } + parts = combined + partWidth *= 2 + } + return parts[0] +} diff --git a/amd64_lower_shuffle_bytes_test.go b/amd64_lower_shuffle_bytes_test.go new file mode 100644 index 000000000..5f2763e78 --- /dev/null +++ b/amd64_lower_shuffle_bytes_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedByteShuffleCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 uses ymshufb for PSHUFB and _yvandnpd for VPSHUFB. + // The legacy instruction accepts X/m128, X. The vector instruction has + // VEX X/Y forms and EVEX X/Y/Z forms, including K1-K7 merge/zero masks. + // In 386 mode the Go frontend still exposes EVEX X/Y0-31 and Z0-7, but + // cannot express the four-operand mask rows. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zLast := 21 + vectorData, vectorDestination := 20, 21 + if target.goarch == "386" { + legacyLast = 7 + zLast = 7 + vectorData, vectorDestination = 6, 7 + } + var src strings.Builder + src.WriteString("TEXT packedbyteshuffleforms(SB),NOSPLIT,$0-0\n") + fmt.Fprintf(&src, "\tPSHUFB X0, X%d\n", legacyLast) + fmt.Fprintf(&src, "\tPSHUFB (AX), X%d\n", legacyLast) + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&src, "\tVPSHUFB %s0, %s1, %s2\n", width, width, width) + fmt.Fprintf(&src, "\tVPSHUFB (AX), %s%d, %s%d\n", width, vectorData, width, vectorDestination) + } + fmt.Fprintf(&src, "\tVPSHUFB Z0, Z1, Z%d\n", zLast) + fmt.Fprintf(&src, "\tVPSHUFB (AX), Z6, Z%d\n", zLast) + if target.goarch == "amd64" { + for _, width := range []string{"X", "Y", "Z"} { + fmt.Fprintf(&src, "\tVPSHUFB %s20, %s21, K1, %s22\n", width, width, width) + fmt.Fprintf(&src, "\tVPSHUFB.Z (AX), %s21, K2, %s22\n", width, width) + } + } + src.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, src.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedbyteshuffleforms": {Name: "packedbyteshuffleforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-byte-shuffle-"+target.name+".ll", "packed-byte-shuffle-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedByteShuffleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSHUFB.Z X0, X1", + "PSHUFB X0", + "PSHUFB Y0, Y1", + "PSHUFB X0, (AX)", + "VPSHUFB X0, X1", + "VPSHUFB X0, Y1, Y2", + "VPSHUFB X0, X1, (AX)", + "VPSHUFB.BCST (AX), X1, X2", + "VPSHUFB.SAE X0, X1, X2", + "VPSHUFB.Z X0, X1, X2", + "VPSHUFB X0, X1, K0, X2", + "VPSHUFB X0, X1, K1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShuffleRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + + for _, instruction := range []string{ + "PSHUFB X8, X0", + "PSHUFB X0, X8", + "VPSHUFB Z8, Z0, Z1", + "VPSHUFB Z0, Z1, Z8", + "VPSHUFB X0, X1, K1, X2", + "VPSHUFB.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedByteShuffleRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedByteShuffleRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's ymshufb/_yvandnpd forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_shuffle_float.go b/amd64_lower_shuffle_float.go new file mode 100644 index 000000000..41d1aed7f --- /dev/null +++ b/amd64_lower_shuffle_float.go @@ -0,0 +1,213 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerPackedFloatShuffle implements the complete Go 1.27 SHUFPS/SHUFPD +// and VSHUFPS/VSHUFPD operand tables, using integer lanes to preserve bits. +func (c *amd64Ctx) lowerPackedFloatShuffle(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + suffix := "" + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp, suffix = rawOp[:dot], rawOp[dot+1:] + } + laneBits, vector, recognized := amd64PackedFloatShuffleProperties(baseOp) + if !recognized { + return false, false, nil + } + if !vector { + return c.lowerLegacyPackedFloatShuffle(baseOp, suffix, laneBits, ins) + } + return c.lowerVectorPackedFloatShuffle(baseOp, suffix, laneBits, ins) +} + +func amd64PackedFloatShuffleProperties(op string) (laneBits int, vector, ok bool) { + switch op { + case "SHUFPS": + return 32, false, true + case "SHUFPD": + return 64, false, true + case "VSHUFPS": + return 32, true, true + case "VSHUFPD": + return 64, true, true + default: + return 0, false, false + } +} + +func amd64UnsignedImm8(arg Operand) (uint8, bool) { + if arg.Kind != OpImm || arg.Imm < 0 || arg.Imm > 255 { + return 0, false + } + return uint8(arg.Imm), true +} + +func (c *amd64Ctx) lowerLegacyPackedFloatShuffle(baseOp, suffix string, laneBits int, ins Instr) (bool, bool, error) { + if suffix != "" { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 3 { + return true, false, fmt.Errorf("%s %s expects $u8, X/m128, X: %q", c.goarch, baseOp, ins.Raw) + } + imm, ok := amd64UnsignedImm8(ins.Args[0]) + if !ok { + return true, false, fmt.Errorf("%s %s immediate is outside Go 1.27's unsigned-imm8 class: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[2].Kind != OpReg || !c.isGoLegacyXReg(ins.Args[2].Reg) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's legacy X class: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[1].Kind == OpReg { + if !c.isGoLegacyXReg(ins.Args[1].Reg) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's legacy X class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("%s %s source must be X or memory: %q", c.goarch, baseOp, ins.Raw) + } + firstBytes, err := c.loadPackedCompareBytes(ins.Args[1], 16) + if err != nil { + return true, false, err + } + secondBytes, err := c.loadX(ins.Args[2].Reg) + if err != nil { + return true, false, err + } + lanes := 128 / laneBits + first := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, firstBytes) + second := c.bitcastVectorBytesToIntegerLanes(16, lanes, laneBits, secondBytes) + result := c.emitPackedFloatShuffle(laneBits, 16, imm, first, second) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <16 x i8>\n", out, lanes, laneBits, result) + return true, false, c.storeX(ins.Args[2].Reg, "%"+out) +} + +func (c *amd64Ctx) lowerVectorPackedFloatShuffle(baseOp, suffix string, laneBits int, ins Instr) (bool, bool, error) { + if c.goarch != "amd64" { + return true, false, fmt.Errorf("386 %s exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + broadcast, zeroing := false, false + switch suffix { + case "": + case "Z": + zeroing = true + case "BCST": + broadcast = true + case "BCST.Z": + broadcast, zeroing = true, true + default: + return true, false, fmt.Errorf("amd64 %s has a suffix absent from its Go 1.27 optab: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 4 && len(ins.Args) != 5 { + return true, false, fmt.Errorf("amd64 %s expects $u8, src1, src2, [K mask,] destination: %q", baseOp, ins.Raw) + } + imm, ok := amd64UnsignedImm8(ins.Args[0]) + if !ok { + return true, false, fmt.Errorf("amd64 %s immediate is outside Go 1.27's unsigned-imm8 class: %q", baseOp, ins.Raw) + } + masked := len(ins.Args) == 5 + if zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s zeroing requires a K1-K7 mask: %q", baseOp, ins.Raw) + } + dstArg := ins.Args[len(ins.Args)-1] + if dstArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects a vector-register destination: %q", baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(dstArg.Reg) + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 { + return true, false, fmt.Errorf("amd64 %s expects an X, Y, or Z destination: %q", baseOp, ins.Raw) + } + if !amd64EVEXVectorRegister(dstArg, byteWidth) || !amd64EVEXVectorRegister(ins.Args[2], byteWidth) { + return true, false, fmt.Errorf("amd64 %s second source and destination must be same-width vector registers: %q", baseOp, ins.Raw) + } + if broadcast { + if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory first source: %q", baseOp, ins.Raw) + } + } else if ins.Args[1].Kind == OpReg { + if !amd64EVEXVectorRegister(ins.Args[1], byteWidth) { + return true, false, fmt.Errorf("amd64 %s first source must match its destination width: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s first source must be a vector register or memory: %q", baseOp, ins.Raw) + } + + var mask string + if masked { + if !amd64NonzeroKOperand(ins.Args[3]) { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + var err error + mask, err = c.loadK(ins.Args[3].Reg) + if err != nil { + return true, false, err + } + } + lanes := byteWidth * 8 / laneBits + var first string + var err error + if broadcast { + first, err = c.evalIntSized(ins.Args[1], amd64IntegerTypeForBits(laneBits)) + if err == nil { + first = amd64SplatInteger(c, lanes, laneBits, first) + } + } else { + var firstBytes string + firstBytes, err = c.loadPackedCompareBytes(ins.Args[1], byteWidth) + if err == nil { + first = c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, firstBytes) + } + } + if err != nil { + return true, false, err + } + secondBytes, err := c.loadPackedCompareBytes(ins.Args[2], byteWidth) + if err != nil { + return true, false, err + } + second := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, secondBytes) + result := c.emitPackedFloatShuffle(laneBits, byteWidth, imm, first, second) + if masked { + oldBytes, err := c.loadPackedCompareBytes(dstArg, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, byteWidth) + return true, false, c.storeVectorBytes(dstArg.Reg, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPackedFloatShuffle(laneBits, byteWidth int, imm uint8, first, second string) string { + lanes := byteWidth * 8 / laneBits + groupLanes := 128 / laneBits + vecType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + result := "poison" + for lane := 0; lane < lanes; lane++ { + groupBase := lane / groupLanes * groupLanes + position := lane % groupLanes + source := second + selector := 0 + if laneBits == 32 { + selector = int(imm>>(2*position)) & 3 + if position >= 2 { + source = first + } + } else { + selector = int(imm>>lane) & 1 + if position == 1 { + source = first + } + } + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", extracted, vecType, source, groupBase+selector) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %%%s, i32 %d\n", inserted, vecType, result, laneBits, extracted, lane) + result = "%" + inserted + } + return result +} diff --git a/amd64_lower_shuffle_words.go b/amd64_lower_shuffle_words.go new file mode 100644 index 000000000..c802d534f --- /dev/null +++ b/amd64_lower_shuffle_words.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Preserve the helper entry point while all dword/word/MMX siblings use one +// complete grammar. Dispatch identity cannot grant a contradictory spelling. +func (c *amd64Ctx) lowerPackedWordShuffle(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86ImmediateShuffleSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("immediate-shuffle dispatch/source mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86ImmediateShuffleProperties(ins, c.goarch) + if err != nil || !known { + return known, false, err + } + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + destination := ins.Args[form.destination] + if form.spec.legacy && !form.spec.mmx && form.memory >= 0 { + return true, false, c.lowerLegacyImmediateShuffleMemory(ins, form) + } + var source string + if form.spec.mmx { + source, err = c.loadPackedUnpackMMXSource(form.source, form.memoryBytes) + if err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast i64 %s to <8 x i8>\n", value, source) + source = "%" + value + } + } else if form.broadcast { + bytes, loadErr := c.loadX86PackedShiftBytes(form.source, 4, true) + err = loadErr + if err == nil { + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i8> %s, <4 x i8> zeroinitializer, <%d x i32> %s\n", + value, bytes, form.byteWidth, llvmRepeatI8Mask(4, form.byteWidth)) + source = "%" + value + } + } else { + // E4NF/E4NF.nb does not suppress memory faults, even at K=0 or when + // the immediate selects only one lane. Keep the full read observable. + source, err = c.loadX86PackedShiftBytes(form.source, form.byteWidth, true) + } + if err != nil { + return true, false, err + } + indices := make([]string, form.byteWidth) + for index := range indices { + // A broadcast has already expanded the m32 to full logical lanes. + if form.broadcast { + indices[index] = fmt.Sprintf("i32 %d", index) + } else { + indices[index] = fmt.Sprintf("i32 %d", form.sourceByte(index)) + } + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i8> %s, <%d x i8> zeroinitializer, <%d x i32> <%s>\n", + value, form.byteWidth, source, form.byteWidth, form.byteWidth, strings.Join(indices, ", ")) + result := "%" + value + if form.spec.mmx { + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i8> %s to i64\n", bits, result) + return true, false, c.storeReg(destination.Reg, "%"+bits) + } + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + laneBits := form.spec.laneBytes * 8 + lanes := form.byteWidth / form.spec.laneBytes + computed := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, laneBits, result) + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + } + masked := amd64ApplyIntegerLaneMask(c, lanes, laneBits, computed, old, mask, form.zeroing) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, laneBits, masked, form.byteWidth) + result = "%" + bytes + } + if form.spec.legacy { + return true, false, c.storeLegacyPackedLogical(destination, result) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, result) +} + +func amd64PackedWordShuffleRequiresEVEX(source, destination Operand, byteWidth int, masked bool) bool { + if byteWidth == 64 || masked { + return true + } + for _, operand := range []Operand{source, destination} { + if operand.Kind == OpReg { + index, ok := amd64VectorRegisterIndex(operand.Reg, byteWidth) + if !ok || index >= 16 { + return true + } + } + } + return false +} + +// Legacy m128 must retain the actual single SSE instruction's alignment and +// fault behavior, not an align-1 LLVM read or a fabricated virtual FP buffer. +func (c *amd64Ctx) lowerLegacyImmediateShuffleMemory(ins Instr, form x86ImmediateShuffleForm) error { + source := form.source + if source.Kind == OpFP || ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + source.Kind == OpMem && !x86OrdinaryNativeMemoryOperand(source, c.goarch) { + return fmt.Errorf("%w: legacy immediate shuffle m128 needs original native backing/alignment", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + mnemonic := "pshufd" + if form.spec.laneBytes == 2 { + mnemonic = "pshuflw" + if form.spec.high { + mnemonic = "pshufhw" + } + } + value, bytes := c.newTmp(), c.newTmp() + assembly := fmt.Sprintf("%s $$%d, %s($1), $0", mnemonic, form.immediate, segment) + fmt.Fprintf(c.b, " %%%s = call <4 x float> asm sideeffect %q, %q(%s %s)\n", + value, assembly, "=x,r,~{memory}", pointerType, pointer) + fmt.Fprintf(c.b, " %%%s = bitcast <4 x float> %%%s to <16 x i8>\n", bytes, value) + return c.storeLegacyPackedLogical(ins.Args[form.destination], "%"+bytes) +} diff --git a/amd64_lower_shuffle_words_test.go b/amd64_lower_shuffle_words_test.go new file mode 100644 index 000000000..2bbf7254a --- /dev/null +++ b/amd64_lower_shuffle_words_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PackedWordShuffleCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + zLast := 31 + if target.goarch == "386" { + legacyLast = 7 + zLast = 7 + } + var source strings.Builder + source.WriteString("TEXT packedwordshuffleforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"PSHUFHW", "PSHUFLW"} { + fmt.Fprintf(&source, "\t%s $0, X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s $255, 8(AX), X%d\n", op, legacyLast) + } + for _, op := range []string{"VPSHUFHW", "VPSHUFLW"} { + source.WriteString("\t" + op + " $-128, X0, X1\n") + source.WriteString("\t" + op + " $127, 8(AX), X1\n") + source.WriteString("\t" + op + " $255, Y0, Y1\n") + source.WriteString("\t" + op + " $1, 16(AX), Y2\n") + if target.goarch == "amd64" { + source.WriteString("\t" + op + " $2, X16, X31\n") + } + fmt.Fprintf(&source, "\t%s $3, Z0, Z%d\n", op, zLast) + if target.goarch == "amd64" { + source.WriteString("\t" + op + " $4, X20, K1, X21\n") + source.WriteString("\t" + op + ".Z $5, 24(AX), K2, Y22\n") + source.WriteString("\t" + op + " $6, Z16, K3, Z31\n") + } + } + source.WriteString("\tRET\n") + goAssemblerOracleOutput(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4"), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "packedwordshuffleforms": {Name: "packedwordshuffleforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "packed-word-shuffle-"+target.name+".ll", "packed-word-shuffle-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PackedWordShuffleRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "PSHUFHW $-1, X0, X1", + "PSHUFLW $256, X0, X1", + "PSHUFHW $1, X16, X1", + "PSHUFLW $1, Y0, Y1", + "PSHUFHW.Z $1, X0, X1", + "PSHUFLW $1, X0, (AX)", + "VPSHUFHW $-129, X0, X1", + "VPSHUFLW $256, X0, X1", + "VPSHUFHW $-1, X16, X1", + "VPSHUFLW $-1, Z0, Z1", + "VPSHUFHW $1, X0, Y1", + "VPSHUFLW $1, X0, K0, X1", + "VPSHUFHW.Z $1, X0, X1", + "VPSHUFLW.BCST $1, (AX), X1", + "VPSHUFHW $1, X0, (AX)", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedWordShuffleRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "PSHUFHW $1, X8, X0", + "PSHUFLW $1, X0, X8", + "VPSHUFHW $1, X0, K1, X1", + "VPSHUFLW.Z $1, X0, K1, X1", + "VPSHUFHW $1, Z8, Z0", + } { + t.Run("386_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PackedWordShuffleRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86PackedWordShuffleRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's PSHUFHW/PSHUFLW tables for %s", instruction, goarch) + } +} diff --git a/amd64_lower_sqrt_scalar_test.go b/amd64_lower_sqrt_scalar_test.go new file mode 100644 index 000000000..b8969b1b0 --- /dev/null +++ b/amd64_lower_sqrt_scalar_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateX86ScalarSqrtCompleteGo127Forms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT scalarsqrtforms(SB),$0-0\n") + for _, op := range []string{"VSQRTSS", "VSQRTSD"} { + fmt.Fprintf(&source, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&source, "\t%s 8(BX), X3, X4\n", op) + fmt.Fprintf(&source, "\t%s X29, X30, X31\n", op) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X5, X6, K1, X7\n", op) + fmt.Fprintf(&source, "\t%s.Z 16(BX), X8, K2, X9\n", op) + } + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s X10, X11, X12\n", op, rounding) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.%s.Z X13, X14, K3, X15\n", op, rounding) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "scalarsqrtforms": {Name: "scalarsqrtforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "scalar-sqrt-"+target.name+".ll", "scalar-sqrt-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86ScalarSqrtRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VSQRTSS X0, X1", + "VSQRTSD X0, X1, X2, X3, X4", + "VSQRTSS Y0, X1, X2", + "VSQRTSD X0, Y1, X2", + "VSQRTSS X0, X1, K0, X2", + "VSQRTSD.Z X0, X1, X2", + "VSQRTSS.BCST 8(BX), X1, X2", + "VSQRTSD.SAE X0, X1, X2", + "VSQRTSS.RN_SAE 8(BX), X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + assertX86ScalarSqrtRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86ScalarSqrtRejected(t, "386", "i386-unknown-linux-gnu", "VSQRTSS X0, X1, K1, X2") +} + +func assertX86ScalarSqrtRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VSQRTSS/VSQRTSD _yvaddsd forms for %s", instruction, goarch) + } +} + +func TestAMD64ScalarSqrtRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT scalarsqrt(SB),NOSPLIT,$0-64 + MOVQ out+0(FP), AX + MOVQ source64+8(FP), BX + MOVQ passthrough64+16(FP), CX + MOVQ source32+24(FP), DX + MOVQ passthrough32+32(FP), SI + MOVQ old64+40(FP), DI + MOVQ old32+48(FP), R8 + MOVQ mask+56(FP), R9 + KMOVQ R9, K1 + + VMOVUPD (BX), X0 + VMOVUPD (CX), X1 + VSQRTSD.RD_SAE X0, X1, X2 + VMOVUPD X2, 0(AX) + + VMOVUPS (DX), X0 + VMOVUPS (SI), X1 + VSQRTSS.RU_SAE X0, X1, X2 + VMOVUPS X2, 16(AX) + + VMOVUPD (BX), X0 + VMOVUPD (CX), X1 + VMOVUPD (DI), X4 + VSQRTSD X0, X1, K1, X4 + VMOVUPD X4, 32(AX) + + VMOVUPS (DX), X0 + VMOVUPS (SI), X1 + VMOVUPS (R8), X4 + VSQRTSS.Z X0, X1, K1, X4 + VMOVUPS X4, 48(AX) + VZEROUPPER + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: Ptr, Index: 5, Field: -1}, + {Offset: 48, Type: Ptr, Index: 6, Field: -1}, + {Offset: 56, Type: I64, Index: 7, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "scalarsqrt": {Name: "scalarsqrt", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +extern void scalarsqrt(uint8_t *, const double *, const double *, const float *, const float *, const double *, const float *, uint64_t); +int main(void) { + const double source64[2] = {9.0, 77.0}; + const double passthrough64[2] = {100.0, 101.0}; + const float source32[4] = {16.0f, 17.0f, 18.0f, 19.0f}; + const float passthrough32[4] = {200.0f, 201.0f, 202.0f, 203.0f}; + const double old64[2] = {300.0, 301.0}; + const float old32[4] = {400.0f, 401.0f, 402.0f, 403.0f}; + union { uint8_t bytes[64]; double d64[8]; float f32[16]; } out = {0}; + scalarsqrt(out.bytes, source64, passthrough64, source32, passthrough32, old64, old32, 0); + if (out.d64[0] != 3.0 || out.d64[1] != 101.0) return 1; + if (out.f32[4] != 4.0f || out.f32[5] != 201.0f || out.f32[6] != 202.0f || out.f32[7] != 203.0f) return 2; + if (out.d64[4] != 300.0 || out.d64[5] != 101.0) return 3; + if (out.f32[12] != 0.0f || out.f32[13] != 201.0f || out.f32[14] != 202.0f || out.f32[15] != 203.0f) return 4; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_sqrt", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_stack_width.go b/amd64_lower_stack_width.go new file mode 100644 index 000000000..493a3d582 --- /dev/null +++ b/amd64_lower_stack_width.go @@ -0,0 +1,312 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64StackWidthMode uint8 + +const ( + amd64StackPushValue amd64StackWidthMode = iota + amd64StackPopValue + amd64StackPushFlags + amd64StackPopFlags +) + +type amd64StackWidthSpec struct { + bits int + mode amd64StackWidthMode + segments amd64StackSegmentSet + segmentNativeWidth bool +} + +type amd64StackSegmentSet uint8 + +const ( + amd64StackSegmentES amd64StackSegmentSet = 1 << iota + amd64StackSegmentCS + amd64StackSegmentSS + amd64StackSegmentDS + amd64StackSegmentFS + amd64StackSegmentGS + + amd64StackPushSegments = amd64StackSegmentES | amd64StackSegmentCS | + amd64StackSegmentSS | amd64StackSegmentDS | amd64StackSegmentFS | + amd64StackSegmentGS + amd64StackPopSegments = amd64StackPushSegments &^ amd64StackSegmentCS + amd64StackLongSegments = amd64StackSegmentFS | amd64StackSegmentGS +) + +// amd64StackWidthSpecs models the value/flags, push/pop, width, and segment-set +// axes once for all Go x86 stack forms. Ordinary L spellings are 386-only and +// ordinary Q spellings are amd64-only. Go's separate ymovtab accepts its listed +// segment-register forms in both modes; L/Q use the target's native stack width. +var amd64StackWidthSpecs = map[Op]amd64StackWidthSpec{ + "PUSHW": {bits: 16, mode: amd64StackPushValue, segments: amd64StackPushSegments}, + "POPW": {bits: 16, mode: amd64StackPopValue, segments: amd64StackPopSegments}, + "PUSHL": {bits: 32, mode: amd64StackPushValue, segments: amd64StackPushSegments, segmentNativeWidth: true}, + "POPL": {bits: 32, mode: amd64StackPopValue, segments: amd64StackPopSegments, segmentNativeWidth: true}, + "PUSHQ": {bits: 64, mode: amd64StackPushValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "POPQ": {bits: 64, mode: amd64StackPopValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "PUSHFW": {bits: 16, mode: amd64StackPushFlags}, + "POPFW": {bits: 16, mode: amd64StackPopFlags}, + "PUSHFL": {bits: 32, mode: amd64StackPushFlags}, + "POPFL": {bits: 32, mode: amd64StackPopFlags}, + "PUSHFQ": {bits: 64, mode: amd64StackPushFlags}, + "POPFQ": {bits: 64, mode: amd64StackPopFlags}, +} + +func (c *amd64Ctx) lowerStackWidth(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64StackWidthSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + + switch spec.mode { + case amd64StackPushFlags: + if err := c.validateOrdinaryStackWidthMode(baseOp, spec.bits, ins); err != nil { + return true, false, err + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + value := c.packX86Flags() + if spec.bits != 64 { + value = c.truncI64(value, amd64StackLLVMType(spec.bits)) + } + return true, false, c.pushStackWidth(spec.bits, value) + case amd64StackPopFlags: + if err := c.validateOrdinaryStackWidthMode(baseOp, spec.bits, ins); err != nil { + return true, false, err + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.popStackWidth(spec.bits) + if err != nil { + return true, false, err + } + if spec.bits != 64 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", extended, amd64StackLLVMType(spec.bits), value) + value = "%" + extended + } + c.unpackX86Flags(value, spec.bits) + return true, false, nil + case amd64StackPushValue, amd64StackPopValue: + if len(ins.Args) != 1 { + role := "src" + if spec.mode == amd64StackPopValue { + role = "dst" + } + return true, false, fmt.Errorf("%s %s expects %s: %q", c.goarch, baseOp, role, ins.Raw) + } + default: + panic("unknown x86 stack-width mode") + } + + operand := ins.Args[0] + segmentStack := x86SegmentRegisterOperand(operand) + if segmentStack { + if !spec.segments.contains(operand.Reg) { + return true, false, fmt.Errorf("%s %s does not accept segment register %s in Go 1.27's ymovtab: %q", c.goarch, baseOp, operand.Reg, ins.Raw) + } + bits := spec.bits + if spec.segmentNativeWidth { + bits = c.x86NativeStackBits() + } + if spec.mode == amd64StackPushValue { + selector := c.loadX86SegmentSelector(operand.Reg) + value := selector + switch bits { + case 16: + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i32 %s to i16\n", narrow, selector) + value = "%" + narrow + case 32: + case 64: + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, selector) + value = "%" + wide + default: + panic("invalid x86 segment stack width") + } + return true, false, c.pushStackWidth(bits, value) + } + value, err := c.popStackWidth(bits) + if err != nil { + return true, false, err + } + if bits != 16 { + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to i16\n", narrow, amd64StackLLVMType(bits), value) + value = "%" + narrow + } + return true, false, c.storeX86SegmentSelector(operand.Reg, value) + } + if err := c.validateOrdinaryStackWidthMode(baseOp, spec.bits, ins); err != nil { + return true, false, err + } + if spec.mode == amd64StackPushValue { + if operand.Kind == OpReg { + if !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s source is outside Go 1.27's Yrl class: %q", c.goarch, baseOp, ins.Raw) + } + } else if operand.Kind != OpImm && !isAMD64MemoryOperand(operand) && + !x86AbsoluteRelocationSource(operand, c.goarch, spec.bits) { + return true, false, fmt.Errorf("%s %s expects Yrl/memory/immediate src: %q", c.goarch, baseOp, ins.Raw) + } + if operand.Kind == OpMem && !x86MemoryRegistersValidForArch(operand.Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s memory source uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.evalIntSized(operand, amd64StackLLVMType(spec.bits)) + if err != nil { + return true, false, err + } + return true, false, c.pushStackWidth(spec.bits, value) + } + + if operand.Kind == OpReg { + if !isX86YrlRegisterForArch(operand.Reg, c.goarch) { + return true, false, fmt.Errorf("%s %s destination is outside Go 1.27's Yrl class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(operand) { + return true, false, fmt.Errorf("%s %s expects reg/mem dst in the Yrl class: %q", c.goarch, baseOp, ins.Raw) + } + if operand.Kind == OpMem && !x86MemoryRegistersValidForArch(operand.Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s memory destination uses an out-of-range address register: %q", c.goarch, baseOp, ins.Raw) + } + value, err := c.popStackWidth(spec.bits) + if err != nil { + return true, false, err + } + return true, false, c.storeStackWidth(operand, spec.bits, value) +} + +func (c *amd64Ctx) validateOrdinaryStackWidthMode(op string, bits int, ins Instr) error { + if c.goarch == "386" && bits == 64 { + return fmt.Errorf("386 %s requires GOARCH=amd64: %q", op, ins.Raw) + } + if c.goarch != "386" && bits == 32 { + return fmt.Errorf("%s requires GOARCH=386: %q", op, ins.Raw) + } + return nil +} + +func (c *amd64Ctx) x86NativeStackBits() int { + if c.goarch == "386" { + return 32 + } + return 64 +} + +func (set amd64StackSegmentSet) contains(reg Reg) bool { + var bit amd64StackSegmentSet + switch reg { + case ES: + bit = amd64StackSegmentES + case CS: + bit = amd64StackSegmentCS + case SS: + bit = amd64StackSegmentSS + case DS: + bit = amd64StackSegmentDS + case FS: + bit = amd64StackSegmentFS + case GS: + bit = amd64StackSegmentGS + default: + return false + } + return set&bit != 0 +} + +func amd64StackLLVMType(bits int) LLVMType { + switch bits { + case 16: + return I16 + case 32: + return I32 + case 64: + return I64 + default: + panic("invalid x86 stack width") + } +} + +func (c *amd64Ctx) pushStackWidth(bits int, value string) error { + if c.sourceStack != nil { + return c.pushSourceStackValue(amd64StackLLVMType(bits), int64(bits/8), value) + } + if c.goarch == "386" { + switch bits { + case 16: + return c.pushI16(value) + case 32: + return c.pushI32(value) + } + return fmt.Errorf("386 cannot push %d-bit stack values", bits) + } + if bits != 64 { + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext %s %s to i64\n", extended, amd64StackLLVMType(bits), value) + value = "%" + extended + } + c.pushI64(value) + return nil +} + +func (c *amd64Ctx) popStackWidth(bits int) (string, error) { + if c.sourceStack != nil { + return c.popSourceStackValue(amd64StackLLVMType(bits), int64(bits/8)) + } + if c.goarch == "386" { + switch bits { + case 16: + return c.popI16() + case 32: + return c.popI32() + } + return "", fmt.Errorf("386 cannot pop %d-bit stack values", bits) + } + value := c.popI64() + if bits == 64 { + return value, nil + } + return c.truncI64(value, amd64StackLLVMType(bits)), nil +} + +func (c *amd64Ctx) storeStackWidth(destination Operand, bits int, value string) error { + typ := amd64StackLLVMType(bits) + switch destination.Kind { + case OpReg: + return c.storeRegSized(destination.Reg, typ, value) + case OpMem: + ptr, ptrType, err := c.ptrFromMem(destination.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", typ, value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(destination.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", typ, value, ptr) + return nil + case OpFP: + return c.storeFPResult(destination.FPOffset, typ, value) + default: + return fmt.Errorf("%s stack destination is not writable", destination.String()) + } +} diff --git a/amd64_lower_stack_width_test.go b/amd64_lower_stack_width_test.go new file mode 100644 index 000000000..63202b820 --- /dev/null +++ b/amd64_lower_stack_width_test.go @@ -0,0 +1,258 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "strings" + "testing" +) + +func TestAMD64StackWidthGrammarIsComplete(t *testing.T) { + want := map[Op]amd64StackWidthSpec{ + "PUSHW": {bits: 16, mode: amd64StackPushValue, segments: amd64StackPushSegments}, + "POPW": {bits: 16, mode: amd64StackPopValue, segments: amd64StackPopSegments}, + "PUSHL": {bits: 32, mode: amd64StackPushValue, segments: amd64StackPushSegments, segmentNativeWidth: true}, + "POPL": {bits: 32, mode: amd64StackPopValue, segments: amd64StackPopSegments, segmentNativeWidth: true}, + "PUSHQ": {bits: 64, mode: amd64StackPushValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "POPQ": {bits: 64, mode: amd64StackPopValue, segments: amd64StackLongSegments, segmentNativeWidth: true}, + "PUSHFW": {bits: 16, mode: amd64StackPushFlags}, + "POPFW": {bits: 16, mode: amd64StackPopFlags}, + "PUSHFL": {bits: 32, mode: amd64StackPushFlags}, + "POPFL": {bits: 32, mode: amd64StackPopFlags}, + "PUSHFQ": {bits: 64, mode: amd64StackPushFlags}, + "POPFQ": {bits: 64, mode: amd64StackPopFlags}, + } + if !reflect.DeepEqual(amd64StackWidthSpecs, want) { + t.Fatalf("stack-width grammar = %+v, want %+v", amd64StackWidthSpecs, want) + } +} + +func TestTranslateX86StackWidthCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + high := "R11" + if target.goarch == "386" { + high = "DX" + } + source := "TEXT stackwidthforms(SB),$0-0\n" + + "\tPUSHW AX\n\tPOPW " + high + "\n" + + "\tPUSHW $61731\n\tPOPW (BX)\n" + + "\tPUSHW (BX)\n\tPOPW AX\n" + + "\tPUSHFW\n\tPOPFW\n" + if target.goarch == "amd64" { + source += "\tPUSHQ R11\n\tPOPQ (BX)\n\tPUSHQ FS\n\tPUSHQ GS\n\tPOPQ FS\n\tPOPQ GS\n\tPUSHFQ\n\tPOPFQ\n" + } else { + source += "\tPUSHL DX\n\tPOPL (BX)\n\tPUSHFL\n\tPOPFL\n" + } + source += "\tRET\n" + + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"stackwidthforms": {Name: "stackwidthforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "stack-width-"+target.name+".ll", "stack-width-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86StackSegmentCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + const source = `TEXT stacksegmentforms(SB),$0-0 + PUSHW CS + PUSHW SS + PUSHW DS + PUSHW ES + PUSHW FS + PUSHW GS + POPW SS + POPW DS + POPW ES + POPW FS + POPW GS + PUSHL CS + PUSHL SS + PUSHL DS + PUSHL ES + PUSHL FS + PUSHL GS + POPL SS + POPL DS + POPL ES + POPL FS + POPL GS + PUSHQ FS + PUSHQ GS + POPQ FS + POPQ GS + POPW DS + POPL DS + RET +` + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"stacksegmentforms": {Name: "stacksegmentforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "stack-segment-"+target.name+".ll", "stack-segment-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86StackWidthRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PUSHW X0"}, + {goarch: "amd64", instruction: "POPW $1"}, + {goarch: "amd64", instruction: "PUSHFW AX"}, + {goarch: "amd64", instruction: "POPQ X0"}, + {goarch: "amd64", instruction: "PUSHL AX"}, + {goarch: "amd64", instruction: "PUSHQ DS"}, + {goarch: "amd64", instruction: "POPQ ES"}, + {goarch: "amd64", instruction: "POPW CS"}, + {goarch: "amd64", instruction: "POPL CS"}, + {goarch: "386", instruction: "PUSHQ AX"}, + {goarch: "386", instruction: "POPW R11"}, + {goarch: "386", instruction: "POPFQ"}, + {goarch: "386", instruction: "PUSHQ DS"}, + {goarch: "386", instruction: "POPQ ES"}, + {goarch: "386", instruction: "POPW CS"}, + {goarch: "386", instruction: "POPL CS"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's stack tables", test.instruction) + } + }) + } +} + +func TestAMD64StackWidthRuntimeValuesAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT stackwidthsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVW $0x1234, AX + PUSHW AX + POPW CX + MOVW CX, 0(DI) + MOVW $0x5678, 2(DI) + PUSHW 2(DI) + POPW DX + MOVW DX, 4(DI) + PUSHW $61731 + POPW 6(DI) + MOVQ $0x123456789abcdef, AX + PUSHQ AX + POPQ 8(DI) + STC + PUSHFW + CLC + POPFW + SETCS 16(DI) + PUSHFQ + CLC + POPFQ + SETCS 17(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"stackwidthsemantics": { + Name: "stackwidthsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void stackwidthsemantics(uint8_t *); +static uint16_t load16(const uint8_t *p) { uint16_t v; memcpy(&v,p,2); return v; } +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v,p,8); return v; } +int main(void) { + uint8_t out[24] = {0}; stackwidthsemantics(out); + if (load16(out) != 0x1234 || load16(out+4) != 0x5678 || load16(out+6) != 61731) return 10; + if (load64(out+8) != UINT64_C(0x123456789abcdef)) return 11; + if (out[16] != 1 || out[17] != 1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "stack_width_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_string.go b/amd64_lower_string.go index 5ae46f59d..08e76b31b 100644 --- a/amd64_lower_string.go +++ b/amd64_lower_string.go @@ -1,50 +1,82 @@ package plan9asm -import "fmt" +import ( + "fmt" +) + +type x86StringSpec struct { + kind string + typ LLVMType + width int +} + +var x86StringSpecs = map[Op]x86StringSpec{ + "MOVSB": {kind: "movs", typ: I8, width: 1}, + "MOVSW": {kind: "movs", typ: I16, width: 2}, + "MOVSL": {kind: "movs", typ: I32, width: 4}, + "MOVSQ": {kind: "movs", typ: I64, width: 8}, + "STOSB": {kind: "stos", typ: I8, width: 1}, + "STOSW": {kind: "stos", typ: I16, width: 2}, + "STOSL": {kind: "stos", typ: I32, width: 4}, + "STOSQ": {kind: "stos", typ: I64, width: 8}, + "SCASB": {kind: "scas", typ: I8, width: 1}, + "SCASW": {kind: "scas", typ: I16, width: 2}, + "SCASL": {kind: "scas", typ: I32, width: 4}, + "SCASQ": {kind: "scas", typ: I64, width: 8}, + "CMPSB": {kind: "cmps", typ: I8, width: 1}, + "CMPSW": {kind: "cmps", typ: I16, width: 2}, + "CMPSL": {kind: "cmps", typ: I32, width: 4}, + "CMPSQ": {kind: "cmps", typ: I64, width: 8}, + "LODSB": {kind: "lods", typ: I8, width: 1}, + "LODSW": {kind: "lods", typ: I16, width: 2}, + "LODSL": {kind: "lods", typ: I32, width: 4}, + "LODSQ": {kind: "lods", typ: I64, width: 8}, +} + +func x86StringProperties(op Op) (kind string, typ LLVMType, width int, ok bool) { + spec, ok := x86StringSpecs[Op(normalizeInstructionOpcode(op))] + if !ok { + return "", "", 0, false + } + return spec.kind, spec.typ, spec.width, true +} func (c *amd64Ctx) lowerString(op Op, ins Instr) (ok bool, terminated bool, err error) { - if c.goarch != "386" { + kind, typ, width, ok := x86StringProperties(op) + if !ok { return false, false, nil } - switch op { - case "MOVSB", "MOVSL", "STOSL", "SCASB": - default: - return false, false, nil + if op != Op(normalizeInstructionOpcode(op)) { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, op, ins.Raw) } - prefix := c.repeatPrefix c.repeatPrefix = "" - if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + if c.goarch == "386" && typ == I64 { + return true, false, fmt.Errorf("386 %s is not in the Go assembler's 32-bit instruction table", op) } - repeated := prefix != "" - if prefix == "REPN" && op != "SCASB" { - return true, false, fmt.Errorf("386 REPN is unsupported for %s", op) - } - if prefix == "REP" && op == "SCASB" { - return true, false, fmt.Errorf("386 REP SCASB is not yet supported") + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, op, ins.Raw) } count := "1" - if repeated { - cx32, err := c.evalIntSized(Operand{Kind: OpReg, Reg: CX}, I32) + if prefix != "" { + cx, err := c.loadReg(CX) if err != nil { return true, false, err } - wide := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, cx32) - count = "%" + wide + if c.goarch == "386" { + cx32 := c.truncI64(cx, I32) + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, cx32) + count = "%" + wide + } else { + count = cx + } } direction := c.loadDirectionFlag() - switch op { - case "MOVSB", "MOVSL": - width := 1 - helper := "__plan9asm_movsb" - if op == "MOVSL" { - width = 4 - helper = "__plan9asm_movsl" - } + switch kind { + case "movs": si, err := c.loadReg(SI) if err != nil { return true, false, err @@ -53,7 +85,7 @@ func (c *amd64Ctx) lowerString(op Op, ins Instr) (ok bool, terminated bool, err if err != nil { return true, false, err } - fmt.Fprintf(c.b, " call void @%s(i64 %s, i64 %s, i64 %s, i1 %s)\n", helper, di, si, count, direction) + fmt.Fprintf(c.b, " call void @%s(i64 %s, i64 %s, i64 %s, i1 %s)\n", x86StringHelperName(kind, width), di, si, count, direction) bytes := c.newTmp() fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", bytes, count, width) if err := c.storeStringIndex(SI, si, "%"+bytes, direction); err != nil { @@ -63,67 +95,174 @@ func (c *amd64Ctx) lowerString(op Op, ins Instr) (ok bool, terminated bool, err return true, false, err } - case "STOSL": + case "stos": di, err := c.loadReg(DI) if err != nil { return true, false, err } - ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) if err != nil { return true, false, err } - fmt.Fprintf(c.b, " call void @__plan9asm_rep_stosl(i64 %s, i32 %s, i64 %s, i1 %s)\n", di, ax, count, direction) + fmt.Fprintf(c.b, " call void @%s(i64 %s, %s %s, i64 %s, i1 %s)\n", x86StringHelperName(kind, width), di, typ, value, count, direction) bytes := c.newTmp() - fmt.Fprintf(c.b, " %%%s = mul i64 %s, 4\n", bytes, count) + fmt.Fprintf(c.b, " %%%s = mul i64 %s, %d\n", bytes, count, width) if err := c.storeStringIndex(DI, di, "%"+bytes, direction); err != nil { return true, false, err } - case "SCASB": + case "scas": di, err := c.loadReg(DI) if err != nil { return true, false, err } - needle, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AL}, I8) + needle, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) if err != nil { return true, false, err } + whileEqual := prefix == "REP" + resultType := fmt.Sprintf("{ i64, i64, %s, i1 }", typ) call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call { i64, i64, i1 } @__plan9asm_repne_scasb(i64 %s, i8 %s, i64 %s, i1 %s)\n", call, di, needle, count, direction) + fmt.Fprintf(c.b, " %%%s = call %s @%s(i64 %s, %s %s, i64 %s, i1 %s, i1 %t)\n", call, resultType, x86StringHelperName(kind, width), di, typ, needle, count, direction, whileEqual) next := c.newTmp() remaining := c.newTmp() - equal := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i64, i1 } %%%s, 0\n", next, call) - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i64, i1 } %%%s, 1\n", remaining, call) - fmt.Fprintf(c.b, " %%%s = extractvalue { i64, i64, i1 } %%%s, 2\n", equal, call) + lastValue := c.newTmp() + executed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", next, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", remaining, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 2\n", lastValue, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 3\n", executed, resultType, call) if err := c.storeReg(DI, "%"+next); err != nil { return true, false, err } - zf := "%" + equal - if repeated { - oldZF := c.loadFlag(c.flagsZSlot) - executed := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ne i64 %s, 0\n", executed, count) - preserved := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i1 %%%s, i1 %s\n", preserved, executed, equal, oldZF) - zf = "%" + preserved + // SCAS sets all six status flags from accumulator minus the last + // element read, not just the equality flag used to control repetition. + c.setStringCompareFlags(typ, needle, "%"+lastValue, "%"+executed, prefix != "") + if prefix != "" { + if err := c.storeStringCount("%" + remaining); err != nil { + return true, false, err + } + } + + case "cmps": + si, err := c.loadReg(SI) + if err != nil { + return true, false, err + } + di, err := c.loadReg(DI) + if err != nil { + return true, false, err + } + whileEqual := prefix == "REP" + resultType := fmt.Sprintf("{ i64, i64, i64, %s, %s, i1 }", typ, typ) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(i64 %s, i64 %s, i64 %s, i1 %s, i1 %t)\n", call, resultType, x86StringHelperName(kind, width), si, di, count, direction, whileEqual) + nextSI := c.newTmp() + nextDI := c.newTmp() + remaining := c.newTmp() + left := c.newTmp() + right := c.newTmp() + executed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", nextSI, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", nextDI, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 2\n", remaining, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 3\n", left, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 4\n", right, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 5\n", executed, resultType, call) + if err := c.storeReg(SI, "%"+nextSI); err != nil { + return true, false, err + } + if err := c.storeReg(DI, "%"+nextDI); err != nil { + return true, false, err + } + if prefix != "" { + if err := c.storeStringCount("%" + remaining); err != nil { + return true, false, err + } } - fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", zf, c.flagsZSlot) - if repeated { - if err := c.storeRegSized(CX, I32, c.truncI64("%"+remaining, I32)); err != nil { + c.setStringCompareFlags(typ, "%"+left, "%"+right, "%"+executed, prefix != "") + + case "lods": + si, err := c.loadReg(SI) + if err != nil { + return true, false, err + } + resultType := fmt.Sprintf("{ i64, i64, %s, i1 }", typ) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s @%s(i64 %s, i64 %s, i1 %s)\n", call, resultType, x86StringHelperName(kind, width), si, count, direction) + nextSI := c.newTmp() + remaining := c.newTmp() + loaded := c.newTmp() + executed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 0\n", nextSI, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 1\n", remaining, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 2\n", loaded, resultType, call) + fmt.Fprintf(c.b, " %%%s = extractvalue %s %%%s, 3\n", executed, resultType, call) + if err := c.storeReg(SI, "%"+nextSI); err != nil { + return true, false, err + } + value := "%" + loaded + if prefix != "" { + old, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, typ) + if err != nil { + return true, false, err + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, %s %%%s, %s %s\n", selected, executed, typ, loaded, typ, old) + value = "%" + selected + if err := c.storeStringCount("%" + remaining); err != nil { return true, false, err } } + if err := c.storeRegSized(AX, typ, value); err != nil { + return true, false, err + } } - if repeated && op != "SCASB" { - if err := c.storeRegSized(CX, I32, "0"); err != nil { + if prefix != "" && kind != "scas" && kind != "cmps" { + if err := c.storeStringCount("0"); err != nil { return true, false, err } } return true, false, nil } +func x86StringHelperName(kind string, width int) string { + suffix := map[int]string{1: "b", 2: "w", 4: "l", 8: "q"}[width] + if kind == "movs" { + return "__plan9asm_movs" + suffix + } + return "__plan9asm_rep_" + kind + suffix +} + +func (c *amd64Ctx) setStringCompareFlags(typ LLVMType, left, right, executed string, preserveEmpty bool) { + flags := c.x86StatusFlagSlots() + old := make([]string, len(flags)) + if preserveEmpty { + for index, flag := range flags { + old[index] = c.loadFlag(flag.slot) + } + } + c.setCmpFlagsSized(typ, left, right) + if preserveEmpty { + // REP/REPN with a zero initial count performs no comparison and + // preserves all six status flags. The helper also performs no load. + for index, flag := range flags { + current := c.loadFlag(flag.slot) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i1 %s, i1 %s\n", selected, executed, current, old[index]) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", selected, flag.slot) + } + } +} + +func (c *amd64Ctx) storeStringCount(value string) error { + if c.goarch == "386" { + return c.storeRegSized(CX, I32, c.truncI64(value, I32)) + } + return c.storeRegSized(CX, I64, value) +} + func (c *amd64Ctx) loadDirectionFlag() string { t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load i1, ptr %s\n", t, c.directionSlot) diff --git a/amd64_lower_string_compare.go b/amd64_lower_string_compare.go new file mode 100644 index 000000000..6aab05547 --- /dev/null +++ b/amd64_lower_string_compare.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedStringCompareSpec struct { + explicitLength bool + maskResult bool + vector bool +} + +// amd64PackedStringCompareSpecs models Go 1.27's complete SSE4.2 packed-string +// grammar as three independent axes: explicit/implicit lengths, index/mask +// result, and legacy/VEX encoding. Every spelling shares the same imm8 and +// X/m128 operand grammar and the same six architectural flag results. +var amd64PackedStringCompareSpecs = map[Op]amd64PackedStringCompareSpec{ + "PCMPESTRI": {explicitLength: true}, + "PCMPESTRM": {explicitLength: true, maskResult: true}, + "PCMPISTRI": {}, + "PCMPISTRM": {maskResult: true}, + "VPCMPESTRI": {explicitLength: true, vector: true}, + "VPCMPESTRM": {explicitLength: true, maskResult: true, vector: true}, + "VPCMPISTRI": {vector: true}, + "VPCMPISTRM": {maskResult: true, vector: true}, +} + +func (c *amd64Ctx) lowerPackedStringCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + spec, recognized, err := amd64PackedStringCompareProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + source, destination := ins.Args[1], ins.Args[2] + left, err := c.loadX(destination.Reg) + if err != nil { + return true, false, err + } + right, err := c.loadPackedCompareBytes(source, 16) + if err != nil { + return true, false, err + } + lengthLeft, lengthRight := "", "" + if spec.explicitLength { + lengthLeft, err = c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + if err != nil { + return true, false, err + } + lengthRight, err = c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I32) + if err != nil { + return true, false, err + } + } + imm := uint8(ins.Args[0].Imm) + stem := "pcmpi" + if spec.explicitLength { + stem = "pcmpe" + } + + output, width, zeroUpper := spec.resultEffect() + if spec.maskResult { + result := c.emitPackedStringCompareIntrinsic(stem+"strm128", "<16 x i8>", left, right, lengthLeft, lengthRight, imm) + if zeroUpper { + err = c.storePackedMoveOperand(Operand{Kind: OpReg, Reg: output}, width, result) + } else { + // The widest shared bank keeps legacy writes limited to low X0. + err = c.storeX(output, result) + } + if err != nil { + return true, false, err + } + } else { + result := c.emitPackedStringCompareIntrinsic(stem+"stri128", "i32", left, right, lengthLeft, lengthRight, imm) + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", extended, result) + if err := c.storeReg(output, "%"+extended); err != nil { + return true, false, err + } + } + + carry := c.emitPackedStringCompareFlag(stem+"stric128", left, right, lengthLeft, lengthRight, imm) + overflow := c.emitPackedStringCompareFlag(stem+"strio128", left, right, lengthLeft, lengthRight, imm) + sign := c.emitPackedStringCompareFlag(stem+"stris128", left, right, lengthLeft, lengthRight, imm) + zero := c.emitPackedStringCompareFlag(stem+"striz128", left, right, lengthLeft, lengthRight, imm) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", carry, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", zero, c.flagsZSlot) + // This slot stores architectural SF. Signed conditions apply SF XOR OF + // when observing it; precomputing that condition would corrupt PUSHF/LAHF. + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", sign, c.flagsSltSlot) + c.clearX86StatusFlags(x86StatusPF | x86StatusAF) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", overflow, c.flagsOFSlot) + return true, false, nil +} + +// This descriptor is syntax and architectural effects, not caller definedness +// or address permission. Unregistered source-stack consumers remain Context. +func (spec amd64PackedStringCompareSpec) resultEffect() (Reg, int, bool) { + if spec.maskResult { + return "X0", 16, spec.vector + } + return CX, 4, false +} + +func amd64PackedStringCompareProperties(op Op, ins Instr, goarch string) (amd64PackedStringCompareSpec, bool, error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64PackedStringCompareSpecs[Op(baseOp)] + if !recognized { + return spec, false, nil + } + if rawOp != baseOp || goarch != "amd64" && goarch != "386" { + return spec, true, fmt.Errorf("%s %s has no suffixed or non-x86 form in Go: %q", goarch, baseOp, ins.Raw) + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != rawOp { + return spec, true, fmt.Errorf("packed-string dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + minimumImmediate := int64(0) + if spec.vector { + // _yvaeskeygenassist has both Yu8 and Yi8 rows, while the legacy + // yxshuf table has only Yu8. + minimumImmediate = -128 + } + if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[0].ImmIsFloat || ins.Args[0].ImmRaw != "" || + ins.Args[0].Imm < minimumImmediate || ins.Args[0].Imm > 255 { + return spec, true, fmt.Errorf("%s %s immediate is outside its Go byte domain, or operands are not imm8, X/m128, X: %q", goarch, baseOp, ins.Raw) + } + + source, destination := ins.Args[1], ins.Args[2] + if destination.Kind != OpReg || !amd64PackedStringCompareRegister(spec, destination, goarch) { + return spec, true, fmt.Errorf("%s %s destination is outside its Go X register class: %q", goarch, baseOp, ins.Raw) + } + if source.Kind == OpReg { + if !amd64PackedStringCompareRegister(spec, source, goarch) { + return spec, true, fmt.Errorf("%s %s source is outside its Go X register class: %q", goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return spec, true, fmt.Errorf("%s %s source must be X or m128: %q", goarch, baseOp, ins.Raw) + } + if !spec.vector && source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, goarch) { + return spec, true, fmt.Errorf("%s %s memory source uses a register outside the legacy address class: %q", goarch, baseOp, ins.Raw) + } + ins.Op = op + return spec, true, x86Mode32VectorRegisterContext(ins, goarch) +} + +func amd64PackedStringCompareRegister(spec amd64PackedStringCompareSpec, operand Operand, goarch string) bool { + if spec.vector { + return amd64VEXVectorRegister(operand, 16) + } + index, valid := amd64ParseXReg(operand.Reg) + limit := 16 + if goarch == "386" { + limit = 8 + } + return operand.Kind == OpReg && valid && index < limit +} + +func (c *amd64Ctx) emitPackedStringCompareIntrinsic(name, resultType, left, right, lengthLeft, lengthRight string, imm uint8) string { + result := c.newTmp() + if lengthLeft == "" { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.x86.sse42.%s(<16 x i8> %s, <16 x i8> %s, i8 %d)\n", + result, resultType, name, left, right, imm) + } else { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.x86.sse42.%s(<16 x i8> %s, i32 %s, <16 x i8> %s, i32 %s, i8 %d)\n", + result, resultType, name, left, lengthLeft, right, lengthRight, imm) + } + return "%" + result +} + +func (c *amd64Ctx) emitPackedStringCompareFlag(name, left, right, lengthLeft, lengthRight string, imm uint8) string { + result := c.emitPackedStringCompareIntrinsic(name, "i32", left, right, lengthLeft, lengthRight, imm) + flag := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i32 %s to i1\n", flag, result) + return "%" + flag +} diff --git a/amd64_lower_string_compare_test.go b/amd64_lower_string_compare_test.go new file mode 100644 index 000000000..583c69481 --- /dev/null +++ b/amd64_lower_string_compare_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +type packedStringCompareTestSpec struct { + op string + explicit bool + mask bool + vector bool +} + +func go127PackedStringCompareSpecs() []packedStringCompareTestSpec { + return []packedStringCompareTestSpec{ + {op: "PCMPESTRI", explicit: true}, + {op: "PCMPESTRM", explicit: true, mask: true}, + {op: "PCMPISTRI"}, + {op: "PCMPISTRM", mask: true}, + {op: "VPCMPESTRI", explicit: true, vector: true}, + {op: "VPCMPESTRM", explicit: true, mask: true, vector: true}, + {op: "VPCMPISTRI", vector: true}, + {op: "VPCMPISTRM", mask: true, vector: true}, + } +} + +func TestAMD64PackedStringCompareGrammarCoversCompleteGoFamily(t *testing.T) { + expected := go127PackedStringCompareSpecs() + if len(amd64PackedStringCompareSpecs) != len(expected) { + t.Fatalf("packed-string grammar has %d entries, want %d", len(amd64PackedStringCompareSpecs), len(expected)) + } + for _, want := range expected { + got, ok := amd64PackedStringCompareSpecs[Op(want.op)] + if !ok { + t.Errorf("packed-string grammar omitted %s", want.op) + continue + } + if got.explicitLength != want.explicit || got.maskResult != want.mask || got.vector != want.vector { + t.Errorf("packed-string grammar %s = %+v, want explicit=%v mask=%v vector=%v", want.op, got, want.explicit, want.mask, want.vector) + } + } +} + +func TestTranslateX86PackedStringCompareCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT packedstringcompareforms(SB),$0-0\n") + for index, spec := range go127PackedStringCompareSpecs() { + last := 15 + if target.goarch == "386" { + last = 7 + } + fmt.Fprintf(&source, "\t%s $0, X1, X%d\n", spec.op, last) + fmt.Fprintf(&source, "\t%s $255, %d(AX), X%d\n", spec.op, index*16, last) + if spec.vector { + fmt.Fprintf(&source, "\t%s $-128, X1, X%d\n", spec.op, last) + } + } + source.WriteString("\tRET\n") + + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"packedstringcompareforms": {Name: "packedstringcompareforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, intrinsic := range []string{ + "@llvm.x86.sse42.pcmpestri128", "@llvm.x86.sse42.pcmpestrm128", + "@llvm.x86.sse42.pcmpistri128", "@llvm.x86.sse42.pcmpistrm128", + "@llvm.x86.sse42.pcmpestric128", "@llvm.x86.sse42.pcmpistriz128", + } { + if !strings.Contains(ir, intrinsic) { + t.Fatalf("translation omitted %s:\n%s", intrinsic, ir) + } + } + wantFeatures := `"target-features"="+avx,+sse4.2"` + if !strings.Contains(ir, wantFeatures) { + t.Fatalf("translation omitted %s:\n%s", wantFeatures, ir) + } + compileLLVMToObject(t, llc, target.triple, "packed-string-compare-"+target.name+".ll", "packed-string-compare-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86PackedStringCompareRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PCMPESTRI $-1, X0, X1"}, + {goarch: "amd64", instruction: "PCMPISTRM $256, X0, X1"}, + {goarch: "amd64", instruction: "VPCMPESTRI.Z $7, X0, X1"}, + {goarch: "amd64", instruction: "VPCMPESTRI $-129, X0, X1"}, + {goarch: "amd64", instruction: "PCMPISTRI $7, Y0, X1"}, + {goarch: "amd64", instruction: "VPCMPISTRM $7, X0, X16"}, + {goarch: "386", instruction: "PCMPESTRM $7, X0, X8"}, + {goarch: "386", instruction: "VPCMPESTRM $7, X0, X16"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", ",", "", "$", "", ".", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's packed-string tables", test.instruction) + } + }) + } +} + +func TestAMD64PackedStringCompareRuntimeSemanticsAndFlags(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT packedstringcomparesemantics(SB),NOSPLIT,$0-40 + MOVQ left+0(FP), SI + MOVOU (SI), X1 + MOVQ right+8(FP), DI + MOVOU (DI), X2 + MOVQ out+16(FP), BX + MOVQ lenLeft+24(FP), AX + MOVQ lenRight+32(FP), DX + + PCMPESTRI $12, X2, X1 + MOVQ CX, 0(BX) + SETCS 8(BX) + SETEQ 9(BX) + SETLT 10(BX) + SETOS 11(BX) + SETPS 12(BX) + + PCMPESTRM $64, X2, X1 + MOVOU X0, 16(BX) + SETCS 32(BX) + SETEQ 33(BX) + SETLT 34(BX) + SETOS 35(BX) + SETPS 36(BX) + + VPCMPISTRI $0, X2, X1 + MOVQ CX, 40(BX) + SETCS 48(BX) + SETEQ 49(BX) + SETLT 50(BX) + SETOS 51(BX) + SETPS 52(BX) + + VPCMPISTRM $64, X2, X1 + MOVOU X0, 56(BX) + SETCS 72(BX) + SETEQ 73(BX) + SETLT 74(BX) + SETOS 75(BX) + SETPS 76(BX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"packedstringcomparesemantics": { + Name: "packedstringcomparesemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern void packedstringcomparesemantics(const uint8_t *, const uint8_t *, uint8_t *, uint64_t, uint64_t); +static uint64_t load64(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v; } +static int flags_equal(const uint8_t *p, int c, int z, int s, int o) { + return p[0] == c && p[1] == z && p[2] == (s ^ o) && p[3] == o && p[4] == 0; +} +__attribute__((target("sse4.2"))) static int check(void) { + const uint8_t left[16] = {'a','b','c','d','e',0,'x','y','z',0,1,2,3,4,5,6}; + const uint8_t right[16] = {'q','q','a','b','c','d','e',0,'x',0,9,8,7,6,5,4}; + const int la = 5, lb = 9; + uint8_t out[80] = {0}, want[16]; + __m128i a = _mm_loadu_si128((const __m128i *)left); + __m128i b = _mm_loadu_si128((const __m128i *)right); + packedstringcomparesemantics(left, right, out, la, lb); + if (load64(out) != (uint64_t)_mm_cmpestri(a, la, b, lb, 12)) return 10; + if (!flags_equal(out + 8, _mm_cmpestrc(a,la,b,lb,12), _mm_cmpestrz(a,la,b,lb,12), _mm_cmpestrs(a,la,b,lb,12), _mm_cmpestro(a,la,b,lb,12))) return 11; + _mm_storeu_si128((__m128i *)want, _mm_cmpestrm(a,la,b,lb,64)); + if (memcmp(out + 16, want, 16)) return 12; + if (!flags_equal(out + 32, _mm_cmpestrc(a,la,b,lb,64), _mm_cmpestrz(a,la,b,lb,64), _mm_cmpestrs(a,la,b,lb,64), _mm_cmpestro(a,la,b,lb,64))) return 13; + if (load64(out + 40) != (uint64_t)_mm_cmpistri(a,b,0)) return 14; + if (!flags_equal(out + 48, _mm_cmpistrc(a,b,0), _mm_cmpistrz(a,b,0), _mm_cmpistrs(a,b,0), _mm_cmpistro(a,b,0))) return 15; + _mm_storeu_si128((__m128i *)want, _mm_cmpistrm(a,b,64)); + if (memcmp(out + 56, want, 16)) return 16; + if (!flags_equal(out + 72, _mm_cmpistrc(a,b,64), _mm_cmpistrz(a,b,64), _mm_cmpistrs(a,b,64), _mm_cmpistro(a,b,64))) return 17; + return 0; +} +int main(void) { return check(); } +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_string_compare_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_subvec.go b/amd64_lower_subvec.go new file mode 100644 index 000000000..4320bfe2c --- /dev/null +++ b/amd64_lower_subvec.go @@ -0,0 +1,181 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedSubtractMode uint8 + +const ( + amd64PackedSubtractWrap amd64PackedSubtractMode = iota + amd64PackedSubtractSignedSaturating + amd64PackedSubtractUnsignedSaturating +) + +// lowerPackedIntegerSubtract implements the complete Go 1.27 packed-integer +// subtract family: legacy yxm XMM forms and the shared _yvandnpd VEX/EVEX +// forms. Go's 386 frontend accepts low-bank X/Y/Z three-operand forms but +// rejects explicit source masks; high banks retain their physical Context. +func (c *amd64Ctx) lowerPackedIntegerSubtract(op Op, ins Instr) (ok bool, terminated bool, err error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, _, _, known := amd64PackedSubtractProperties(base); !known { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed subtract dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, known, err := x86PackedSubtractProperties(ins, c.goarch) + if err != nil || !known { + return known, false, err + } + first := ins.Args[form.first] + if form.legacy && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed subtract needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.laneBits + lanes := form.byteWidth * 8 / bits + second, err := c.loadPackedCompareLanes(ins.Args[form.second], form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + var mask, value string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.legacy && form.memory >= 0 { + mnemonic := strings.ToLower(base) + if base == "PSUBL" { + mnemonic = "psubd" + } + value, err = c.emitPackedBinaryNativeMemory(first, bits, second, mnemonic) + } else { + // Go's EVEX subtract rows are E4, not E4NF. Preserve every active + // source access even for dead outputs, and suppress inactive lanes. + input, loadErr := c.loadX86PackedShiftLanes(first, form.byteWidth, bits, form.broadcast, mask) + err = loadErr + if err == nil { + value = c.emitPackedIntegerSubtract(lanes, bits, input, second, form.mode) + } + } + if err != nil { + return true, false, err + } + destination := ins.Args[form.destination] + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + } + value = amd64ApplyIntegerLaneMask(c, lanes, bits, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, bits, value, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func amd64PackedSubtractProperties(op string) (laneBits int, mode amd64PackedSubtractMode, vector, ok bool) { + switch op { + case "PSUBB": + return 8, amd64PackedSubtractWrap, false, true + case "PSUBW": + return 16, amd64PackedSubtractWrap, false, true + case "PSUBL": + return 32, amd64PackedSubtractWrap, false, true + case "PSUBQ": + return 64, amd64PackedSubtractWrap, false, true + case "PSUBSB": + return 8, amd64PackedSubtractSignedSaturating, false, true + case "PSUBSW": + return 16, amd64PackedSubtractSignedSaturating, false, true + case "PSUBUSB": + return 8, amd64PackedSubtractUnsignedSaturating, false, true + case "PSUBUSW": + return 16, amd64PackedSubtractUnsignedSaturating, false, true + case "VPSUBB": + return 8, amd64PackedSubtractWrap, true, true + case "VPSUBW": + return 16, amd64PackedSubtractWrap, true, true + case "VPSUBD": + return 32, amd64PackedSubtractWrap, true, true + case "VPSUBQ": + return 64, amd64PackedSubtractWrap, true, true + case "VPSUBSB": + return 8, amd64PackedSubtractSignedSaturating, true, true + case "VPSUBSW": + return 16, amd64PackedSubtractSignedSaturating, true, true + case "VPSUBUSB": + return 8, amd64PackedSubtractUnsignedSaturating, true, true + case "VPSUBUSW": + return 16, amd64PackedSubtractUnsignedSaturating, true, true + default: + return 0, 0, false, false + } +} + +// emitPackedIntegerSubtract computes second-first, matching Go assembler's +// rm,v,destination operand order. +func (c *amd64Ctx) emitPackedIntegerSubtract(lanes, laneBits int, first, second string, mode amd64PackedSubtractMode) string { + vecType := fmt.Sprintf("<%d x i%d>", lanes, laneBits) + if mode == amd64PackedSubtractWrap { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", result, vecType, second, first) + return "%" + result + } + + wideBits := laneBits * 2 + wideType := fmt.Sprintf("<%d x i%d>", lanes, wideBits) + extend := "sext" + if mode == amd64PackedSubtractUnsignedSaturating { + extend = "zext" + } + firstWide := c.newTmp() + secondWide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", firstWide, extend, vecType, first, wideType) + fmt.Fprintf(c.b, " %%%s = %s %s %s to %s\n", secondWide, extend, vecType, second, wideType) + difference := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub %s %%%s, %%%s\n", difference, wideType, secondWide, firstWide) + + clamped := "%" + difference + minValue := int64(0) + if mode == amd64PackedSubtractSignedSaturating { + minValue = -int64(uint64(1) << (laneBits - 1)) + } + below := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt %s %s, %s\n", below, wideType, clamped, llvmSplatSignedInteger(lanes, wideBits, minValue)) + boundedBelow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", boundedBelow, lanes, below, wideType, llvmSplatSignedInteger(lanes, wideBits, minValue), wideType, clamped) + clamped = "%" + boundedBelow + if mode == amd64PackedSubtractSignedSaturating { + maxValue := int64((uint64(1) << (laneBits - 1)) - 1) + above := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp sgt %s %s, %s\n", above, wideType, clamped, llvmSplatSignedInteger(lanes, wideBits, maxValue)) + boundedAbove := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", boundedAbove, lanes, above, wideType, llvmSplatSignedInteger(lanes, wideBits, maxValue), wideType, clamped) + clamped = "%" + boundedAbove + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc %s %s to %s\n", result, wideType, clamped, vecType) + return "%" + result +} diff --git a/amd64_lower_svm.go b/amd64_lower_svm.go new file mode 100644 index 000000000..365790d55 --- /dev/null +++ b/amd64_lower_svm.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// lowerX86SystemManagement implements AMD's fixed no-operand 0F 01 D8-DF +// family and Intel's VMCALL. Their architectural operands are implicit: +// VMRUN/VMLOAD/VMSAVE use rAX, SKINIT uses EAX, INVLPGA uses rAX and ECX, +// and VMCALL/VMMCALL follow the common +// KVM hypercall convention with rAX as the result and rBX/rCX/rDX/rSI as input +// arguments. The remaining instructions have no general-register operands. +func (c *amd64Ctx) lowerX86SystemManagement(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + switch baseOp { + case "VMCALL", "VMRUN", "VMMCALL", "VMLOAD", "VMSAVE", "STGI", "CLGI", "SKINIT", "INVLPGA": + // handled below + default: + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + + wordType := I64 + if c.goarch == "386" { + wordType = I32 + } + loadWordReg := func(reg Reg) (string, error) { + return c.evalIntSized(Operand{Kind: OpReg, Reg: reg}, wordType) + } + clobbers := "~{dirflag},~{fpsr},~{flags},~{memory}" + switch baseOp { + case "STGI", "CLGI": + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", strings.ToLower(baseOp), clobbers) + case "VMRUN", "VMLOAD", "VMSAVE": + ax, err := loadWordReg(AX) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s %s)\n", strings.ToLower(baseOp), "{ax},"+clobbers, wordType, ax) + case "SKINIT": + ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect \"skinit\", %q(i32 %s)\n", "{ax},"+clobbers, ax) + case "INVLPGA": + ax, err := loadWordReg(AX) + if err != nil { + return true, false, err + } + cx, err := c.evalIntSized(Operand{Kind: OpReg, Reg: CX}, I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect \"invlpga\", %q(%s %s, i32 %s)\n", "{ax},{cx},"+clobbers, wordType, ax, cx) + case "VMCALL", "VMMCALL": + regs := []Reg{AX, BX, CX, DX, SI} + args := make([]string, len(regs)) + for i, reg := range regs { + value, err := loadWordReg(reg) + if err != nil { + return true, false, err + } + args[i] = value + } + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s, %s %s, %s %s, %s %s, %s %s)\n", + call, wordType, strings.ToLower(baseOp), "={ax},0,{bx},{cx},{dx},{si},"+clobbers, + wordType, args[0], wordType, args[1], wordType, args[2], wordType, args[3], wordType, args[4], + ) + if err := c.storeRegSized(AX, wordType, "%"+call); err != nil { + return true, false, err + } + } + return true, false, nil +} diff --git a/amd64_lower_syscall.go b/amd64_lower_syscall.go index 8fa1ce4ad..be2d768f4 100644 --- a/amd64_lower_syscall.go +++ b/amd64_lower_syscall.go @@ -7,6 +7,21 @@ import ( func (c *amd64Ctx) lowerSyscall(op Op, ins Instr) (ok bool, terminated bool, err error) { switch op { + case "WAIT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("x86 WAIT expects no operands: %q", ins.Raw) + } + c.b.WriteString(" call void asm sideeffect \"wait\", \"~{memory}\"()\n") + return true, false, nil + case "HLT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("x86 HLT expects no operands: %q", ins.Raw) + } + c.b.WriteString(" call void asm sideeffect \"hlt\", \"~{memory}\"()\n") + c.b.WriteString(" unreachable\n") + return true, true, nil + case "UD2": + return c.lowerX86UndefinedTrap(op, ins) case "INT": if c.goarch == "386" && (len(ins.Args) != 1 || ins.Args[0].Kind != OpImm) { return true, false, fmt.Errorf("386 INT expects an immediate vector: %q", ins.Raw) diff --git a/amd64_lower_system_address.go b/amd64_lower_system_address.go new file mode 100644 index 000000000..c5ed43fab --- /dev/null +++ b/amd64_lower_system_address.go @@ -0,0 +1,134 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SystemAddressKind uint8 + +const ( + amd64SystemAddressInvalidatePage amd64SystemAddressKind = iota + amd64SystemAddressInvalidatePCID + amd64SystemAddressDemoteCacheLine +) + +type amd64SystemAddressSpec struct { + kind amd64SystemAddressKind + memoryBytes int + register bool +} + +// amd64SystemAddressSpecs is the complete Go 1.27 address-shaped system +// instruction grammar. The single-memory and memory-plus-register rows share +// memory/address validation; register compatibility closures are represented +// explicitly and trap just as their architecturally invalid ModRM encodings do. +var amd64SystemAddressSpecs = map[Op]amd64SystemAddressSpec{ + "INVLPG": {kind: amd64SystemAddressInvalidatePage, memoryBytes: 1}, + "INVPCID": {kind: amd64SystemAddressInvalidatePCID, memoryBytes: 16, register: true}, + "CLDEMOTE": {kind: amd64SystemAddressDemoteCacheLine, memoryBytes: 1}, +} + +type amd64SystemAddressForm struct { + memory Operand + register Reg + invalidModRM bool +} + +func (c *amd64Ctx) lowerSystemAddress(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64SystemAddressSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + form, err := c.parseSystemAddressForm(baseOp, spec, ins) + if err != nil { + return true, false, err + } + return c.emitSystemAddress(baseOp, spec, form) +} + +func (c *amd64Ctx) parseSystemAddressForm(op string, spec amd64SystemAddressSpec, ins Instr) (amd64SystemAddressForm, error) { + var form amd64SystemAddressForm + wantArgs := 1 + if spec.register { + wantArgs = 2 + } + if len(ins.Args) != wantArgs { + return form, fmt.Errorf("%s %s expects %d operand(s): %q", c.goarch, op, wantArgs, ins.Raw) + } + address := ins.Args[0] + if isAMD64MemoryOperand(address) { + if address.Kind == OpMem && !x86MemoryRegistersValidForArch(address.Mem, c.goarch) { + return form, fmt.Errorf("%s %s address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + form.memory = address + } else if address.Kind == OpReg && spec.kind != amd64SystemAddressDemoteCacheLine { + valid := isX86YrlRegisterForArch(address.Reg, c.goarch) + if spec.kind == amd64SystemAddressInvalidatePage { + valid = isGoYmbRegisterForArch(address.Reg, c.goarch) + } + if !valid { + return form, fmt.Errorf("%s %s first operand is outside Go 1.27's compatibility class: %q", c.goarch, op, ins.Raw) + } + form.invalidModRM = true + } else { + return form, fmt.Errorf("%s %s expects Go 1.27 memory/address form: %q", c.goarch, op, ins.Raw) + } + if spec.register { + typeOperand := ins.Args[1] + if typeOperand.Kind != OpReg || !isX86YrlRegisterForArch(typeOperand.Reg, c.goarch) { + return form, fmt.Errorf("%s %s type operand must be a Go 1.27 Yrl register: %q", c.goarch, op, ins.Raw) + } + form.register = typeOperand.Reg + } + return form, nil +} + +func (c *amd64Ctx) emitSystemAddress(op string, spec amd64SystemAddressSpec, form amd64SystemAddressForm) (bool, bool, error) { + if form.invalidModRM { + c.b.WriteString(" call void asm sideeffect \"ud2\", \"~{memory}\"()\n") + c.b.WriteString(" unreachable\n") + return true, true, nil + } + pointer, pointerType, err := c.x86DescriptorMemoryPointer(form.memory) + if err != nil { + return true, false, err + } + clobbers := "~{memory},~{dirflag},~{fpsr},~{flags}" + if spec.kind != amd64SystemAddressInvalidatePCID { + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([%d x i8]) %s)\n", + strings.ToLower(op)+" $0", "*m,"+clobbers, pointerType, spec.memoryBytes, pointer) + return true, false, nil + } + + if c.goarch == "386" { + // INVPCID is a 64-bit-mode instruction, but Go 1.27 accepts it for + // 386. Reproduce Go's 66 0F 38 82 /r encoding with the calculated + // address fixed in EBX and the type fixed in EAX. In 32-bit mode the + // bytes fault architecturally, but remain object-level compatible. + address := c.newTmp() + fmt.Fprintf(c.b, " %%%s = ptrtoint %s %s to i32\n", address, pointerType, pointer) + typeValue, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.register}, I32) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(i32 %%%s, i32 %s)\n", + ".byte 0x66, 0x0f, 0x38, 0x82, 0x03", "{bx},{ax},"+clobbers, address, typeValue) + return true, false, nil + } + typeValue, err := c.evalIntSized(Operand{Kind: OpReg, Reg: form.register}, I64) + if err != nil { + return true, false, err + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s elementtype([16 x i8]) %s, i64 %s)\n", + "invpcid $0, $1", "*m,r,"+clobbers, pointerType, pointer, typeValue) + return true, false, nil +} diff --git a/amd64_lower_system_address_test.go b/amd64_lower_system_address_test.go new file mode 100644 index 000000000..c18ffd917 --- /dev/null +++ b/amd64_lower_system_address_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64SystemAddressGrammarIsComplete(t *testing.T) { + want := map[Op]amd64SystemAddressSpec{ + "INVLPG": {kind: amd64SystemAddressInvalidatePage, memoryBytes: 1}, + "INVPCID": {kind: amd64SystemAddressInvalidatePCID, memoryBytes: 16, register: true}, + "CLDEMOTE": {kind: amd64SystemAddressDemoteCacheLine, memoryBytes: 1}, + } + if !reflect.DeepEqual(amd64SystemAddressSpecs, want) { + t.Fatalf("system-address grammar = %+v, want %+v", amd64SystemAddressSpecs, want) + } +} + +func TestTranslateX86SystemAddressCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + base := "BX" + if target.goarch == "amd64" { + base = "R11" + } + source := "TEXT systemaddressforms(SB),$0-0\n" + + "\tINVLPG 8(" + base + ")\n" + + "\tCLDEMOTE 16(" + base + ")\n" + + "\tINVPCID 32(" + base + "), AX\n" + + "\tRET\n" + + "TEXT invlpgcompat(SB),$0-0\n\tINVLPG AX\n" + + "TEXT invpcidcompat(SB),$0-0\n\tINVPCID AX, BX\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "systemaddressforms": {Name: "systemaddressforms", Ret: Void}, + "invlpgcompat": {Name: "invlpgcompat", Ret: Void}, + "invpcidcompat": {Name: "invpcidcompat", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"target-features"="+cldemote,+invpcid"`, `asm sideeffect "ud2"`, "unreachable"} { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if target.goarch == "386" && !strings.Contains(ir, ".byte 0x66, 0x0f, 0x38, 0x82, 0x03") { + t.Errorf("386 IR is missing Go-compatible INVPCID encoding:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "system-address-"+target.name+".ll", "system-address-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SystemAddressRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "INVLPG X1"}, + {goarch: "amd64", instruction: "INVLPG.Z (BX)"}, + {goarch: "amd64", instruction: "CLDEMOTE AX"}, + {goarch: "amd64", instruction: "CLDEMOTE (BX), (CX)"}, + {goarch: "amd64", instruction: "INVPCID (BX), X1"}, + {goarch: "amd64", instruction: "INVPCID X1, BX"}, + {goarch: "amd64", instruction: "INVPCID (BX)"}, + {goarch: "386", instruction: "INVLPG (R11)"}, + {goarch: "386", instruction: "CLDEMOTE (R11)"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's system-address tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_system_control_test.go b/amd64_lower_system_control_test.go new file mode 100644 index 000000000..e6d4c560e --- /dev/null +++ b/amd64_lower_system_control_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86SystemControlLRETRawAndWAIT(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, test := range []struct { + name string + source string + want string + }{ + { + name: "lret", + source: "TEXT lret(SB),NOSPLIT,$0-0\n\tBYTE $0x48; BYTE $0xcb\n", + want: `asm sideeffect ".byte 0x48, 0xcb"`, + }, + { + name: "wait", + source: "TEXT wait(SB),NOSPLIT,$0-0\n\tWAIT\n\tRET\n", + want: `asm sideeffect "wait"`, + }, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, test.source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{test.name: {Name: test.name, Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, test.want) { + t.Fatalf("translation omitted %q:\n%s", test.want, ir) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "x86-system-control-"+test.name+".ll", "x86-system-control-"+test.name+".o", ir) + }) + } +} diff --git a/amd64_lower_system_transfer.go b/amd64_lower_system_transfer.go new file mode 100644 index 000000000..149b571ef --- /dev/null +++ b/amd64_lower_system_transfer.go @@ -0,0 +1,117 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64SystemTransferKind uint8 + +const ( + amd64SystemEnter amd64SystemTransferKind = iota + amd64SystemExit + amd64SystemReturn +) + +type amd64SystemTransferInputs uint8 + +const ( + amd64SystemTransferCX amd64SystemTransferInputs = 1 << iota + amd64SystemTransferDX + amd64SystemTransferR11 +) + +type amd64SystemTransferSpec struct { + kind amd64SystemTransferKind + mode64 bool + inputs amd64SystemTransferInputs +} + +// amd64SystemTransferSpecs is the complete Go 1.27 SYSENTER/SYSEXIT/SYSRET +// grammar. SYSCALL is intentionally separate because plan9asm provides a +// hosted libc-backed syscall implementation for it. These raw architectural +// transfers retain exact Go bytes and explicitly materialize the GP registers +// consumed by their exit/return conventions. +var amd64SystemTransferSpecs = map[Op]amd64SystemTransferSpec{ + "SYSENTER": {kind: amd64SystemEnter}, + "SYSENTER64": {kind: amd64SystemEnter, mode64: true}, + "SYSEXIT": {kind: amd64SystemExit, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSEXIT64": {kind: amd64SystemExit, mode64: true, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSRET": {kind: amd64SystemReturn, inputs: amd64SystemTransferCX | amd64SystemTransferR11}, +} + +func (spec amd64SystemTransferSpec) terminates() bool { + return spec.kind != amd64SystemEnter +} + +func (spec amd64SystemTransferSpec) encoding() string { + var opcode byte + switch spec.kind { + case amd64SystemEnter: + opcode = 0x34 + case amd64SystemExit: + opcode = 0x35 + case amd64SystemReturn: + opcode = 0x07 + default: + panic("unknown amd64 system-transfer kind") + } + if spec.mode64 { + return fmt.Sprintf(".byte 0x48, 0x0f, 0x%02x", opcode) + } + return fmt.Sprintf(".byte 0x0f, 0x%02x", opcode) +} + +func (c *amd64Ctx) lowerSystemTransfer(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64SystemTransferSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.mode64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s %s takes no operands: %q", c.goarch, baseOp, ins.Raw) + } + constraints := make([]string, 0, 7) + arguments := make([]string, 0, 3) + for _, input := range []struct { + bit amd64SystemTransferInputs + reg Reg + constraint string + }{ + {bit: amd64SystemTransferCX, reg: CX, constraint: "{cx}"}, + {bit: amd64SystemTransferDX, reg: DX, constraint: "{dx}"}, + {bit: amd64SystemTransferR11, reg: Reg("R11"), constraint: "{r11}"}, + } { + if spec.inputs&input.bit == 0 || (c.goarch == "386" && input.bit == amd64SystemTransferR11) { + continue + } + typ := I32 + if spec.mode64 || spec.kind == amd64SystemReturn { + typ = I64 + } + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: input.reg}, typ) + if err != nil { + return true, false, err + } + constraints = append(constraints, input.constraint) + arguments = append(arguments, fmt.Sprintf("%s %s", typ, value)) + } + constraints = append(constraints, "~{memory}", "~{dirflag}", "~{fpsr}", "~{flags}") + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", + spec.encoding(), strings.Join(constraints, ","), strings.Join(arguments, ", ")) + if !spec.terminates() { + return true, false, nil + } + c.b.WriteString(" unreachable\n") + return true, true, nil +} diff --git a/amd64_lower_system_transfer_test.go b/amd64_lower_system_transfer_test.go new file mode 100644 index 000000000..58823da96 --- /dev/null +++ b/amd64_lower_system_transfer_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestAMD64SystemTransferGrammarIsComplete(t *testing.T) { + want := map[Op]amd64SystemTransferSpec{ + "SYSENTER": {kind: amd64SystemEnter}, + "SYSENTER64": {kind: amd64SystemEnter, mode64: true}, + "SYSEXIT": {kind: amd64SystemExit, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSEXIT64": {kind: amd64SystemExit, mode64: true, inputs: amd64SystemTransferCX | amd64SystemTransferDX}, + "SYSRET": {kind: amd64SystemReturn, inputs: amd64SystemTransferCX | amd64SystemTransferR11}, + } + if !reflect.DeepEqual(amd64SystemTransferSpecs, want) { + t.Fatalf("system-transfer grammar = %+v, want %+v", amd64SystemTransferSpecs, want) + } +} + +func TestTranslateX86SystemTransferCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT sysenterform(SB),$0-0 + SYSENTER + RET +TEXT sysexitform(SB),$0-0 + SYSEXIT +TEXT sysretform(SB),$0-0 + SYSRET +` + sigs := map[string]FuncSig{ + "sysenterform": {Name: "sysenterform", Ret: Void}, + "sysexitform": {Name: "sysexitform", Ret: Void}, + "sysretform": {Name: "sysretform", Ret: Void}, + } + if target.goarch == "amd64" { + source += `TEXT sysenter64form(SB),$0-0 + SYSENTER64 + RET +TEXT sysexit64form(SB),$0-0 + SYSEXIT64 +` + sigs["sysenter64form"] = FuncSig{Name: "sysenter64form", Ret: Void} + sigs["sysexit64form"] = FuncSig{Name: "sysexit64form", Ret: Void} + } + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `.byte 0x0f, 0x34`, + `.byte 0x0f, 0x35`, + `.byte 0x0f, 0x07`, + `{cx},{dx},~{memory}`, + } { + if !strings.Contains(ir, want) { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + wantUnreachable := 2 + if target.goarch == "amd64" { + wantUnreachable = 3 + for _, want := range []string{ + `.byte 0x48, 0x0f, 0x34`, + `.byte 0x48, 0x0f, 0x35`, + `{cx},{r11},~{memory}`, + } { + if !strings.Contains(ir, want) { + t.Errorf("amd64 IR is missing %s:\n%s", want, ir) + } + } + } + if got := strings.Count(ir, "unreachable"); got != wantUnreachable { + t.Errorf("unreachable count = %d, want %d", got, wantUnreachable) + } + compileLLVMToObject(t, llc, target.triple, "system-transfer-"+target.name+".ll", "system-transfer-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86SystemTransferRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "SYSENTER AX"}, + {goarch: "amd64", instruction: "SYSENTER64.P"}, + {goarch: "amd64", instruction: "SYSEXIT AX"}, + {goarch: "amd64", instruction: "SYSEXIT64 AX"}, + {goarch: "amd64", instruction: "SYSRET AX"}, + {goarch: "386", instruction: "SYSENTER64"}, + {goarch: "386", instruction: "SYSEXIT64"}, + {goarch: "386", instruction: "SYSRET.P"}, + } { + t.Run(test.goarch+"_"+strings.ReplaceAll(test.instruction, " ", "_"), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's system-transfer tables", test.instruction) + } + }) + } +} diff --git a/amd64_lower_ternary_logic.go b/amd64_lower_ternary_logic.go new file mode 100644 index 000000000..53df324d9 --- /dev/null +++ b/amd64_lower_ternary_logic.go @@ -0,0 +1,114 @@ +package plan9asm + +import "fmt" + +// Both Go _yvalignd opcodes have the same forms; D/Q changes mask lanes and +// the scalar broadcast tuple, not the bitwise Boolean truth table. +var amd64TernaryLogicSpecs = map[Op]int{"VPTERNLOGD": 32, "VPTERNLOGQ": 64} + +func (c *amd64Ctx) lowerTernaryLogic(op Op, ins Instr) (bool, bool, error) { + form, recognized, err := x86TernaryLogicProperties(op, ins, c.goarch) + if !recognized || err != nil { + return recognized, false, err + } + if form.memory >= 0 && ins.Args[form.memory].Kind == OpFP { + if err := c.validateX86PackedShiftFPSpan(ins.Args[form.memory].FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + var mask string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + dependencies := form.dependencies() + if form.mask >= 0 && !form.zeroing { + dependencies |= 1 + } + inputs := [3]string{"zeroinitializer", "zeroinitializer", "zeroinitializer"} + for role, argument := range form.inputs { + if argument == form.memory { + // E4 reads are observable even if the truth table ignores C. + // Only active mask lanes suppress faults; never LLVM DCE. + // Intel SDM Vol.2A 2.8.4, Tables 2-43/2-49 (E4, not E4NF): + // https://cdrdv2-public.intel.com/789573/252046-sdm-change-document.pdf + lanes, err := c.loadX86PackedShiftLanes(ins.Args[argument], form.byteWidth, form.laneBits, form.broadcast, mask) + if err != nil { + return true, false, err + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, form.byteWidth*8/form.laneBits, form.laneBits, lanes, form.byteWidth) + inputs[role] = "%" + bytes + } else if dependencies>>uint(role)&1 != 0 { + inputs[role], err = c.loadPackedCompareBytes(ins.Args[argument], form.byteWidth) + if err != nil { + return true, false, err + } + } + } + result := c.emitTernaryLogicTerms(form.byteWidth, form.terms, inputs) + if form.mask >= 0 { + lanes := form.byteWidth * 8 / form.laneBits + computed := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.laneBits, result) + old := c.bitcastVectorBytesToIntegerLanes(form.byteWidth, lanes, form.laneBits, inputs[0]) + masked := amd64ApplyIntegerLaneMask(c, lanes, form.laneBits, computed, old, mask, form.zeroing) + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", bytes, lanes, form.laneBits, masked, form.byteWidth) + result = "%" + bytes + } + return true, false, c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, result) +} + +// Normalization removes irrelevant inputs and aliases before LLVM sees undef. +func (c *amd64Ctx) emitTernaryLogicBytes(width int, immediate uint8, a, b, third string) string { + return c.emitTernaryLogicTerms(width, x86TernaryLogicTerms(immediate, [3]int{0, 1, 2}), [3]string{a, b, third}) +} + +func (c *amd64Ctx) emitTernaryLogicTerms(width int, terms uint8, inputs [3]string) string { + result := "" + for subset := 0; subset < 8; subset++ { + if terms>>uint(subset)&1 == 0 { + continue + } + term := "" + for role, input := range inputs { + if subset>>uint(role)&1 == 0 { + continue + } + if term == "" { + term = input + continue + } + name := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %s, %s\n", name, width, term, input) + term = "%" + name + } + if term == "" { + term = llvmAllOnesI8Vec(width) + } + if result == "" { + result = term + continue + } + name := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor <%d x i8> %s, %s\n", name, width, result, term) + result = "%" + name + } + if result == "" { + return "zeroinitializer" + } + return result +} + +func amd64TernaryLogicBit(immediate uint8, a, b, third bool) bool { + index := 0 + for _, value := range []bool{a, b, third} { + index <<= 1 + if value { + index++ + } + } + return immediate>>uint(index)&1 != 0 +} diff --git a/amd64_lower_unary_imm_float.go b/amd64_lower_unary_imm_float.go new file mode 100644 index 000000000..6edc76ce1 --- /dev/null +++ b/amd64_lower_unary_imm_float.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "fmt" +) + +type amd64UnaryImmediateFloatingShape struct { + laneBits int + scalar bool + twoSource bool +} + +type amd64UnaryImmediateFloatingForm struct { + properties amd64BinaryFloatingSuffix + source Operand + passthrough Operand + destination Operand + byteWidth int + mask string + immediate uint8 +} + +// parseUnaryImmediateFloatingForm models Go 1.27's related unary-immediate +// and FIXUPIMM operand grammars. Semantic families supply only their lane +// width, packed/scalar shape, and whether packed forms have two sources; +// width, memory, BCST, SAE, mask, zeroing, and 386 frontend constraints are +// validated once here. +func (c *amd64Ctx) parseUnaryImmediateFloatingForm(baseOp, suffix string, shape amd64UnaryImmediateFloatingShape, ins Instr) (amd64UnaryImmediateFloatingForm, error) { + var form amd64UnaryImmediateFloatingForm + properties, validSuffix := parseAMD64BinaryFloatingSuffix(suffix) + if !validSuffix || properties.rounding != "" { + return form, fmt.Errorf("%s %s has a suffix absent from Go 1.27's unary-immediate floating optabs: %q", c.goarch, baseOp, ins.Raw) + } + if shape.scalar && properties.broadcast { + return form, fmt.Errorf("%s %s scalar forms do not support broadcast: %q", c.goarch, baseOp, ins.Raw) + } + hasPassthrough := shape.scalar || shape.twoSource + wantArgs := 3 + if hasPassthrough { + wantArgs = 4 + } + if len(ins.Args) != wantArgs && len(ins.Args) != wantArgs+1 { + return form, fmt.Errorf("%s %s has the wrong operand count for its packed/scalar grammar: %q", c.goarch, baseOp, ins.Raw) + } + if ins.Args[0].Kind != OpImm || ins.Args[0].Imm < 0 || ins.Args[0].Imm > 255 { + return form, fmt.Errorf("%s %s immediate must be an unsigned byte: %q", c.goarch, baseOp, ins.Raw) + } + masked := len(ins.Args) == wantArgs+1 + if c.goarch == "386" && (hasPassthrough || masked) { + return form, fmt.Errorf("386 %s form exceeds the Go assembler frontend's three-operand limit: %q", baseOp, ins.Raw) + } + if properties.zeroing && !masked { + return form, fmt.Errorf("%s %s zeroing requires a K1-K7 mask: %q", c.goarch, baseOp, ins.Raw) + } + maskIndex := len(ins.Args) - 2 + if masked && !amd64NonzeroKOperand(ins.Args[maskIndex]) { + return form, fmt.Errorf("%s %s masked form expects K1-K7: %q", c.goarch, baseOp, ins.Raw) + } + destination := ins.Args[len(ins.Args)-1] + if destination.Kind != OpReg { + return form, fmt.Errorf("%s %s destination must be a vector register: %q", c.goarch, baseOp, ins.Raw) + } + byteWidth := amd64VectorByteWidth(destination.Reg) + passthrough := Operand{} + if shape.scalar { + passthrough = ins.Args[2] + if byteWidth != 16 || !c.isGoEVEXVectorRegister(destination, 16) || !c.isGoEVEXVectorRegister(passthrough, 16) { + return form, fmt.Errorf("%s %s scalar pass-through and destination must be X registers: %q", c.goarch, baseOp, ins.Raw) + } + } else { + if byteWidth != 16 && byteWidth != 32 && byteWidth != 64 || !c.isGoEVEXVectorRegister(destination, byteWidth) { + return form, fmt.Errorf("%s %s packed destination must be an in-range X, Y, or Z register: %q", c.goarch, baseOp, ins.Raw) + } + if shape.twoSource { + passthrough = ins.Args[2] + if !c.isGoEVEXVectorRegister(passthrough, byteWidth) { + return form, fmt.Errorf("%s %s second packed source must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } + } + source := ins.Args[1] + if properties.broadcast { + if !isAMD64MemoryOperand(source) { + return form, fmt.Errorf("%s %s.BCST requires a scalar memory source: %q", c.goarch, baseOp, ins.Raw) + } + } else if source.Kind == OpReg { + if !c.isGoEVEXVectorRegister(source, byteWidth) { + return form, fmt.Errorf("%s %s source register must match the destination width: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return form, fmt.Errorf("%s %s source must be a matching vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if properties.sae && (source.Kind != OpReg || !shape.scalar && byteWidth != 64) { + return form, fmt.Errorf("%s %s.SAE requires a register source and scalar or Z form: %q", c.goarch, baseOp, ins.Raw) + } + mask := "" + if masked { + var err error + mask, err = c.loadK(ins.Args[maskIndex].Reg) + if err != nil { + return form, err + } + } + return amd64UnaryImmediateFloatingForm{ + properties: properties, source: source, passthrough: passthrough, + destination: destination, byteWidth: byteWidth, mask: mask, + immediate: uint8(ins.Args[0].Imm), + }, nil +} diff --git a/amd64_lower_unpack.go b/amd64_lower_unpack.go new file mode 100644 index 000000000..aecd7569e --- /dev/null +++ b/amd64_lower_unpack.go @@ -0,0 +1,160 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// One complete grammar is consumed by lowering and caller byte proofs. +// Interleaving is bitwise, including floating spellings: it never changes MXCSR. +func (c *amd64Ctx) lowerPackedUnpack(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, recognized := x86PackedUnpackSpecs[Op(base)]; !recognized { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed unpack dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + form, recognized, err := x86PackedUnpackProperties(ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + if err := x86LegacyMMXReadEmitterContext(ins, c.goarch); err != nil { + return true, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + laneBits := form.spec.laneBits + lanes := form.byteWidth * 8 / laneBits + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + if form.mmx { + firstBits, err := c.loadPackedUnpackMMXSource(first, form.memoryBytes) + if err != nil { + return true, false, err + } + secondBits, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + a := c.bitcastI64ToIntegerLanes(lanes, laneBits, firstBits) + b := c.bitcastI64ToIntegerLanes(lanes, laneBits, secondBits) + result := c.emitPackedUnpackLanes(form.byteWidth, laneBits, form.spec.high, a, b) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", bits, lanes, laneBits, result) + return true, false, c.storeReg(destination.Reg, "%"+bits) + } + // Intel SDM Vol.2A Tables 2-44/45 classify these EVEX permutations as + // E4NF, not E4. A mask (even all zero) is not memory-fault suppression. + // Do not reuse the logical family's masked-memory reader here. + a, err := c.loadPackedUnpackSource(first, form) + if err != nil { + return true, false, err + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + result := c.emitPackedUnpackLanes(form.byteWidth, laneBits, form.spec.high, a, b) + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old, err := c.loadPackedCompareLanes(destination, form.byteWidth, laneBits, false) + if err != nil { + return true, false, err + } + result = amd64ApplyIntegerLaneMask(c, lanes, laneBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, laneBits, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) loadPackedUnpackSource(source Operand, form x86PackedUnpackForm) (string, error) { + if source.Kind == OpReg || source.Kind == OpFP { + return c.loadPackedCompareLanes(source, form.byteWidth, form.spec.laneBits, form.broadcast) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + value := c.newTmp() + bits := form.spec.laneBits + lanes := form.byteWidth * 8 / bits + // An E4NF access is mandatory even when every output lane is masked off + // or observes only part of the source. A plain LLVM load can disappear. + if form.broadcast { + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", value, bits, pointerType, pointer) + return amd64SplatInteger(c, lanes, bits, "%"+value), nil + } + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i%d>, %s %s, align 1\n", value, lanes, bits, pointerType, pointer) + return "%" + value, nil +} + +func (c *amd64Ctx) loadPackedUnpackMMXSource(source Operand, memoryBytes int) (string, error) { + if source.Kind == OpFP { + bytes, err := c.loadFPVectorBytes(source.FPOffset, memoryBytes) + if err != nil { + return "", err + } + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to i%d\n", bits, memoryBytes, bytes, 8*memoryBytes) + if memoryBytes == 8 { + return "%" + bits, nil + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, bits) + return "%" + wide, nil + } + if memoryBytes == 0 { + return c.loadLegacyMMXPackedSource(source) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", value, 8*memoryBytes, pointerType, pointer) + if memoryBytes == 8 { + return "%" + value, nil + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", wide, value) + return "%" + wide, nil +} + +// emitPackedUnpackLanes interleaves the second (vvvv/old destination) source +// with the first (r/m) source independently within each 128-bit vector lane. +func (c *amd64Ctx) emitPackedUnpackLanes(byteWidth, laneBits int, high bool, first, second string) string { + lanes := byteWidth * 8 / laneBits + groupBits := 128 + if byteWidth == 8 { + groupBits = 64 + } + groupLanes := groupBits / laneBits + half := groupLanes / 2 + start := 0 + if high { + start = half + } + mask := make([]string, 0, lanes) + for group := 0; group < lanes/groupLanes; group++ { + base := group * groupLanes + for lane := start; lane < start+half; lane++ { + index := base + lane + mask = append(mask, fmt.Sprintf("i32 %d", index), fmt.Sprintf("i32 %d", lanes+index)) + } + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %s, <%d x i%d> %s, <%d x i32> <%s>\n", + result, lanes, laneBits, second, lanes, laneBits, first, lanes, strings.Join(mask, ", ")) + return "%" + result +} diff --git a/amd64_lower_unpack_float_test.go b/amd64_lower_unpack_float_test.go new file mode 100644 index 000000000..2f05ed4c8 --- /dev/null +++ b/amd64_lower_unpack_float_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86FloatingUnpackCompleteGoAssemblerForms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast, zLast, second, destination := 15, 22, 20, 21 + if target.goarch == "386" { + legacyLast, zLast = 7, 7 + second, destination = 6, 7 + } + var source strings.Builder + source.WriteString("TEXT floatingunpackforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{"UNPCKLPS", "UNPCKHPS", "UNPCKLPD", "UNPCKHPD"} { + fmt.Fprintf(&source, "\t%s X0, X%d\n", op, legacyLast) + fmt.Fprintf(&source, "\t%s 8(AX), X%d\n", op, legacyLast) + vector := "V" + op + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "\t%s %s0, %s1, %s2\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s 8(AX), %s%d, %s%d\n", vector, width, second, width, destination) + fmt.Fprintf(&source, "\t%s.BCST 8(AX), %s%d, %s%d\n", vector, width, second, width, destination) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s20, %s21, K1, %s22\n", vector, width, width, width) + fmt.Fprintf(&source, "\t%s.BCST.Z 8(AX), %s21, K7, %s22\n", vector, width, width) + } + } + fmt.Fprintf(&source, "\t%s Z0, Z6, Z%d\n", vector, zLast) + fmt.Fprintf(&source, "\t%s 8(AX), Z6, Z%d\n", vector, zLast) + fmt.Fprintf(&source, "\t%s.BCST 8(AX), Z6, Z%d\n", vector, zLast) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s Z20, Z21, K2, Z22\n", vector) + fmt.Fprintf(&source, "\t%s.Z 8(AX), Z21, K6, Z22\n", vector) + } + } + source.WriteString("\tRET\n") + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "floatingunpackforms": {Name: "floatingunpackforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "floating-unpack-"+target.name+".ll", "floating-unpack-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86FloatingUnpackRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "UNPCKLPS.Z X0, X1", + "UNPCKHPD X0", + "UNPCKLPD M0, M1", + "UNPCKHPS Y0, X1", + "UNPCKLPS X0, (AX)", + "VUNPCKLPD X0, X1", + "VUNPCKHPS X0, Y1, Y2", + "VUNPCKLPD X0, X1, (AX)", + "VUNPCKHPD X0, X1, K0, X2", + "VUNPCKLPS.Z X0, X1, X2", + "VUNPCKHPS.BCST X0, X1, X2", + "VUNPCKLPD.BCST 8(AX), X1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86FloatingUnpackRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "UNPCKLPS X8, X0", + "UNPCKHPD X0, X8", + "VUNPCKLPD Z8, Z1, Z2", + "VUNPCKHPS Z0, Z1, Z8", + "VUNPCKLPS X0, X1, K1, X2", + "VUNPCKHPD.Z X0, X1, K1, X2", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86FloatingUnpackRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86FloatingUnpackRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's floating unpack forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_variable_shift.go b/amd64_lower_variable_shift.go new file mode 100644 index 000000000..5808d3b9e --- /dev/null +++ b/amd64_lower_variable_shift.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PerLaneVariableShiftSpec struct { + laneBits int + operation string + arithmetic bool + // Shared raw grammar: 66.0F38 opcode and optional VEX generation. + // W is fixed: W0 for dwords, W1 for words/qwords. + opcode byte + vex bool +} + +var amd64PerLaneVariableShiftSpecs = map[string]amd64PerLaneVariableShiftSpec{ + "VPSLLVW": {laneBits: 16, operation: "shl", opcode: 0x12}, + "VPSLLVD": {laneBits: 32, operation: "shl", opcode: 0x47, vex: true}, + "VPSLLVQ": {laneBits: 64, operation: "shl", opcode: 0x47, vex: true}, + "VPSRLVW": {laneBits: 16, operation: "lshr", opcode: 0x10}, + "VPSRLVD": {laneBits: 32, operation: "lshr", opcode: 0x45, vex: true}, + "VPSRLVQ": {laneBits: 64, operation: "lshr", opcode: 0x45, vex: true}, + "VPSRAVW": {laneBits: 16, operation: "ashr", arithmetic: true, opcode: 0x11}, + "VPSRAVD": {laneBits: 32, operation: "ashr", arithmetic: true, opcode: 0x46, vex: true}, + "VPSRAVQ": {laneBits: 64, operation: "ashr", arithmetic: true, opcode: 0x46}, +} + +// lowerPerLaneVariableShift implements the full Go 1.27 per-lane packed +// variable-shift family. The first Plan 9 source supplies one count per lane. +func (c *amd64Ctx) lowerPerLaneVariableShift(op Op, ins Instr) (ok bool, terminated bool, err error) { + if _, recognized, _ := x86PackedShiftProperties(op, ins, c.goarch); recognized { + return c.lowerX86PackedShift(op, ins) + } + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, ok := amd64PerLaneVariableShiftSpecs[baseOp] + if !ok { + return false, false, nil + } + suffixes, err := amd64ParseTwoSourcePermuteSuffixes(rawOp, baseOp) + if err != nil { + return true, false, fmt.Errorf("%w: %q", err, ins.Raw) + } + if suffixes.broadcast && spec.laneBits == 16 { + return true, false, fmt.Errorf("amd64 %s does not enable broadcast in Go 1.27: %q", baseOp, ins.Raw) + } + if len(ins.Args) != 3 && len(ins.Args) != 4 { + return true, false, fmt.Errorf("amd64 %s expects per-lane counts, source, [K mask,] destination: %q", baseOp, ins.Raw) + } + masked := len(ins.Args) == 4 + if masked && c.goarch == "386" && !ins.x86Encoded { + return true, false, fmt.Errorf("386 %s masked form exceeds the Go assembler frontend's operand limit: %q", baseOp, ins.Raw) + } + if suffixes.zeroing && !masked { + return true, false, fmt.Errorf("amd64 %s .Z requires a K1-K7 mask: %q", baseOp, ins.Raw) + } + + destination := ins.Args[len(ins.Args)-1] + byteWidth := 0 + if destination.Kind == OpReg { + byteWidth = amd64VectorByteWidth(destination.Reg) + } + if byteWidth == 0 || !c.isGoPackedVectorMoveRegister(destination, byteWidth) { + return true, false, fmt.Errorf("amd64 %s expects an in-range X, Y, or Z destination: %q", baseOp, ins.Raw) + } + data := ins.Args[1] + if !c.isGoPackedVectorMoveRegister(data, byteWidth) { + return true, false, fmt.Errorf("amd64 %s data source must match the destination width: %q", baseOp, ins.Raw) + } + counts := ins.Args[0] + if counts.Kind == OpReg { + if !c.isGoPackedVectorMoveRegister(counts, byteWidth) { + return true, false, fmt.Errorf("amd64 %s count register must match the destination width: %q", baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(counts) { + return true, false, fmt.Errorf("amd64 %s counts must be a matching vector register or memory: %q", baseOp, ins.Raw) + } + if suffixes.broadcast && !isAMD64MemoryOperand(counts) { + return true, false, fmt.Errorf("amd64 %s.BCST requires a memory count source: %q", baseOp, ins.Raw) + } + + mask := "" + if masked { + maskArg := ins.Args[2] + if maskArg.Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + maskIndex, validMask := amd64ParseKReg(maskArg.Reg) + if !validMask || maskIndex == 0 { + return true, false, fmt.Errorf("amd64 %s masked form expects K1-K7: %q", baseOp, ins.Raw) + } + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + + lanes := byteWidth * 8 / spec.laneBits + countValues, err := c.loadMaskedPackedCompareLanes(counts, byteWidth, spec.laneBits, suffixes.broadcast, mask) + if err != nil { + return true, false, err + } + dataValues, err := c.loadPackedCompareLanes(data, byteWidth, spec.laneBits, false) + if err != nil { + return true, false, err + } + result := c.emitPerLaneVariableShift(spec, lanes, countValues, dataValues) + if masked { + oldBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + old := c.bitcastVectorBytesToIntegerLanes(byteWidth, lanes, spec.laneBits, oldBytes) + result = amd64ApplyIntegerLaneMask(c, lanes, spec.laneBits, result, old, mask, suffixes.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, spec.laneBits, result, byteWidth) + return true, false, c.storePackedMoveOperand(destination, byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitPerLaneVariableShift(spec amd64PerLaneVariableShiftSpec, lanes int, counts, data string) string { + typeName := fmt.Sprintf("<%d x i%d>", lanes, spec.laneBits) + limit := amd64SplatInteger(c, lanes, spec.laneBits, fmt.Sprintf("%d", spec.laneBits)) + inRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult %s %s, %s\n", inRange, typeName, counts, limit) + fallbackCount := "zeroinitializer" + if spec.arithmetic { + fallbackCount = amd64SplatInteger(c, lanes, spec.laneBits, fmt.Sprintf("%d", spec.laneBits-1)) + } + safeCounts := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %s, %s %s\n", safeCounts, lanes, inRange, typeName, counts, typeName, fallbackCount) + shifted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %s, %%%s\n", shifted, spec.operation, typeName, data, safeCounts) + if spec.arithmetic { + return "%" + shifted + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %%%s, %s %%%s, %s zeroinitializer\n", result, lanes, inRange, typeName, shifted, typeName) + return "%" + result +} diff --git a/amd64_lower_variable_shift_test.go b/amd64_lower_variable_shift_test.go new file mode 100644 index 000000000..68b465c69 --- /dev/null +++ b/amd64_lower_variable_shift_test.go @@ -0,0 +1,114 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateX86PerLaneVariableShiftCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27 exposes the W/D/Q per-lane left, logical-right, and + // arithmetic-right shifts. Every opcode has X/Y/Z rows and optional K + // masking/.Z; D/Q additionally enable scalar count broadcast. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + lastZ, dataReg, dstReg := 20, 20, 21 + if target.goarch == "386" { + lastZ, dataReg, dstReg = 7, 6, 7 + } + var source strings.Builder + source.WriteString("TEXT perlanevariableshiftforms(SB),NOSPLIT,$0-0\n") + for _, op := range []string{ + "VPSLLVW", "VPSLLVD", "VPSLLVQ", + "VPSRLVW", "VPSRLVD", "VPSRLVQ", + "VPSRAVW", "VPSRAVD", "VPSRAVQ", + } { + fmt.Fprintf(&source, "\t%s X1, X%d, X%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s 8(AX), Y%d, Y%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s Z1, Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X1, X20, K1, X21\n", op) + fmt.Fprintf(&source, "\t%s.Z Y1, Y20, K2, Y21\n", op) + fmt.Fprintf(&source, "\t%s.Z 40(AX), Z20, K3, Z21\n", op) + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + fmt.Fprintf(&source, "\t%s.BCST 104(AX), X%d, X%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s.BCST 112(AX), Y%d, Y%d\n", op, dataReg, dstReg) + fmt.Fprintf(&source, "\t%s.BCST 120(AX), Z2, Z%d\n", op, lastZ) + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s.BCST.Z 128(AX), Z20, K4, Z21\n", op) + } + } + } + source.WriteString("\tRET\n") + assembleX87ControlBytes(t, target.goarch, strings.ReplaceAll(source.String(), "NOSPLIT", "4")) + + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "perlanevariableshiftforms": {Name: "perlanevariableshiftforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "per-lane-variable-shift-"+target.name+".ll", "per-lane-variable-shift-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86PerLaneVariableShiftRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VPSLLVQ X0, X1", + "VPSLLVQ $1, X1, X2", + "VPSLLVQ X0, Y1, Y2", + "VPSLLVQ X0, (AX), X2", + "VPSLLVQ X0, X1, K0, X2", + "VPSLLVQ X0, X1, AX", + "VPSLLVQ.Z X0, X1, X2", + "VPSLLVQ.BCST X0, X1, X2", + "VPSLLVW.BCST (AX), X1, X2", + "VPSLLVQ.Z.BCST (AX), X1, K1, X2", + "VPSLLVQ.RN_SAE X0, X1, X2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86PerLaneVariableShiftRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + assertX86PerLaneVariableShiftRejected(t, "386", "i386-unknown-linux-gnu", "VPSLLVQ X0, X1, K1, X2") + assertX86PerLaneVariableShiftRejected(t, "386", "i386-unknown-linux-gnu", "VPSLLVQ Z0, Z1, Z8") +} + +func assertX86PerLaneVariableShiftRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's per-lane variable shift forms", instruction) + } +} diff --git a/amd64_lower_vcomi.go b/amd64_lower_vcomi.go new file mode 100644 index 000000000..c2a59488f --- /dev/null +++ b/amd64_lower_vcomi.go @@ -0,0 +1,83 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerVectorScalarFlagCompare(op Op, ins Instr) (ok bool, terminated bool, err error) { + name, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86COMISpecs[Op(name)]; !known { + return false, false, nil + } + if !strings.EqualFold(string(op), string(ins.Op)) { + return true, false, fmt.Errorf("scalar flag comparison dispatch/source mismatch") + } + form, known, err := x86COMIProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + if !x86ScalarFloatIntegerNativeTarget(c.goarch, c.targetTriple) { + return true, false, fmt.Errorf("%w: scalar flag comparison requires its native x86 MXCSR target", ErrProbeNeedsContext) + } + if ins.x86AddressBits != 0 { + return true, false, fmt.Errorf("%w: scalar flag comparison needs original address-size binding", ErrProbeNeedsContext) + } + return true, false, c.emitNativeCOMI(ins, form) +} + +// A relation-only LLVM fcmp neither distinguishes COMI/UCOMI signaling nor +// preserves exceptions after FLAGS die. Keep one whole native comparison and +// capture CF/ZF/PF before compiler-generated instructions can overwrite them. +func (c *amd64Ctx) emitNativeCOMI(ins Instr, form x86COMIForm) error { + destination, err := c.evalScalarFloatForInteger(ins.Args[1], form.spec.bits) + if err != nil { + return err + } + typ := string(amd64FloatingTypeForBits(form.spec.bits)) + registerConstraint := "x" + if form.evex { + registerConstraint = "v" + } + source := ins.Args[0] + var value, sourceType, constraint, operand string + if source.Kind == OpReg || source.Kind == OpFP { + value, err = c.evalScalarFloatForInteger(source, form.spec.bits) + sourceType, constraint, operand = typ, registerConstraint, "$3" + } else { + var pointerType string + value, pointerType, _, err = c.packedVectorMemoryPointer(source) + sourceType = fmt.Sprintf("%s elementtype([%d x i8])", pointerType, form.spec.bits/8) + constraint, operand = "*m", "$3" + if pointerType == "ptr addrspace(256)" { + operand = "%gs:" + operand + } else if pointerType == "ptr addrspace(257)" { + operand = "%fs:" + operand + } + } + if err != nil { + return err + } + assembly := strings.ToLower(form.name) + if form.evex { + assembly = "{evex} " + assembly + } + if form.sae { + assembly += " {sae}," + } + assembly += " " + operand + ", $4; setb $0; sete $1; setp $2" + result := c.newTmp() + constraints := "=q,=q,=q," + constraint + "," + registerConstraint + ",~{flags},~{fpsr},~{memory}" + fmt.Fprintf(c.b, " %%%s = call { i8, i8, i8 } asm sideeffect %q, %q(%s %s, %s %s)\n", + result, assembly, constraints, sourceType, value, typ, destination) + for index, slot := range []string{c.flagsCFSlot, c.flagsZSlot, c.flagsPFSlot} { + byteValue, boolean := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractvalue { i8, i8, i8 } %%%s, %d\n", byteValue, result, index) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", boolean, byteValue) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", boolean, slot) + } + for _, slot := range []string{c.flagsSltSlot, c.flagsOFSlot, c.flagsAFSlot} { + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", slot) + } + return nil +} diff --git a/amd64_lower_vcvtss2sd_test.go b/amd64_lower_vcvtss2sd_test.go new file mode 100644 index 000000000..b9086dbde --- /dev/null +++ b/amd64_lower_vcvtss2sd_test.go @@ -0,0 +1,199 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestTranslateAMD64VCVTSS2SDCompleteGo127Forms(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT vcvtss2sdforms(SB),$0-0\n") + for _, op := range []string{"VCVTSS2SD", "VCVTSD2SS"} { + fmt.Fprintf(&source, "\t%s X0, X1, X2\n", op) + fmt.Fprintf(&source, "\t%s (AX), X3, X4\n", op) + fmt.Fprintf(&source, "\t%s X31, X30, X29\n", op) + if op == "VCVTSS2SD" { + fmt.Fprintf(&source, "\t%s.SAE X31, X30, X29\n", op) + } else { + for _, rounding := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + fmt.Fprintf(&source, "\t%s.%s X31, X30, X29\n", op, rounding) + } + } + if target.goarch == "amd64" { + fmt.Fprintf(&source, "\t%s X31, X30, K1, X29\n", op) + fmt.Fprintf(&source, "\t%s.Z (AX), X30, K2, X29\n", op) + if op == "VCVTSS2SD" { + fmt.Fprintf(&source, "\t%s.SAE.Z X31, X30, K3, X29\n", op) + } else { + fmt.Fprintf(&source, "\t%s.RD_SAE.Z X31, X30, K3, X29\n", op) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"vcvtss2sdforms": {Name: "vcvtss2sdforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "vcvtss2sd-"+target.name+".ll", "vcvtss2sd-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ScalarPrecisionConvertRejectsFormsOutsideGo127(t *testing.T) { + tests := []struct { + goarch string + triple string + instruction string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD.RD_SAE X0, X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSD2SS.SAE X0, X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD.SAE (AX), X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSD2SS.RN_SAE (AX), X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD.Z X0, X1, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSD2SS X0, X1, K0, X2"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu", instruction: "VCVTSS2SD Y0, X1, X2"}, + {goarch: "386", triple: "i386-unknown-linux-gnu", instruction: "VCVTSD2SS X0, X1, K1, X2"}, + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),NOSPLIT,$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err == nil { + _, err = Translate(file, Options{ + TargetTriple: test.triple, + Goarch: test.goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }) + } + if err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar precision-conversion forms for %s", test.instruction, test.goarch) + } + }) + } +} + +func TestAMD64ScalarPrecisionConvertRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT scalarPrecisionConvertSemantics(SB),NOSPLIT,$0-48 + MOVQ out+0(FP), AX + MOVQ doubles+8(FP), BX + MOVQ floats+16(FP), CX + MOVQ upper+24(FP), DX + MOVQ old+32(FP), SI + KMOVQ mask+40(FP), K1 + MOVSD (BX), X0 + MOVUPS (DX), X1 + MOVUPS (SI), X2 + VCVTSD2SS.RN_SAE X0, X1, X2 + MOVUPS X2, 0(AX) + VCVTSD2SS.RD_SAE X0, X1, X2 + MOVUPS X2, 16(AX) + VCVTSD2SS.RU_SAE X0, X1, X2 + MOVUPS X2, 32(AX) + VCVTSD2SS.RZ_SAE X0, X1, X2 + MOVUPS X2, 48(AX) + MOVUPS (SI), X2 + VCVTSD2SS.RU_SAE X0, X1, K1, X2 + MOVUPS X2, 64(AX) + MOVUPS (SI), X2 + VCVTSD2SS.RU_SAE.Z X0, X1, K1, X2 + MOVUPS X2, 80(AX) + MOVSS (CX), X0 + VCVTSS2SD.SAE X0, X1, X2 + MOVUPS X2, 96(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: Ptr, Index: 3, Field: -1}, + {Offset: 32, Type: Ptr, Index: 4, Field: -1}, + {Offset: 40, Type: I64, Index: 5, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "scalarPrecisionConvertSemantics": {Name: "scalarPrecisionConvertSemantics", Args: []LLVMType{Ptr, Ptr, Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void scalarPrecisionConvertSemantics(uint32_t *, const double *, const float *, const uint32_t *, const uint32_t *, uint64_t); +int main(void) { + double input = 1.0 + 0x1p-24; + float input32 = 1.5f; + uint32_t upper[4] = {0xaaaaaaaaU, 0x11111111U, 0x22222222U, 0x33333333U}; + uint32_t old[4] = {0xdeadbeefU, 0xbbbbbbbbU, 0xccccccccU, 0xddddddddU}; + uint32_t out[28] = {0}; + scalarPrecisionConvertSemantics(out, &input, &input32, upper, old, 0); + uint32_t low[4] = {0x3f800000U, 0x3f800000U, 0x3f800001U, 0x3f800000U}; + for (int group = 0; group < 4; group++) { + if (out[group*4] != low[group]) return 1 + group; + for (int lane = 1; lane < 4; lane++) if (out[group*4+lane] != upper[lane]) return 10 + group*4 + lane; + } + if (out[16] != old[0]) return 40; + if (out[20] != 0) return 41; + for (int lane = 1; lane < 4; lane++) { + if (out[16+lane] != upper[lane]) return 42 + lane; + if (out[20+lane] != upper[lane]) return 46 + lane; + } + uint64_t converted = 0, want = 0x3ff8000000000000ULL; + memcpy(&converted, &out[24], sizeof(converted)); + if (converted != want) return 60; + if (out[26] != upper[2] || out[27] != upper[3]) return 61; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_precision_convert", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_vec.go b/amd64_lower_vec.go index ebfb7a52f..f17c01aa0 100644 --- a/amd64_lower_vec.go +++ b/amd64_lower_vec.go @@ -6,130 +6,192 @@ import ( ) func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err error) { + if known, term, err := c.lowerLegacyPackedMove(op, ins); known { + return known, term, err + } + if ok, term, err := c.lowerFloatMulDiv(op, ins); ok { + return ok, term, err + } + if ok, terminated, err := c.lowerVectorZero(op, ins); ok { + return ok, terminated, err + } + if ok, term, err := c.lowerInsertExtractFamily(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFloatAddSub(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedLogicalFamily(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerFourSource(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedStringCompare(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerExplicitMaskMove(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerIndexedPermute(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerQwordBitLookup(op, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerMaskVectorMove(op, ins); ok { + return ok, term, err + } + rawOp := op + if ok, term, err := c.lowerSingleSourceIntegerNarrow(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedIntegerInsert(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerScalarFloatBroadcast(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerAES(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedWordShuffle(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedByteShuffle(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedBitCount(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedConflict(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedExpand(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerGFNI(rawOp, ins); ok { + return ok, term, err + } + if ok, term, err := c.lowerPackedCompress(rawOp, ins); ok { + return ok, term, err + } + zeroMasking := false if raw := string(op); strings.Contains(raw, ".") { + for _, suffix := range strings.Split(raw, ".")[1:] { + if suffix == "Z" { + zeroMasking = true + } + } if i := strings.IndexByte(raw, '.'); i >= 0 { op = Op(raw[:i]) } } + if op == "MOVD" { + if rawOp != op { + return true, false, fmt.Errorf("%s MOVD does not accept instruction suffixes: %q", c.goarch, ins.Raw) + } + if err := c.validateMOVDAliasForm(ins); err != nil { + return true, false, err + } + // cmd/asm maps the Plan 9 spelling MOVD to AMOVQ. In particular, + // memory/XMM forms transfer 64 bits; MOVD is not Intel's 32-bit MOVD + // mnemonic here. + op = "MOVQ" + } + if _, ok := amd64MaskMoveSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) + } + return true, false, c.lowerMaskMove(op, ins) + } + if _, ok := amd64GatherSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) + } + return true, false, c.lowerGather(op, ins) + } + if _, ok := amd64VSIBPrefetchSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) + } + return true, false, c.lowerGather(op, ins) + } + if _, ok := amd64MaskLogicalSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) + } + return true, false, c.lowerMaskLogical(op, ins) + } + if _, ok := amd64MaskArithmeticSpecs[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, op, ins.Raw) + } + return true, false, c.lowerMaskArithmetic(op, ins) + } + if _, ok := amd64MaskUnpackBits[op]; ok { + if rawOp != op { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, op, ins.Raw) + } + return true, false, c.lowerMaskUnpack(op, ins) + } + if _, ok := amd64TernaryLogicSpecs[op]; ok { + return c.lowerTernaryLogic(rawOp, ins) + } + if _, ok := amd64VectorAlignSpecs[op]; ok { + return c.lowerVectorAlign(rawOp, ins) + } + if _, ok := amd64IFMASpecs[op]; ok { + return c.lowerIFMA(rawOp, ins) + } + if _, ok := amd64FunnelShiftSpecs[op]; ok { + return c.lowerFunnelShift(rawOp, ins) + } + if _, ok := amd64PackedRotateSpecs[op]; ok { + return c.lowerPackedRotate(rawOp, ins) + } + if _, ok := amd64FMA3Specs[op]; ok { + return c.lowerFMA3(rawOp, ins) + } + if _, ok := amd64BinaryFloatingSpecs[op]; ok { + return c.lowerBinaryFloating(rawOp, ins) + } + if _, ok := amd64HorizontalFloatingSpecs[op]; ok { + return c.lowerHorizontalFloating(rawOp, ins) + } switch op { - case "MOVOU", "MOVOA", "MOVUPS", "MOVAPS", "MOVO", "MOVQ", "MOVL", "MOVD", + case "MOVOU", "MOVOA", "MOVUPS", "MOVAPS", "MOVO", "MOVQ", "VMOVQ", "MOVL", "MOVD", "VMOVDQU", "VMOVDQA", "VMOVNTDQ", "VMOVDQU64", "VMOVDQA64", "VMOVAPS", "VMOVAPD", - "VPCMPEQB", "VPCMPGTB", "VPMOVMSKB", "VZEROUPPER", "VZEROALL", "VPBROADCASTB", "VPBROADCASTQ", "VPAND", "VPXOR", "VPOR", "VPADDD", "VPADDQ", "VPTEST", - "VPANDQ", "VPXORQ", "VPORQ", "VPCLMULQDQ", "VPTERNLOGD", "VEXTRACTF32X4", - "VPERMB", "VGF2P8AFFINEQB", "VPERMI2B", "VPCOMPRESSQ", "VPOPCNTB", "VPCMPUQ", - "VBROADCASTI128", "VBROADCASTF32X2", "VBROADCASTSD", "VXORPD", - "VPSHUFB", "VPSHUFD", "VPSLLD", "VPSRLD", "VPSRLQ", "VPSLLQ", "VPSRLDQ", "VPSLLDQ", "VPUNPCKLBW", "VPUNPCKHBW", + "VPCMPEQB", "VPCMPGTB", "VPMOVMSKB", "VZEROUPPER", "VZEROALL", "VPBROADCASTB", "VPBROADCASTD", "VPBROADCASTQ", "VPAND", "VPXOR", "VPOR", "VPADDD", "VPADDQ", "VPSUBB", "VPTEST", "VTESTPD", "VTESTPS", + "VPCLMULQDQ", "VEXTRACTF32X4", + "VPCMPUQ", + "VBROADCASTF32X2", "VBROADCASTSD", + "VPSHUFB", "VPSHUFD", "VPSLLD", "VPSLLQ", "VPSLLW", "VPSRAD", "VPSRAW", "VPSRLD", "VPSRLQ", "VPSRLW", "VPSRLDQ", "VPSLLDQ", "VPUNPCKLBW", "VPUNPCKHBW", "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ", "VPALIGNR", "VPERM2I128", "VPERM2F128", "VINSERTI128", "VPBLENDD", - "PXOR", "POR", "PAND", "PANDN", "PADDD", "PADDL", "PSUBB", "PSUBL", "PMULULQ", "PCLMULQDQ", "PCMPEQB", "PCMPEQL", "PCMPGTB", "PMOVMSKB", - "PSHUFB", "PSRLDQ", "PSLLDQ", "PSRLQ", "PSLLW", "PSRLW", "PSRLL", "PSLLL", "PSRAL", "PEXTRD", "PEXTRB", "PEXTRQ", - "PINSRQ", "PINSRD", "PINSRB", "PINSRW", "PALIGNR", "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PSHUFL", "PSHUFD", "PSHUFHW", "SHUFPS", - "PBLENDW", "PADDQ", "KMOVB", "KMOVW", "KMOVQ", "KXORQ", + "PXOR", "POR", "PAND", "PADDB", "PADDD", "PADDL", "PSUBB", "PSUBL", "PSUBUSB", "PMULLW", "PMULULQ", "PCLMULQDQ", "PCMPEQB", "PCMPEQW", "PCMPEQL", "PCMPGTB", "PABSD", "PTEST", "PMOVMSKB", + "PMINUB", "PMINSB", "PMINUW", "PMINSW", "PMINUD", "PMINSD", "PMAXUB", "PMAXSB", "PMAXUW", "PMAXSW", "PMAXUD", "PMAXSD", + "PSHUFB", "PSRLDQ", "PSLLDQ", "PSRLO", "PSLLO", "PSRLQ", "PSLLQ", "PSLLW", "PSRLW", "PSRLL", "PSLLL", "PSRAL", "PEXTRD", "PEXTRB", "PEXTRQ", + "PALIGNR", "PUNPCKLBW", "PUNPCKHBW", "PUNPCKLLQ", "PUNPCKHLQ", "PUNPCKLQDQ", "PUNPCKHQDQ", "PSHUFL", "PSHUFD", "SHUFPS", + "PBLENDW", "PADDQ", "SHA1NEXTE", "SHA1MSG1", "SHA1MSG2", "SHA1RNDS4", "SHA256MSG1", "SHA256MSG2", "SHA256RNDS2", - "AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC", "AESKEYGENASSIST", "PCMPESTRI": + "AESENC", "AESENCLAST", "AESDEC", "AESDECLAST", "AESIMC", "AESKEYGENASSIST": // ok default: return false, false, nil } - // MOVL src, Xn (seed vector low 32 bits). - if op == "MOVL" && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); ok { - var v64 string - var err error - switch ins.Args[0].Kind { - case OpImm, OpReg, OpFP, OpMem, OpSym: - v64, err = c.evalI64(ins.Args[0]) - default: - return true, false, fmt.Errorf("amd64 MOVL to X reg unsupported src: %q", ins.Raw) - } - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - // Build <4 x i32> { crc, 0, 0, 0 } then bitcast to <16 x i8>. - v0 := "%" + tr - tvec := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> zeroinitializer, i32 %s, i32 0\n", tvec, v0) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc, tvec) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - } - } - // MOVL Xn, dst stores the low 32 bits of the vector register. - if op == "MOVL" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { - vec, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - words := c.newTmp() - low := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, vec) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 0\n", low, words) - switch ins.Args[1].Kind { - case OpReg: - return true, false, c.storeRegSized(ins.Args[1].Reg, I32, "%"+low) - case OpFP: - return true, false, c.storeFPResult(ins.Args[1].FPOffset, I32, "%"+low) - case OpMem: - p, ptrType, err := c.ptrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i32 %%%s, %s %s, align 1\n", low, ptrType, p) - return true, false, nil - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store i32 %%%s, ptr %s, align 1\n", low, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 MOVL from X reg unsupported dst: %q", ins.Raw) - } - } - } - - // MOVD reg, Xn (seed vector with low 32-bit value) - if op == "MOVD" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg && ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); ok { - v64, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - tr := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", tr, v64) - // Build <4 x i32> { v, 0, 0, 0 } then bitcast to <16 x i8>. - v0 := "%" + tr - tvec := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> zeroinitializer, i32 %s, i32 0\n", tvec, v0) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc, tvec) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - } - } - - if op == "VZEROUPPER" || op == "VZEROALL" { - // These instructions affect registers 0-15, including their ZMM view. - for i := 0; i < 16; i++ { - c.clearVectorUpper(i, 16) - if op == "VZEROALL" { - if slot := c.xRegSlot[i]; slot != "" { - fmt.Fprintf(c.b, " store <16 x i8> zeroinitializer, ptr %s\n", slot) - } - } - } - return true, false, nil - } - if op == "VMOVDQA" { op = "VMOVDQU" } if op == "VMOVDQA64" { op = "VMOVDQU64" } + if op == "PSLLO" { + op = "PSLLDQ" + } + if op == "PSRLO" { + op = "PSRLDQ" + } if op == "VPERM2F128" { op = "VPERM2I128" } @@ -143,310 +205,8 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } } } - if op == "VPXORQ" || op == "VPANDQ" || op == "VPORQ" { - if len(ins.Args) == 3 && ins.Args[2].Kind == OpReg && !isAMD64ZReg(ins.Args[2].Reg) { - switch op { - case "VPXORQ": - op = "VPXOR" - case "VPANDQ": - op = "VPAND" - case "VPORQ": - op = "VPOR" - } - } - } - - // MOVUPS/MOVAPS/MOVO are aliases for unaligned/aligned 128-bit xmm moves. - if op == "MOVUPS" || op == "MOVAPS" || op == "MOVO" { - op = "MOVOU" - } - - // MOVQ src, Xn (load low 64 bits into Xn) - if op == "MOVQ" && len(ins.Args) == 2 && ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); ok { - var low string - switch ins.Args[0].Kind { - case OpImm: - low = fmt.Sprintf("%d", ins.Args[0].Imm) - case OpReg: - v, err := c.loadReg(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - low = v - case OpFP: - v, err := c.evalFPToI64(ins.Args[0].FPOffset) - if err != nil { - return true, false, err - } - low = v - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - low = "%" + ld - case OpSym: - p, err := c.ptrFromSB(ins.Args[0].Sym) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load i64, ptr %s, align 1\n", ld, p) - low = "%" + ld - default: - return true, false, fmt.Errorf("amd64 MOVQ to X reg unsupported src: %q", ins.Raw) - } - // Build <2 x i64> { low, 0 }. - ins0 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> zeroinitializer, i64 %s, i32 0\n", ins0, low) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, ins0) - return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - } - } - - // MOVQ Xn, dst (extract low 64 bits from Xn). - if op == "MOVQ" && len(ins.Args) == 2 && ins.Args[0].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { - xv, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bc, xv) - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", lo, bc) - switch ins.Args[1].Kind { - case OpReg: - return true, false, c.storeReg(ins.Args[1].Reg, "%"+lo) - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - fmt.Fprintf(c.b, " store i64 %%%s, ptr %s, align 1\n", lo, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 MOVQ from X reg unsupported dst: %q", ins.Raw) - } - } - } switch op { - case "KXORQ": - if len(ins.Args) != 3 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 KXORQ expects Ksrc1, Ksrc2, Kdst: %q", ins.Raw) - } - if _, ok := amd64ParseKReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseKReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseKReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - a, err := c.loadK(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - b, err := c.loadK(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - x := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor i64 %s, %s\n", x, a, b) - return true, false, c.storeK(ins.Args[2].Reg, "%"+x) - - case "KMOVB", "KMOVW", "KMOVQ": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects 2 operands: %q", op, ins.Raw) - } - bits := 64 - loadTy := "i64" - switch op { - case "KMOVB": - bits, loadTy = 8, "i8" - case "KMOVW": - bits, loadTy = 16, "i16" - } - maskValue := func(v string) string { - if bits == 64 { - return v - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i64 %s, %d\n", t, v, (uint64(1)< %s to <64 x i8>\n", out, outVec) return true, false, c.storeZ(ins.Args[2].Reg, "%"+out) - case "VPXORQ", "VPANDQ", "VPORQ": - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects src1, src2, dstReg: %q", op, ins.Raw) - } - if _, ok := amd64ParseZReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - a, err := c.loadZVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadZVecOperand(ins.Args[1]) - if err != nil { - return true, false, err + case "VPBROADCASTD": + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 VPBROADCASTD expects scalar src, vector dst: %q", ins.Raw) } - ab := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", ab, a) - bb := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", bb, b) - t := c.newTmp() - switch op { - case "VPXORQ": - fmt.Fprintf(c.b, " %%%s = xor <8 x i64> %%%s, %%%s\n", t, ab, bb) - case "VPANDQ": - fmt.Fprintf(c.b, " %%%s = and <8 x i64> %%%s, %%%s\n", t, ab, bb) - case "VPORQ": - fmt.Fprintf(c.b, " %%%s = or <8 x i64> %%%s, %%%s\n", t, ab, bb) + var low32 string + if ins.Args[0].Kind == OpReg { + if _, ok := amd64ParseXReg(ins.Args[0].Reg); ok { + value, err := c.loadX(ins.Args[0].Reg) + if err != nil { + return true, false, err + } + words := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", words, value) + extracted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <4 x i32> %%%s, i32 0\n", extracted, words) + low32 = "%" + extracted + } } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i64> %%%s to <64 x i8>\n", out, t) - return true, false, c.storeZ(ins.Args[2].Reg, "%"+out) - - case "VBROADCASTI128": - // VBROADCASTI128 Xsrc|mem, Ydst - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VBROADCASTI128 expects Xsrc|mem, Ydst: %q", ins.Raw) + if low32 == "" { + var value64 string + var err error + switch ins.Args[0].Kind { + case OpMem: + addr, addrErr := c.addrFromMem(ins.Args[0].Mem) + if addrErr != nil { + return true, false, addrErr + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", loaded, c.ptrFromAddrI64(addr)) + low32 = "%" + loaded + case OpSym: + p, ptrErr := c.ptrFromSB(ins.Args[0].Sym) + if ptrErr != nil { + return true, false, ptrErr + } + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i32, ptr %s, align 1\n", loaded, p) + low32 = "%" + loaded + default: + value64, err = c.evalI64(ins.Args[0]) + if err != nil { + return true, false, err + } + } + if low32 == "" { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", truncated, value64) + low32 = "%" + truncated + } } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { + lanes := 0 + store := func(string) error { return nil } + switch { + case isAMD64XReg(ins.Args[1].Reg): + lanes = 4 + store = func(value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %s to <16 x i8>\n", bytesValue, value) + return c.storeX(ins.Args[1].Reg, "%"+bytesValue) + } + case isAMD64YReg(ins.Args[1].Reg): + lanes = 8 + store = func(value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <8 x i32> %s to <32 x i8>\n", bytesValue, value) + return c.storeY(ins.Args[1].Reg, "%"+bytesValue) + } + case isAMD64ZReg(ins.Args[1].Reg): + lanes = 16 + store = func(value string) error { + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i32> %s to <64 x i8>\n", bytesValue, value) + return c.storeZ(ins.Args[1].Reg, "%"+bytesValue) + } + default: return false, false, nil } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i32> zeroinitializer, i32 %s, i32 0\n", seed, lanes, low32) + mask := make([]string, lanes) + for i := range mask { + mask[i] = "i32 0" } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> %s, <32 x i32> %s\n", out, src, src, llvmRepeatI8Mask(16, 32)) - return true, false, c.storeY(ins.Args[1].Reg, "%"+out) + broadcast := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i32> %%%s, <%d x i32> zeroinitializer, <%d x i32> <%s>\n", broadcast, lanes, seed, lanes, lanes, strings.Join(mask, ", ")) + return true, false, store("%" + broadcast) case "VBROADCASTF32X2", "VBROADCASTSD", "VPBROADCASTQ": // These instructions broadcast the low 64 source bits. Their lane @@ -651,54 +450,7 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i8> %%%s, <8 x i8> %%%s, <64 x i32> %s\n", out, chunk, chunk, llvmRepeatI8Mask(8, 64)) return true, false, c.storeZ(ins.Args[1].Reg, "%"+out) - case "VXORPD": - // VXORPD src1, src2, dst; this is a bitwise operation despite the - // floating-point mnemonic, so byte-vector xor preserves all bits. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VXORPD expects src1, src2, dst: %q", ins.Raw) - } - if _, ok := amd64ParseZReg(ins.Args[2].Reg); ok { - a, err := c.loadZVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadZVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <64 x i8> %s, %s\n", t, a, b) - return true, false, c.storeZ(ins.Args[2].Reg, "%"+t) - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { - a, err := c.loadYVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %s, %s\n", t, a, b) - return true, false, c.storeY(ins.Args[2].Reg, "%"+t) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { - a, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, a, b) - return true, false, c.storeX(ins.Args[2].Reg, "%"+t) - } - return false, false, nil - - case "VPXOR", "VPOR", "VPADDD", "VPADDQ": + case "VPADDD", "VPADDQ", "VPSUBB": // V* three-operand op; support both X and Y destinations. if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { return true, false, fmt.Errorf("amd64 %s expects src1, src2, dstReg: %q", op, ins.Raw) @@ -714,10 +466,6 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } t := c.newTmp() switch op { - case "VPXOR": - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %s, %s\n", t, a, b) - case "VPOR": - fmt.Fprintf(c.b, " %%%s = or <32 x i8> %s, %s\n", t, a, b) case "VPADDD": ab := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x i32>\n", ab, a) @@ -734,6 +482,8 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err add := c.newTmp() fmt.Fprintf(c.b, " %%%s = add <4 x i64> %%%s, %%%s\n", add, ab, bb) fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", t, add) + case "VPSUBB": + fmt.Fprintf(c.b, " %%%s = sub <32 x i8> %s, %s\n", t, b, a) } return true, false, c.storeY(ins.Args[2].Reg, "%"+t) } @@ -748,10 +498,6 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } t := c.newTmp() switch op { - case "VPXOR": - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, a, b) - case "VPOR": - fmt.Fprintf(c.b, " %%%s = or <16 x i8> %s, %s\n", t, a, b) case "VPADDD": ab := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", ab, a) @@ -768,201 +514,173 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err add := c.newTmp() fmt.Fprintf(c.b, " %%%s = add <2 x i64> %%%s, %%%s\n", add, ab, bb) fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", t, add) + case "VPSUBB": + fmt.Fprintf(c.b, " %%%s = sub <16 x i8> %s, %s\n", t, b, a) } return true, false, c.storeX(ins.Args[2].Reg, "%"+t) } return false, false, nil - case "VPSHUFB": - // VPSHUFB Ymask, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPSHUFB expects Ymask, Ysrc, Ydst: %q", ins.Raw) + case "VPSLLD", "VPSLLQ", "VPSLLW", "VPSRAD", "VPSRAW", "VPSRLD", "VPSRLQ", "VPSRLW": + // Go's x86 assembler uses _yvpslld for this whole family: + // $imm8, X|Y|Z src/mem, [Kmask,] X|Y|Z dst + // X|m128 count, X|Y|Z src, [Kmask,] X|Y|Z dst + // The count is uniform: variable forms use the low 64 bits of X|m128. + if (len(ins.Args) != 3 && len(ins.Args) != 4) || ins.Args[len(ins.Args)-1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects count, vector src, [K mask,] vector dst: %q", op, ins.Raw) + } + dst := ins.Args[len(ins.Args)-1].Reg + byteWidth := 0 + var src string + var err error + loadDst := func() (string, error) { return "", nil } + store := func(string) error { return nil } + switch { + case isAMD64XReg(dst): + byteWidth = 16 + src, err = c.loadXVecOperand(ins.Args[1]) + loadDst = func() (string, error) { return c.loadX(dst) } + store = func(v string) error { return c.storeX(dst, v) } + case isAMD64YReg(dst): + byteWidth = 32 + src, err = c.loadYVecOperand(ins.Args[1]) + loadDst = func() (string, error) { return c.loadY(dst) } + store = func(v string) error { return c.storeY(dst, v) } + case isAMD64ZReg(dst): + byteWidth = 64 + src, err = c.loadZVecOperand(ins.Args[1]) + loadDst = func() (string, error) { return c.loadZ(dst) } + store = func(v string) error { return c.storeZ(dst, v) } + default: + return false, false, nil } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { - mask, err := c.loadYVecOperand(ins.Args[0]) - if err != nil { - return true, false, err + if err != nil { + return true, false, err + } + + elemBits := 32 + switch op[len(op)-1] { + case 'Q': + elemBits = 64 + case 'W': + elemBits = 16 + } + lanes := byteWidth * 8 / elemBits + vecType := fmt.Sprintf("<%d x i%d>", lanes, elemBits) + bc := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", bc, byteWidth, src, vecType) + + llvmOp := "shl" + arithmetic := op == "VPSRAD" || op == "VPSRAW" + if arithmetic { + llvmOp = "ashr" + } else if strings.HasPrefix(string(op), "VPSRL") { + llvmOp = "lshr" + } + + shifted := "zeroinitializer" + if ins.Args[0].Kind == OpImm { + count := uint64(ins.Args[0].Imm) & 0xff + if arithmetic && count >= uint64(elemBits) { + count = uint64(elemBits - 1) + } + if arithmetic || count < uint64(elemBits) { + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %s\n", sh, llvmOp, vecType, bc, llvmSplatInteger(lanes, elemBits, count)) + shifted = "%" + sh } - src, err := c.loadYVecOperand(ins.Args[1]) + } else { + countVec, err := c.loadXVecOperand(ins.Args[0]) if err != nil { - return true, false, err + return true, false, fmt.Errorf("amd64 %s count: %w", op, err) + } + countWords := c.newTmp() + count := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", countWords, countVec) + fmt.Fprintf(c.b, " %%%s = extractelement <2 x i64> %%%s, i32 0\n", count, countWords) + inRange := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult i64 %%%s, %d\n", inRange, count, elemBits) + safeCount := c.newTmp() + fallback := 0 + if arithmetic { + fallback = elemBits - 1 + } + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %d\n", safeCount, inRange, count, fallback) + shiftCount := "%" + safeCount + if elemBits < 64 { + truncated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i%d\n", truncated, safeCount, elemBits) + shiftCount = "%" + truncated + } + countSplat := amd64SplatInteger(c, lanes, elemBits, shiftCount) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s %s %%%s, %s\n", sh, llvmOp, vecType, bc, countSplat) + shifted = "%" + sh + if !arithmetic { + condition := amd64SplatI1(c, lanes, "%"+inRange) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <%d x i1> %s, %s %s, %s zeroinitializer\n", selected, lanes, condition, vecType, shifted, vecType) + shifted = "%" + selected } - // Emulate 256-bit PSHUFB by lane-splitting into two 128-bit shuffles. - srcLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", srcLo, src, llvmI32RangeMask(0, 16)) - srcHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", srcHi, src, llvmI32RangeMask(16, 16)) - maskLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", maskLo, mask, llvmI32RangeMask(0, 16)) - maskHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", maskHi, mask, llvmI32RangeMask(16, 16)) - outLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %%%s, <16 x i8> %%%s)\n", outLo, srcLo, maskLo) - outHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %%%s, <16 x i8> %%%s)\n", outHi, srcHi, maskHi) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <32 x i32> %s\n", out, outLo, outHi, llvmI32RangeMask(0, 32)) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { - mask, err := c.loadXVecOperand(ins.Args[0]) + + if len(ins.Args) == 4 { + if ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s masked form expects K mask: %q", op, ins.Raw) + } + if _, ok := amd64ParseKReg(ins.Args[2].Reg); !ok { + return false, false, nil + } + maskValue, err := c.loadK(ins.Args[2].Reg) if err != nil { return true, false, err } - src, err := c.loadXVecOperand(ins.Args[1]) + oldBytes, err := loadDst() if err != nil { return true, false, err } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %s, <16 x i8> %s)\n", call, src, mask) - return true, false, c.storeX(ins.Args[2].Reg, "%"+call) - } - return false, false, nil - - case "VPSHUFD": - // VPSHUFD $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPSHUFD expects $imm, Ysrc, Ydst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := uint64(ins.Args[0].Imm) & 0xff - idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err + old := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", old, byteWidth, oldBytes, vecType) + shifted = amd64ApplyIntegerLaneMask(c, lanes, elemBits, shifted, "%"+old, maskValue, zeroMasking) } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x i32>\n", bc, src) - mask := fmt.Sprintf("<8 x i32> ", - idx(0), idx(1), idx(2), idx(3), - 4+idx(0), 4+idx(1), 4+idx(2), 4+idx(3)) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i32> %%%s, <8 x i32> zeroinitializer, %s\n", sh, bc, mask) out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i32> %%%s to <32 x i8>\n", out, sh) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", out, vecType, shifted, byteWidth) + return true, false, store("%" + out) - case "VPSLLD", "VPSRLD": - // VPS*D $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Ysrc, Ydst: %q", op, ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm & 31 - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <8 x i32>\n", bc, src) - sh := c.newTmp() - if op == "VPSLLD" { - fmt.Fprintf(c.b, " %%%s = shl <8 x i32> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) - } else { - fmt.Fprintf(c.b, " %%%s = lshr <8 x i32> %%%s, \n", sh, bc, n, n, n, n, n, n, n, n) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i32> %%%s to <32 x i8>\n", out, sh) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) + case "VPALIGNR": + return c.lowerPackedAlignRight(rawOp, ins) - case "VPSRLQ", "VPSLLQ": - // VPSR/LQ $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Ysrc, Ydst: %q", op, ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm & 63 - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err + case "VPERM2I128": + // VPERM2I128 $imm, Ysrc1, Ysrc2, Ydst + if rawOp != "VPERM2I128" && rawOp != "VPERM2F128" { + return true, false, fmt.Errorf("amd64 %s does not accept instruction suffixes: %q", op, ins.Raw) } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", bc, src) - sh := c.newTmp() - if op == "VPSLLQ" { - fmt.Fprintf(c.b, " %%%s = shl <4 x i64> %%%s, \n", sh, bc, n, n, n, n) - } else { - fmt.Fprintf(c.b, " %%%s = lshr <4 x i64> %%%s, \n", sh, bc, n, n, n, n) + if c.goarch != "amd64" { + return true, false, fmt.Errorf("%s %s exceeds the Go assembler frontend's operand limit: %q", c.goarch, rawOp, ins.Raw) } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i64> %%%s to <32 x i8>\n", out, sh) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) - - case "VPALIGNR": - // VPALIGNR $imm, Ysrc1, Ysrc2, Ydst ; dst = alignr(src2, src1, imm) - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPALIGNR expects $imm, Ysrc1, Ysrc2, Ydst: %q", ins.Raw) + if len(ins.Args) != 4 || !amd64UnsignedImmediate(ins.Args[0], 8) || ins.Args[2].Kind != OpReg || ins.Args[3].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects $imm8, Y/m256, Ysrc, Ydst: %q", rawOp, ins.Raw) } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil + if ins.Args[1].Kind == OpReg { + if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { + return true, false, fmt.Errorf("amd64 %s expects a Y/m256 first source: %q", rawOp, ins.Raw) + } + } else if ins.Args[1].Kind != OpMem && ins.Args[1].Kind != OpSym { + return true, false, fmt.Errorf("amd64 %s expects a Y/m256 first source: %q", rawOp, ins.Raw) } if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 { - n = 0 - } - if n > 255 { - n = 255 - } - src1, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - src2, err := c.loadY(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - lo1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", lo1, src1, llvmI32RangeMask(0, 16)) - hi1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", hi1, src1, llvmI32RangeMask(16, 16)) - lo2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", lo2, src2, llvmI32RangeMask(0, 16)) - hi2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", hi2, src2, llvmI32RangeMask(16, 16)) - outLo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <16 x i32> %s\n", outLo, lo2, lo1, llvmAlignRightBytesMask(n)) - outHi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <16 x i32> %s\n", outHi, hi2, hi1, llvmAlignRightBytesMask(n)) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <32 x i32> %s\n", out, outLo, outHi, llvmI32RangeMask(0, 32)) - return true, false, c.storeY(ins.Args[3].Reg, "%"+out) - - case "VPERM2I128": - // VPERM2I128 $imm, Ysrc1, Ysrc2, Ydst - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPERM2I128 expects $imm, Ysrc1, Ysrc2, Ydst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 %s expects a Y-register second source: %q", rawOp, ins.Raw) } if _, ok := amd64ParseYReg(ins.Args[3].Reg); !ok { - return false, false, nil + return true, false, fmt.Errorf("amd64 %s expects a Y-register destination: %q", rawOp, ins.Raw) } - imm := uint64(ins.Args[0].Imm) & 0xff + imm := uint64(ins.Args[0].Imm) // AT&T order: imm, src1, src2, dst ; choose lanes from src2 (arg2) + src1 (arg1). src1, err := c.loadYVecOperand(ins.Args[1]) if err != nil { return true, false, err } - src2, err := c.loadYVecOperand(ins.Args[2]) + src2, err := c.loadY(ins.Args[2].Reg) if err != nil { return true, false, err } @@ -980,7 +698,7 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } lowSel := sel(imm) hiSel := sel(imm >> 4) - mask := fmt.Sprintf("<4 x i32> ", lowSel, lowSel+4, hiSel, hiSel+4) + mask := fmt.Sprintf("<4 x i32> ", lowSel*2, lowSel*2+1, hiSel*2, hiSel*2+1) b2 := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <4 x i64>\n", b2, src2) b1 := c.newTmp() @@ -1060,36 +778,10 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.storeY(ins.Args[3].Reg, "%"+out) case "VMOVNTDQ": - // VMOVNTDQ Ysrc, mem - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VMOVNTDQ expects Ysrc, dst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - src, err := c.loadY(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - switch ins.Args[1].Kind { - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - fmt.Fprintf(c.b, " store <32 x i8> %s, ptr %s, align 1\n", src, p) - return true, false, nil - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store <32 x i8> %s, ptr %s, align 1\n", src, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 VMOVNTDQ unsupported dst: %q", ins.Raw) - } + // Compatibility entry point for direct lowerVec unit tests. Ordinary + // translation dispatches the whole family through the dedicated + // lowerer before reaching this legacy switch. + return c.lowerNonTemporalVectorMove(op, ins) case "AESENC", "AESENCLAST", "AESDEC", "AESDECLAST": // Two-operand form: OP Xsrc, Xdst @@ -1143,238 +835,35 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) return true, false, c.storeX(ins.Args[1].Reg, "%"+bc) - case "AESKEYGENASSIST": - // Three-operand form: AESKEYGENASSIST $imm, Xsrc, Xdst - if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 AESKEYGENASSIST expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - if ins.Args[0].Kind != OpImm { - return true, false, fmt.Errorf("amd64 AESKEYGENASSIST expects immediate first operand: %q", ins.Raw) - } - imm := ins.Args[0].Imm & 0xff - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - src2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", src2, src) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %%%s, i8 %d)\n", call, src2, imm) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc) - - case "VPTEST": - // VPTEST Ysrc1, Ysrc2 (sets flags based on bitwise tests; used with JZ/JNZ). - // We implement the ZF behavior (ZF=1 iff (a&b)==0) since stdlib uses JNZ. - if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPTEST expects Ysrc1, Ysrc2: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - a, err := c.loadY(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - b, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - and := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and <32 x i8> %s, %s\n", and, a, b) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %%%s to <4 x i64>\n", bc, and) - e0, e1, e2, e3 := c.newTmp(), c.newTmp(), c.newTmp(), c.newTmp() - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 0\n", e0, bc) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 1\n", e1, bc) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 2\n", e2, bc) - fmt.Fprintf(c.b, " %%%s = extractelement <4 x i64> %%%s, i32 3\n", e3, bc) - o01 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", o01, e0, e1) - o23 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", o23, e2, e3) - o := c.newTmp() - fmt.Fprintf(c.b, " %%%s = or i64 %%%s, %%%s\n", o, o01, o23) - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i64 %%%s, 0\n", z, o) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", z, c.flagsZSlot) - // CF/other flags are not used by stdlib here; clear CF to avoid stale reads. - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsCFSlot) - return true, false, nil - - case "PCMPESTRI": - // PCMPESTRI $imm, mem, Xsrc (implicit lengths; result index in ECX). - // - // LLVM 19 has backend issues selecting the native X86ISD::PCMPESTR node - // for the corresponding intrinsic in some environments. For now we - // emulate the subset used by the Go stdlib: - // imm=0x0c: unsigned byte compare, equal ordered, first match. - // - // We compute the first offset i in [0..15] where: - // - bytes(mem[i:]) has a prefix that matches Xsrc[0:lenA] - // - or a partial match at the end of the 16-byte block (to drive the - // stdlib's "partial match" control flow). - // If no match/partial match exists, return 16. - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PCMPESTRI expects $imm, mem, Xsrc: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := int64(ins.Args[0].Imm) & 0xff - if imm != 0x0c { - return true, false, fmt.Errorf("amd64 PCMPESTRI: unsupported imm 0x%x (only 0x0c supported): %q", imm, ins.Raw) - } - - // A: needle bytes. - a, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - // B: 16-byte haystack chunk. - var bvec string - switch ins.Args[1].Kind { - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - bvec = "%" + ld - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - bvec = "%" + ld - default: - return true, false, fmt.Errorf("amd64 PCMPESTRI unsupported mem operand: %q", ins.Raw) - } - - ax, err := c.loadReg(AX) - if err != nil { - return true, false, err - } - la := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", la, ax) - - allOnes := llvmAllOnesI8Vec(16) - idx := c.newTmp() - fmt.Fprintf(c.b, " %%%s = add i32 0, 16\n", idx) - prev := "%" + idx - - // Build first-match index using a select chain. - for i := 0; i < 16; i++ { - // Shuffle B so that element 0 is B[i], padding beyond 16 with 0. - maskElts := make([]string, 0, 16) - for k := 0; k < 16; k++ { - j := i + k - if j < 16 { - maskElts = append(maskElts, fmt.Sprintf("i32 %d", j)) - } else { - // 16 selects element 0 of the second (zero) vector. - maskElts = append(maskElts, "i32 16") - } - } - mask := "<16 x i32> <" + strings.Join(maskElts, ", ") + ">" - - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, %s\n", sh, bvec, mask) - cmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq <16 x i8> %%%s, %s\n", cmp, sh, a) - sel := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select <16 x i1> %%%s, <16 x i8> %s, <16 x i8> zeroinitializer\n", sel, cmp, allOnes) - pm := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %%%s)\n", pm, sel) - - // minLen = min(lenA, 16-i) - cap := 16 - i - lt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp ult i32 %%%s, %d\n", lt, la, cap) - min := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %%%s, i32 %d\n", min, lt, la, cap) - sh1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shl i32 1, %%%s\n", sh1, min) - req := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sub i32 %%%s, 1\n", req, sh1) - have := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i32 %%%s, %%%s\n", have, pm, req) - okt := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i32 %%%s, %%%s\n", okt, have, req) - - unset := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq i32 %s, 16\n", unset, prev) - take := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", take, okt, unset) - next := c.newTmp() - fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i32 %d, i32 %s\n", next, take, i, prev) - prev = "%" + next + case "AESKEYGENASSIST": + // Three-operand form: AESKEYGENASSIST $imm, Xsrc, Xdst + if len(ins.Args) != 3 || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 AESKEYGENASSIST expects $imm, Xsrc, Xdst: %q", ins.Raw) } - - z := c.newTmp() - fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", z, prev) - return true, false, c.storeReg(CX, "%"+z) - - case "VPAND": - // VPAND src1, src2, dst supports both XMM and YMM widths. - if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPAND expects src1, src2, dst: %q", ins.Raw) + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return false, false, nil } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { - for _, arg := range ins.Args[:2] { - if arg.Kind == OpReg { - if _, ok := amd64ParseYReg(arg.Reg); !ok { - return false, false, nil - } - } - } - a, err := c.loadYVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and <32 x i8> %s, %s\n", t, a, b) - return true, false, c.storeY(ins.Args[2].Reg, "%"+t) + if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { + return false, false, nil } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { - for _, arg := range ins.Args[:2] { - if arg.Kind == OpReg { - if _, ok := amd64ParseXReg(arg.Reg); !ok { - return false, false, nil - } - } - } - a, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - b, err := c.loadXVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - t := c.newTmp() - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %s, %s\n", t, a, b) - return true, false, c.storeX(ins.Args[2].Reg, "%"+t) + if ins.Args[0].Kind != OpImm { + return true, false, fmt.Errorf("amd64 AESKEYGENASSIST expects immediate first operand: %q", ins.Raw) } - return false, false, nil + imm := ins.Args[0].Imm & 0xff + src, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + src2 := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", src2, src) + call := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %%%s, i8 %d)\n", call, src2, imm) + bc := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) + return true, false, c.storeX(ins.Args[2].Reg, "%"+bc) + + case "PTEST", "VPTEST", "VTESTPD", "VTESTPS": + return c.lowerVectorTest(rawOp, ins) case "VPBLENDD": // VPBLENDD $imm, Ysrc1, Ysrc2, Ydst @@ -1435,43 +924,7 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.storeY(ins.Args[1].Reg, "%"+spl) case "VPSRLDQ", "VPSLLDQ": - // VPSR/LLDQ $imm, Ysrc, Ydst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Ysrc, Ydst: %q", op, ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 { - n = 0 - } - if n > 16 { - n = 16 - } - src, err := c.loadY(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - lo := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", lo, src, llvmI32RangeMask(0, 16)) - hi := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <32 x i8> %s, <32 x i8> zeroinitializer, <16 x i32> %s\n", hi, src, llvmI32RangeMask(16, 16)) - lo2 := c.newTmp() - hi2 := c.newTmp() - if op == "VPSLLDQ" { - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", lo2, lo, llvmShiftLeftBytesMask(n)) - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", hi2, hi, llvmShiftLeftBytesMask(n)) - } else { - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", lo2, lo, llvmShiftRightBytesMask(n)) - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> zeroinitializer, <16 x i32> %s\n", hi2, hi, llvmShiftRightBytesMask(n)) - } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %%%s, <16 x i8> %%%s, <32 x i32> %s\n", out, lo2, hi2, llvmI32RangeMask(0, 32)) - return true, false, c.storeY(ins.Args[2].Reg, "%"+out) + return c.lowerPackedByteShift(rawOp, ins) case "VPUNPCKLBW", "VPUNPCKHBW": if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { @@ -1673,58 +1126,100 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", out, sh) return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "PSHUFL", "PSHUFD": - // PSHUFL $imm, Xsrc, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects $imm, Xsrc, Xdst: %q", op, ins.Raw) + case "PUNPCKHQDQ": + // PUNPCKHQDQ Xsrc/m128, Xdst interleaves the high quadwords: + // dst = {dst[127:64], src[127:64]}. + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 PUNPCKHQDQ expects Xsrc/m128, Xdst: %q", ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil + src, err := c.loadXVecOperand(ins.Args[0]) + if err != nil { + return true, false, err } - imm := uint64(ins.Args[0].Imm) & 0xff - idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } - src, err := c.loadX(ins.Args[1].Reg) + dstv, err := c.loadX(ins.Args[1].Reg) if err != nil { return true, false, err } - bc1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", bc1, src) - mask := fmt.Sprintf("<4 x i32> ", idx(0), idx(1), idx(2), idx(3)) + dstQ := c.newTmp() + srcQ := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", dstQ, dstv) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", srcQ, src) sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i32> %%%s, <4 x i32> zeroinitializer, %s\n", sh, bc1, mask) - bc2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", bc2, sh) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc2) + fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %%%s, <2 x i64> %%%s, <2 x i32> \n", sh, dstQ, srcQ) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", out, sh) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "PSHUFHW": - // PSHUFHW $imm, Xsrc, Xdst (shuffle high 4 words; low 4 words unchanged). - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSHUFHW expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil + case "VPUNPCKLDQ", "VPUNPCKHDQ", "VPUNPCKLQDQ", "VPUNPCKHQDQ": + if len(ins.Args) != 3 || ins.Args[2].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects src1, src2, dst: %q", op, ins.Raw) } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil + if _, ok := amd64ParseXReg(ins.Args[2].Reg); ok { + a, err := c.loadXVecOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + b, err := c.loadXVecOperand(ins.Args[1]) + if err != nil { + return true, false, err + } + laneType := "i64" + lanes := 2 + mask := "" + if op == "VPUNPCKLQDQ" { + mask = "" + } else if op == "VPUNPCKLDQ" { + laneType, lanes = "i32", 4 + mask = "" + } else if op == "VPUNPCKHDQ" { + laneType, lanes = "i32", 4 + mask = "" + } + aq := c.newTmp() + bq := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x %s>\n", aq, a, lanes, laneType) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <%d x %s>\n", bq, b, lanes, laneType) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x %s> %%%s, <%d x %s> %%%s, <%d x i32> %s\n", sh, lanes, laneType, bq, lanes, laneType, aq, lanes, mask) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x %s> %%%s to <16 x i8>\n", out, lanes, laneType, sh) + return true, false, c.storeX(ins.Args[2].Reg, "%"+out) } - imm := uint64(ins.Args[0].Imm) & 0xff - idx := func(k uint) uint64 { return (imm >> (2 * k)) & 3 } - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err + if _, ok := amd64ParseYReg(ins.Args[2].Reg); ok { + a, err := c.loadYVecOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + b, err := c.loadYVecOperand(ins.Args[1]) + if err != nil { + return true, false, err + } + laneType := "i64" + lanes := 4 + mask := "" + if op == "VPUNPCKLQDQ" { + mask = "" + } else if op == "VPUNPCKLDQ" { + laneType, lanes = "i32", 8 + mask = "" + } else if op == "VPUNPCKHDQ" { + laneType, lanes = "i32", 8 + mask = "" + } + aq := c.newTmp() + bq := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <%d x %s>\n", aq, a, lanes, laneType) + fmt.Fprintf(c.b, " %%%s = bitcast <32 x i8> %s to <%d x %s>\n", bq, b, lanes, laneType) + sh := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x %s> %%%s, <%d x %s> %%%s, <%d x i32> %s\n", sh, lanes, laneType, bq, lanes, laneType, aq, lanes, mask) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x %s> %%%s to <32 x i8>\n", out, lanes, laneType, sh) + return true, false, c.storeY(ins.Args[2].Reg, "%"+out) } - bc1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", bc1, src) - mask := fmt.Sprintf("<8 x i32> ", - 4+idx(0), 4+idx(1), 4+idx(2), 4+idx(3)) - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i16> %%%s, <8 x i16> zeroinitializer, %s\n", sh, bc1, mask) - bc2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %%%s to <16 x i8>\n", bc2, sh) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc2) + return false, false, nil case "SHUFPS": // SHUFPS $imm, Xsrc, Xdst @@ -2183,104 +1678,7 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, or) return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - case "MOVOU", "MOVOA": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("amd64 %s expects src, dst: %q", op, ins.Raw) - } - if ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - dst := ins.Args[1].Reg - switch ins.Args[0].Kind { - case OpReg: - v, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - return true, false, c.storeX(dst, v) - case OpMem: - addr, err := c.addrFromMem(ins.Args[0].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - return true, false, c.storeX(dst, "%"+ld) - case OpSym: - p, err := c.ptrFromSB(ins.Args[0].Sym) - if err != nil { - return true, false, err - } - ld := c.newTmp() - fmt.Fprintf(c.b, " %%%s = load <16 x i8>, ptr %s, align 1\n", ld, p) - return true, false, c.storeX(dst, "%"+ld) - default: - return true, false, fmt.Errorf("amd64 %s unsupported src: %q", op, ins.Raw) - } - } - - if ins.Args[0].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc for store form: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[0].Reg); !ok { - return false, false, nil - } - srcv, err := c.loadX(ins.Args[0].Reg) - if err != nil { - return true, false, err - } - switch ins.Args[1].Kind { - case OpMem: - addr, err := c.addrFromMem(ins.Args[1].Mem) - if err != nil { - return true, false, err - } - p := c.ptrFromAddrI64(addr) - fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", srcv, p) - return true, false, nil - case OpSym: - p, err := c.ptrFromSB(ins.Args[1].Sym) - if err != nil { - return true, false, err - } - fmt.Fprintf(c.b, " store <16 x i8> %s, ptr %s, align 1\n", srcv, p) - return true, false, nil - default: - return true, false, fmt.Errorf("amd64 %s unsupported dst: %q", op, ins.Raw) - } - - case "PXOR", "POR", "PAND", "PANDN": - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - src, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - t := c.newTmp() - if op == "PXOR" { - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", t, dstv, src) - } else if op == "POR" { - fmt.Fprintf(c.b, " %%%s = or <16 x i8> %s, %s\n", t, dstv, src) - } else if op == "PANDN" { - notv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <16 x i8> %s, %s\n", notv, dstv, llvmAllOnesI8Vec(16)) - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %%%s, %s\n", t, notv, src) - } else { - fmt.Fprintf(c.b, " %%%s = and <16 x i8> %s, %s\n", t, dstv, src) - } - return true, false, c.storeX(ins.Args[1].Reg, "%"+t) - - case "PADDL", "PADDD", "PADDQ", "PSUBB", "PSUBL": + case "PADDB", "PADDL", "PADDD", "PADDQ", "PSUBB", "PSUBL", "PMULLW": // Two-operand packed arithmetic: dst = dst (+|-) src. if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) @@ -2297,8 +1695,10 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, err } vecTy := "<4 x i32>" - if op == "PSUBB" { + if op == "PADDB" || op == "PSUBB" { vecTy = "<16 x i8>" + } else if op == "PMULLW" { + vecTy = "<8 x i16>" } else if op == "PADDQ" { vecTy = "<2 x i64>" } @@ -2313,17 +1713,116 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err bs = "%" + bt } x := c.newTmp() - if op == "PADDL" || op == "PADDD" || op == "PADDQ" { + if op == "PADDB" || op == "PADDL" || op == "PADDD" || op == "PADDQ" { fmt.Fprintf(c.b, " %%%s = add %s %s, %s\n", x, vecTy, bs, as) + } else if op == "PMULLW" { + fmt.Fprintf(c.b, " %%%s = mul %s %s, %s\n", x, vecTy, bs, as) } else { fmt.Fprintf(c.b, " %%%s = sub %s %s, %s\n", x, vecTy, bs, as) } if vecTy == "<16 x i8>" { return true, false, c.storeX(ins.Args[1].Reg, "%"+x) } - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecTy, x) - return true, false, c.storeX(ins.Args[1].Reg, "%"+out) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecTy, x) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) + + case "PSUBUSB": + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 PSUBUSB expects Xsrc, Xdst: %q", ins.Raw) + } + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return false, false, nil + } + src, err := c.loadXVecOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + dst, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return false, false, nil + } + underflow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp ult <16 x i8> %s, %s\n", underflow, dst, src) + difference := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub <16 x i8> %s, %s\n", difference, dst, src) + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <16 x i1> %%%s, <16 x i8> zeroinitializer, <16 x i8> %%%s\n", result, underflow, difference) + return true, false, c.storeX(ins.Args[1].Reg, "%"+result) + + case "PABSD": + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 PABSD expects Xsrc, Xdst: %q", ins.Raw) + } + if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { + return false, false, nil + } + src, err := c.loadXVecOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + values := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", values, src) + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt <4 x i32> %%%s, zeroinitializer\n", negative, values) + negated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub <4 x i32> zeroinitializer, %%%s\n", negated, values) + absolute := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select <4 x i1> %%%s, <4 x i32> %%%s, <4 x i32> %%%s\n", absolute, negative, negated, values) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, absolute) + return true, false, c.storeX(ins.Args[1].Reg, "%"+out) + + case "PMINUB", "PMINSB", "PMINUW", "PMINSW", "PMINUD", "PMINSD", + "PMAXUB", "PMAXSB", "PMAXUW", "PMAXSW", "PMAXUD", "PMAXSD": + // Two-operand packed integer min/max: dst = min|max(dst, src). + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("amd64 %s expects Xsrc/mem, Xdst: %q", op, ins.Raw) + } + dst, err := c.loadX(ins.Args[1].Reg) + if err != nil { + return false, false, nil + } + src, err := c.loadXVecOperand(ins.Args[0]) + if err != nil { + return true, false, err + } + vecTy := "<16 x i8>" + maskTy := "<16 x i1>" + if strings.HasSuffix(string(op), "W") { + vecTy = "<8 x i16>" + maskTy = "<8 x i1>" + } else if strings.HasSuffix(string(op), "D") { + vecTy = "<4 x i32>" + maskTy = "<4 x i1>" + } + srcv, dstv := src, dst + if vecTy != "<16 x i8>" { + srcBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", srcBits, src, vecTy) + srcv = "%" + srcBits + dstBits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", dstBits, dst, vecTy) + dstv = "%" + dstBits + } + pred := "slt" + if strings.Contains(string(op), "U") { + pred = "ult" + } + if strings.HasPrefix(string(op), "PMAX") { + pred = strings.Replace(pred, "lt", "gt", 1) + } + cmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp %s %s %s, %s\n", cmp, pred, vecTy, dstv, srcv) + sel := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select %s %%%s, %s %s, %s %s\n", sel, maskTy, cmp, vecTy, dstv, vecTy, srcv) + out := "%" + sel + if vecTy != "<16 x i8>" { + back := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", back, vecTy, sel) + out = "%" + back + } + return true, false, c.storeX(ins.Args[1].Reg, out) case "PSLLW", "PSRLW", "PSLLL", "PSRLL", "PSRAL": // Two-operand packed integer shift immediate. @@ -2374,9 +1873,9 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecTy, sh) return true, false, c.storeX(ins.Args[1].Reg, "%"+out) - case "PCMPEQL": + case "PCMPEQW", "PCMPEQL": if len(ins.Args) != 2 || ins.Args[0].Kind != OpReg || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PCMPEQL expects Xsrc, Xdst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 %s expects Xsrc, Xdst: %q", op, ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[0].Reg); !ok { return false, false, nil @@ -2392,123 +1891,27 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err if err != nil { return true, false, err } + laneType := "i32" + lanes := 4 + if op == "PCMPEQW" { + laneType = "i16" + lanes = 8 + } + vecType := fmt.Sprintf("<%d x %s>", lanes, laneType) as := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", as, src) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", as, src, vecType) bs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", bs, dstv) + fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to %s\n", bs, dstv, vecType) cmp := c.newTmp() - fmt.Fprintf(c.b, " %%%s = icmp eq <4 x i32> %%%s, %%%s\n", cmp, bs, as) + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, %%%s\n", cmp, vecType, bs, as) sext := c.newTmp() - fmt.Fprintf(c.b, " %%%s = sext <4 x i1> %%%s to <4 x i32>\n", sext, cmp) + fmt.Fprintf(c.b, " %%%s = sext <%d x i1> %%%s to %s\n", sext, lanes, cmp, vecType) out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, sext) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <16 x i8>\n", out, vecType, sext) return true, false, c.storeX(ins.Args[1].Reg, "%"+out) case "VPCLMULQDQ": - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPCLMULQDQ expects $imm, Zsrc, Zdst, Zout: %q", ins.Raw) - } - if _, ok := amd64ParseZReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseZReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseZReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - imm := ins.Args[0].Imm & 0xff - src, err := c.loadZ(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - dstv, err := c.loadZ(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bd := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", bd, dstv) - bs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <64 x i8> %s to <8 x i64>\n", bs, src) - lanes := make([]string, 0, 4) - for i := 0; i < 4; i++ { - dlane := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i64> %%%s, <8 x i64> zeroinitializer, <2 x i32> \n", dlane, bd, i*2, i*2+1) - slane := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <8 x i64> %%%s, <8 x i64> zeroinitializer, <2 x i32> \n", slane, bs, i*2, i*2+1) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %%%s, <2 x i64> %%%s, i8 %d)\n", call, dlane, slane, imm) - lanes = append(lanes, "%"+call) - } - merged0 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <4 x i32> \n", merged0, lanes[0], lanes[1]) - merged1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <2 x i64> %s, <2 x i64> %s, <4 x i32> \n", merged1, lanes[2], lanes[3]) - merged := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <4 x i64> %%%s, <4 x i64> %%%s, <8 x i32> \n", merged, merged0, merged1) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i64> %%%s to <64 x i8>\n", out, merged) - return true, false, c.storeZ(ins.Args[3].Reg, "%"+out) - - case "VPTERNLOGD": - if len(ins.Args) != 4 || ins.Args[0].Kind != OpImm || ins.Args[3].Kind != OpReg { - return true, false, fmt.Errorf("amd64 VPTERNLOGD expects $imm, src1, src2, dst: %q", ins.Raw) - } - if _, ok := amd64ParseYReg(ins.Args[3].Reg); ok { - if ins.Args[1].Kind == OpReg { - if _, ok := amd64ParseYReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - } - if ins.Args[2].Kind == OpReg { - if _, ok := amd64ParseYReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - } - if ins.Args[0].Imm != 0x96 { - return true, false, fmt.Errorf("amd64 VPTERNLOGD only supports imm 0x96 for now: %q", ins.Raw) - } - a, err := c.loadYVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - b, err := c.loadYVecOperand(ins.Args[2]) - if err != nil { - return true, false, err - } - dstv, err := c.loadY(ins.Args[3].Reg) - if err != nil { - return true, false, err - } - x1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %s, %s\n", x1, dstv, a) - x2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <32 x i8> %%%s, %s\n", x2, x1, b) - return true, false, c.storeY(ins.Args[3].Reg, "%"+x2) - } - if _, ok := amd64ParseZReg(ins.Args[3].Reg); !ok { - return false, false, nil - } - if ins.Args[0].Imm != 0x96 { - return true, false, fmt.Errorf("amd64 VPTERNLOGD only supports imm 0x96 for now: %q", ins.Raw) - } - a, err := c.loadZVecOperand(ins.Args[1]) - if err != nil { - return true, false, err - } - b, err := c.loadZVecOperand(ins.Args[2]) - if err != nil { - return true, false, err - } - dstv, err := c.loadZ(ins.Args[3].Reg) - if err != nil { - return true, false, err - } - x1 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <64 x i8> %s, %s\n", x1, dstv, a) - x2 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = xor <64 x i8> %%%s, %s\n", x2, x1, b) - return true, false, c.storeZ(ins.Args[3].Reg, "%"+x2) + return c.lowerPackedCarrylessMultiply(rawOp, ins) case "VEXTRACTF32X4": if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { @@ -2530,34 +1933,7 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.storeX(ins.Args[2].Reg, "%"+out) case "PCLMULQDQ": - // PCLMULQDQ $imm, Xsrc, Xdst => Xdst = pclmul(Xdst, Xsrc, imm) - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PCLMULQDQ expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - imm := ins.Args[0].Imm & 0xff - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - dstv, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bd := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bd, dstv) - bs := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bs, src) - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %%%s, <2 x i64> %%%s, i8 %d)\n", call, bd, bs, imm) - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", bc, call) - return true, false, c.storeX(ins.Args[2].Reg, "%"+bc) + return c.lowerPackedCarrylessMultiply(rawOp, ins) case "PCMPEQB", "PCMPGTB": if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { @@ -2612,128 +1988,6 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err fmt.Fprintf(c.b, " %%%s = zext i32 %%%s to i64\n", z, call) return true, false, c.storeReg(ins.Args[1].Reg, "%"+z) - case "PSHUFB": - // PSHUFB Xmask, Xdst - if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSHUFB expects Xmask, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - mask, err := c.loadXVecOperand(ins.Args[0]) - if err != nil { - return true, false, err - } - dst, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - call := c.newTmp() - fmt.Fprintf(c.b, " %%%s = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %s, <16 x i8> %s)\n", call, dst, mask) - return true, false, c.storeX(ins.Args[1].Reg, "%"+call) - - case "PINSRQ": - // PINSRQ $imm, src64, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRQ expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 1 - src, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bc, dst) - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <2 x i64> %%%s, i64 %s, i32 %d\n", insv, bc, src, idx) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", out, insv) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) - - case "PINSRD": - // PINSRD $imm, src32, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRD expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 3 - src64, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - src32 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", src32, src64) - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <4 x i32>\n", bc, dst) - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <4 x i32> %%%s, i32 %%%s, i32 %d\n", insv, bc, src32, idx) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <16 x i8>\n", out, insv) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) - - case "PINSRW": - // PINSRW $imm, src16, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRW expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 7 - src64, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - src16 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i16\n", src16, src64) - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - bc := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <8 x i16>\n", bc, dst) - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <8 x i16> %%%s, i16 %%%s, i32 %d\n", insv, bc, src16, idx) - out := c.newTmp() - fmt.Fprintf(c.b, " %%%s = bitcast <8 x i16> %%%s to <16 x i8>\n", out, insv) - return true, false, c.storeX(ins.Args[2].Reg, "%"+out) - - case "PINSRB": - // PINSRB $imm, src8, Xdst - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PINSRB expects $imm, src, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - idx := ins.Args[0].Imm & 15 - src64, err := c.evalI64(ins.Args[1]) - if err != nil { - return true, false, err - } - src8 := c.newTmp() - fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i8\n", src8, src64) - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - insv := c.newTmp() - fmt.Fprintf(c.b, " %%%s = insertelement <16 x i8> %s, i8 %%%s, i32 %d\n", insv, dst, src8, idx) - return true, false, c.storeX(ins.Args[2].Reg, "%"+insv) - case "PEXTRB": // PEXTRB $imm, Xsrc, dst if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { @@ -2806,84 +2060,27 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err } case "PALIGNR": - // PALIGNR $imm, Xsrc, Xdst ; dst = alignr(dst, src, imm) - if len(ins.Args) != 3 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg || ins.Args[2].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PALIGNR expects $imm, Xsrc, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - if _, ok := amd64ParseXReg(ins.Args[2].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 { - n = 0 - } - if n > 255 { - n = 255 - } - src, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - dst, err := c.loadX(ins.Args[2].Reg) - if err != nil { - return true, false, err - } - sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> %s, <16 x i32> %s\n", sh, dst, src, llvmAlignRightBytesMask(n)) - return true, false, c.storeX(ins.Args[2].Reg, "%"+sh) + return c.lowerPackedAlignRight(rawOp, ins) - case "PSRLDQ": - // PSRLDQ $imm, Xdst - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSRLDQ expects $imm, Xdst: %q", ins.Raw) - } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil - } - n := ins.Args[0].Imm - if n < 0 || n > 16 { - return true, false, fmt.Errorf("amd64 PSRLDQ invalid imm %d: %q", n, ins.Raw) - } - v, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - shuf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <16 x i32> %s\n", shuf, v, llvmShiftRightBytesMask(n)) - return true, false, c.storeX(ins.Args[1].Reg, "%"+shuf) + case "PSRLDQ", "PSLLDQ": + return c.lowerPackedByteShift(rawOp, ins) - case "PSLLDQ": - // PSLLDQ $imm, Xdst + case "PSRLQ", "PSLLQ": + // PSR/LLQ $imm, Xdst (shift each 64-bit lane). Counts at or above + // the lane width clear the destination rather than wrapping modulo 64. if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSLLDQ expects $imm, Xdst: %q", ins.Raw) + return true, false, fmt.Errorf("amd64 %s expects $imm, Xdst: %q", op, ins.Raw) } if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { return false, false, nil } n := ins.Args[0].Imm - if n < 0 || n > 16 { - return true, false, fmt.Errorf("amd64 PSLLDQ invalid imm %d: %q", n, ins.Raw) - } - v, err := c.loadX(ins.Args[1].Reg) - if err != nil { - return true, false, err - } - shuf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <16 x i32> %s\n", shuf, v, llvmShiftLeftBytesMask(n)) - return true, false, c.storeX(ins.Args[1].Reg, "%"+shuf) - - case "PSRLQ": - // PSRLQ $imm, Xdst (shift each 64-bit lane right) - if len(ins.Args) != 2 || ins.Args[0].Kind != OpImm || ins.Args[1].Kind != OpReg { - return true, false, fmt.Errorf("amd64 PSRLQ expects $imm, Xdst: %q", ins.Raw) + if n < 0 { + return true, false, fmt.Errorf("amd64 %s shift count must be non-negative: %q", op, ins.Raw) } - if _, ok := amd64ParseXReg(ins.Args[1].Reg); !ok { - return false, false, nil + if n >= 64 { + return true, false, c.storeX(ins.Args[1].Reg, "zeroinitializer") } - n := ins.Args[0].Imm & 63 v, err := c.loadX(ins.Args[1].Reg) if err != nil { return true, false, err @@ -2891,7 +2088,11 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err bc := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast <16 x i8> %s to <2 x i64>\n", bc, v) sh := c.newTmp() - fmt.Fprintf(c.b, " %%%s = lshr <2 x i64> %%%s, \n", sh, bc, n, n) + shiftOp := "lshr" + if op == "PSLLQ" { + shiftOp = "shl" + } + fmt.Fprintf(c.b, " %%%s = %s <2 x i64> %%%s, \n", sh, shiftOp, bc, n, n) back := c.newTmp() fmt.Fprintf(c.b, " %%%s = bitcast <2 x i64> %%%s to <16 x i8>\n", back, sh) return true, false, c.storeX(ins.Args[1].Reg, "%"+back) @@ -2935,6 +2136,66 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err return false, false, nil } +func (c *amd64Ctx) validateMOVDAliasForm(ins Instr) error { + if len(ins.Args) != 2 { + return fmt.Errorf("%s MOVD expects 2 operands: %q", c.goarch, ins.Raw) + } + src, dst := ins.Args[0], ins.Args[1] + srcMem, dstMem := isAMD64MemoryOperand(src), isAMD64MemoryOperand(dst) + srcX, dstX := isAMD64MOVDXRegister(src), isAMD64MOVDXRegister(dst) + srcM, dstM := isAMD64MOVDMRegister(src), isAMD64MOVDMRegister(dst) + srcGP, dstGP := c.isGoYrlRegister(src), c.isGoYrlRegister(dst) + + valid := false + if c.goarch == "386" { + // In 32-bit mode AMOVQ has only its memory/MMX/XMM rows. Go rejects + // every GP and immediate form as an illegal 64-bit instruction. + valid = (dstMem && (srcX || srcM)) || + (dstX && (srcMem || srcX)) || + (dstM && (srcMem || srcX)) + } else { + valid = (dstGP && (src.Kind == OpImm || srcGP || srcMem || srcX || srcM)) || + (dstMem && (src.Kind == OpImm || srcGP || srcX || srcM)) || + (dstX && (srcGP || srcMem || srcX)) || + (dstM && (srcGP || srcMem || srcX || srcM)) + } + if !valid { + return fmt.Errorf("%s MOVD operands are outside Go 1.27's MOVD/ymovq forms: %q", c.goarch, ins.Raw) + } + return nil +} + +func isAMD64MOVDXRegister(op Operand) bool { + if op.Kind != OpReg { + return false + } + _, ok := amd64ParseXReg(op.Reg) + return ok +} + +func isAMD64MOVDMRegister(op Operand) bool { + if op.Kind != OpReg { + return false + } + _, ok := amd64ParseMReg(op.Reg) + return ok +} + +func (c *amd64Ctx) isGoYrlRegister(op Operand) bool { + if op.Kind != OpReg || !isAMD64YrlRegister(op.Reg) { + return false + } + if c.goarch != "386" { + return true + } + switch op.Reg { + case AX, BX, CX, DX, SP, BP, SI, DI: + return true + default: + return false + } +} + func llvmShiftRightBytesMask(n int64) string { // shufflevector mask for right shift by n bytes. // Use second vector's element 0 (index 16) as the "zero" source. @@ -2975,29 +2236,6 @@ func llvmShiftLeftBytesMask(n int64) string { return sb.String() } -func llvmAlignRightBytesMask(n int64) string { - // shufflevector mask for alignr(dst, src, n): - // result bytes are selected from concatenation [dst, src], right shifted by n. - var sb strings.Builder - sb.WriteString("<") - for i := 0; i < 16; i++ { - if i != 0 { - sb.WriteString(", ") - } - idx := int64(i) + n - switch { - case idx < 16: - fmt.Fprintf(&sb, "i32 %d", idx) - case idx < 32: - fmt.Fprintf(&sb, "i32 %d", idx) - default: - sb.WriteString("i32 16") - } - } - sb.WriteString(">") - return sb.String() -} - func llvmRepeatI8Mask(chunk, width int) string { if chunk <= 0 { chunk = 1 @@ -3014,6 +2252,35 @@ func llvmRepeatI8Mask(chunk, width int) string { return sb.String() } +func llvmSplatInteger(lanes, bits int, value uint64) string { + var b strings.Builder + b.WriteByte('<') + for lane := 0; lane < lanes; lane++ { + if lane != 0 { + b.WriteString(", ") + } + fmt.Fprintf(&b, "i%d %d", bits, value) + } + b.WriteByte('>') + return b.String() +} + +func amd64SplatInteger(c *amd64Ctx, lanes, bits int, value string) string { + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> poison, i%d %s, i32 0\n", seed, lanes, bits, bits, value) + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i%d> %%%s, <%d x i%d> poison, <%d x i32> zeroinitializer\n", splat, lanes, bits, seed, lanes, bits, lanes) + return "%" + splat +} + +func amd64SplatI1(c *amd64Ctx, lanes int, value string) string { + seed := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i1> poison, i1 %s, i32 0\n", seed, lanes, value) + splat := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <%d x i1> %%%s, <%d x i1> poison, <%d x i32> zeroinitializer\n", splat, lanes, seed, lanes, lanes) + return "%" + splat +} + func llvmAllOnesI8Vec(n int) string { if n <= 0 { return "<>" @@ -3035,6 +2302,16 @@ func isAMD64ZReg(r Reg) bool { return ok } +func isAMD64XReg(r Reg) bool { + _, ok := amd64ParseXReg(r) + return ok +} + +func isAMD64YReg(r Reg) bool { + _, ok := amd64ParseYReg(r) + return ok +} + func llvmI32RangeMask(start int, n int) string { // Build . if n <= 0 { @@ -3074,6 +2351,34 @@ func amd64MaskBitI1(c *amd64Ctx, mask string, idx int) string { return "%" + bit } +func amd64ApplyI32LaneMask(c *amd64Ctx, lanes int, computed, old, mask string, zeroing bool) string { + return amd64ApplyIntegerLaneMask(c, lanes, 32, computed, old, mask, zeroing) +} + +func amd64ApplyIntegerLaneMask(c *amd64Ctx, lanes, bits int, computed, old, mask string, zeroing bool) string { + result := old + if zeroing { + result = "zeroinitializer" + } + for lane := 0; lane < lanes; lane++ { + bit := amd64MaskBitI1(c, mask, lane) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", value, lanes, bits, computed, lane) + fallback := "0" + if !zeroing { + oldValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 %d\n", oldValue, lanes, bits, old, lane) + fallback = "%" + oldValue + } + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %%%s, i%d %s\n", selected, bit, bits, value, bits, fallback) + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %%%s, i32 %d\n", inserted, lanes, bits, result, bits, selected, lane) + result = "%" + inserted + } + return result +} + func amd64PackI1x8ToI64(c *amd64Ctx, pred string) string { acc := "0" for i := 0; i < 8; i++ { diff --git a/amd64_lower_vecflags.go b/amd64_lower_vecflags.go new file mode 100644 index 000000000..94e8e6f23 --- /dev/null +++ b/amd64_lower_vecflags.go @@ -0,0 +1,142 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64VectorTestSpec struct { + maxBytes int + vector bool + signLaneBits int +} + +// amd64VectorTestSpecs models the complete Go 1.27 PTEST/VPTEST/VTEST grammar. +// PTEST uses the legacy yxm_q4 X/m128 row. The VEX instructions share +// _yvptest's X/m128,X and Y/m256,Y rows. VTESTPD and VTESTPS consider only the +// sign bit in each floating-point lane; PTEST and VPTEST consider every bit. +var amd64VectorTestSpecs = map[Op]amd64VectorTestSpec{ + "PTEST": {maxBytes: 16}, + "VPTEST": {maxBytes: 32, vector: true}, + "VTESTPD": {maxBytes: 32, vector: true, signLaneBits: 64}, + "VTESTPS": {maxBytes: 32, vector: true, signLaneBits: 32}, +} + +func (c *amd64Ctx) lowerVectorTest(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64VectorTestSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s has no suffixed form in Go 1.27: %q", c.goarch, baseOp, ins.Raw) + } + if len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return true, false, fmt.Errorf("%s %s expects vector/memory source and vector destination: %q", c.goarch, baseOp, ins.Raw) + } + + source, destination := ins.Args[0], ins.Args[1] + byteWidth := amd64VectorByteWidth(destination.Reg) + if (byteWidth != 16 && byteWidth != 32) || byteWidth > spec.maxBytes { + return true, false, fmt.Errorf("%s %s destination width is outside its Go 1.27 table: %q", c.goarch, baseOp, ins.Raw) + } + if spec.vector { + // Go's VEX classes admit X/Y0-15 even in 386 mode. They also admit + // extended address registers there, so do not apply the legacy 386 + // register filter to VEX memory operands. + if !amd64VEXVectorRegister(destination, byteWidth) { + return true, false, fmt.Errorf("%s %s destination is outside Go's VEX X/Y class: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 || !c.isGoLegacyXReg(destination.Reg) { + return true, false, fmt.Errorf("%s %s destination is outside Go's legacy X class: %q", c.goarch, baseOp, ins.Raw) + } + + if source.Kind == OpReg { + if spec.vector { + if !amd64VEXVectorRegister(source, byteWidth) { + return true, false, fmt.Errorf("%s %s source must match the destination's VEX width: %q", c.goarch, baseOp, ins.Raw) + } + } else if byteWidth != 16 || !c.isGoLegacyXReg(source.Reg) { + return true, false, fmt.Errorf("%s %s source is outside Go's legacy X class: %q", c.goarch, baseOp, ins.Raw) + } + } else if !isAMD64MemoryOperand(source) { + return true, false, fmt.Errorf("%s %s source must be a same-width vector register or memory: %q", c.goarch, baseOp, ins.Raw) + } + if !spec.vector && source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, c.goarch) { + return true, false, fmt.Errorf("%s %s source uses an out-of-range legacy address register: %q", c.goarch, baseOp, ins.Raw) + } + + sourceBytes, err := c.loadPackedCompareBytes(source, byteWidth) + if err != nil { + return true, false, err + } + destinationBytes, err := c.loadPackedCompareBytes(destination, byteWidth) + if err != nil { + return true, false, err + } + + intersection := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %s, %s\n", intersection, byteWidth, destinationBytes, sourceBytes) + invertedDestination := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor <%d x i8> %s, %s\n", invertedDestination, byteWidth, destinationBytes, llvmAllOnesI8Vec(byteWidth)) + outsideDestination := c.newTmp() + fmt.Fprintf(c.b, " %%%s = and <%d x i8> %%%s, %s\n", outsideDestination, byteWidth, invertedDestination, sourceBytes) + + var zero, carry string + if spec.signLaneBits == 0 { + // Avoid an i256 scalar: LLVM 22 aborts while legalizing that type for + // i386. Reducing individual bytes compiles on every supported target. + zero = c.emitVectorBytesAreZero(byteWidth, "%"+intersection) + carry = c.emitVectorBytesAreZero(byteWidth, "%"+outsideDestination) + } else { + zero = c.emitNoSignBits(byteWidth, spec.signLaneBits, "%"+intersection) + carry = c.emitNoSignBits(byteWidth, spec.signLaneBits, "%"+outsideDestination) + } + + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", zero, c.flagsZSlot) + fmt.Fprintf(c.b, " store i1 %s, ptr %s\n", carry, c.flagsCFSlot) + // All four instructions compute CF/ZF and clear AF/OF/SF/PF. + c.clearX86StatusFlags(x86StatusAll &^ (x86StatusCF | x86StatusZF)) + return true, false, nil +} + +func (c *amd64Ctx) emitVectorBytesAreZero(byteWidth int, bytes string) string { + // Extract bytes directly. LLVM 22's i386 backend also aborts on some + // otherwise-valid bitcasts between illegal 256-bit vector types. + combined := "0" + for index := 0; index < byteWidth; index++ { + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", element, byteWidth, bytes, index) + merged := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i8 %s, %%%s\n", merged, combined, element) + combined = "%" + merged + } + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq i8 %s, 0\n", zero, combined) + return "%" + zero +} + +func (c *amd64Ctx) emitNoSignBits(byteWidth, laneBits int, bytes string) string { + lanes := byteWidth * 8 / laneBits + any := "false" + for lane := 0; lane < lanes; lane++ { + // X86 is little-endian, so each lane's sign bit is the high bit of + // its final byte. Testing that byte avoids illegal 256-bit vector + // bitcasts in LLVM 22's i386 legalizer. + index := (lane+1)*(laneBits/8) - 1 + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i8> %s, i32 %d\n", element, byteWidth, bytes, index) + negative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp slt i8 %%%s, 0\n", negative, element) + combined := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i1 %s, %%%s\n", combined, any, negative) + any = "%" + combined + } + none := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", none, any) + return "%" + none +} diff --git a/amd64_lower_vecflags_test.go b/amd64_lower_vecflags_test.go new file mode 100644 index 000000000..1710a63d4 --- /dev/null +++ b/amd64_lower_vecflags_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64VectorTestGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64VectorTestSpec{ + "PTEST": {maxBytes: 16}, + "VPTEST": {maxBytes: 32, vector: true}, + "VTESTPD": {maxBytes: 32, vector: true, signLaneBits: 64}, + "VTESTPS": {maxBytes: 32, vector: true, signLaneBits: 32}, + } + if len(amd64VectorTestSpecs) != len(expected) { + t.Fatalf("vector-test grammar has %d entries, want %d", len(amd64VectorTestSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64VectorTestSpecs[op]; !ok { + t.Errorf("vector-test grammar omitted %s", op) + } else if got != want { + t.Errorf("vector-test grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86VectorTestCompleteGoFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + legacyLast := 15 + if target.goarch == "386" { + legacyLast = 7 + } + source := fmt.Sprintf(`TEXT vectortestforms(SB),$0-0 + PTEST X0, X%d + PTEST 0(AX), X%d + VPTEST X0, X15 + VPTEST 16(R9), X15 + VPTEST Y0, Y15 + VPTEST 32(R9), Y15 + VTESTPD X0, X15 + VTESTPD 48(R9), X15 + VTESTPD Y0, Y15 + VTESTPD 64(R9), Y15 + VTESTPS X0, X15 + VTESTPS 80(R9), X15 + VTESTPS Y0, Y15 + VTESTPS 96(R9), Y15 + RET +`, legacyLast, legacyLast) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "vectortestforms": {Name: "vectortestforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"<16 x i8>", "<32 x i8>", "store i1"} { + if !strings.Contains(ir, want) { + t.Fatalf("vector-test lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "vector-test-"+target.name+".ll", "vector-test-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86VectorTestRejectsFormsOutsideGoTables(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "PTEST Y0, Y1"}, + {goarch: "amd64", instruction: "PTEST X0, 0(AX)"}, + {goarch: "amd64", instruction: "PTEST X16, X0"}, + {goarch: "amd64", instruction: "PTEST.Z X0, X1"}, + {goarch: "386", instruction: "PTEST X8, X0"}, + {goarch: "386", instruction: "PTEST 0(R9), X0"}, + {goarch: "amd64", instruction: "VPTEST X0, Y1"}, + {goarch: "amd64", instruction: "VPTEST Z0, Z1"}, + {goarch: "amd64", instruction: "VPTEST X16, X0"}, + {goarch: "amd64", instruction: "VPTEST.Z X0, X1"}, + {goarch: "amd64", instruction: "VTESTPD X0, Y1"}, + {goarch: "amd64", instruction: "VTESTPD Z0, Z1"}, + {goarch: "amd64", instruction: "VTESTPS X16, X0"}, + {goarch: "amd64", instruction: "VTESTPS X0, X1, X2"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside the Go 1.27 vector-test tables", test.instruction) + } + }) + } +} + +func TestAMD64VectorTestRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vectortestsemantics(SB),$0-24 + MOVQ out+0(FP), CX + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + MOVUPS 0(AX), X0 + MOVUPS 0(BX), X1 + MOVB $0x7f, DL + ADDB $1, DL + PTEST X0, X1 + SETEQ 0(CX) + SETCS 1(CX) + SETMI 2(CX) + SETOS 3(CX) + SETPS 4(CX) + + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + VMOVDQU 32(BX), Y1 + XORB DL, DL + VPTEST 32(AX), Y1 + SETEQ 5(CX) + SETCS 6(CX) + SETMI 7(CX) + SETOS 8(CX) + SETPS 9(CX) + + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + MOVUPS 64(BX), X1 + VTESTPS 64(AX), X1 + SETEQ 10(CX) + SETCS 11(CX) + SETMI 12(CX) + SETOS 13(CX) + SETPS 14(CX) + + MOVQ source+8(FP), AX + MOVQ destination+16(FP), BX + VMOVDQU 96(BX), Y1 + VTESTPD 96(AX), Y1 + SETEQ 15(CX) + SETCS 16(CX) + SETMI 17(CX) + SETOS 18(CX) + SETPS 19(CX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"vectortestsemantics": { + Name: "vectortestsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void vectortestsemantics(uint8_t *, const uint8_t *, const uint8_t *); +int main(void) { + uint8_t source[128] = {0}, destination[128] = {0}, out[20] = {0}; + source[0] = 1; + source[32] = 1; destination[32] = 1; + source[64] = 1; destination[64] = 1; + source[103] = 0x80; + vectortestsemantics(out, source, destination); + const uint8_t want[20] = { + 1, 0, 0, 0, 0, + 0, 1, 0, 0, 0, + 1, 1, 0, 0, 0, + 1, 0, 0, 0, 0, + }; + for (int i = 0; i < 20; i++) if (out[i] != want[i]) return 10 + i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vector_test_semantics", triple, ir, mainC, runPrefix) + + // Exercise the raw-byte path with the same independently checked flag + // oracle. Keep the displacements and operand order identical to the named + // Go source above, including both 128-bit and 256-bit memory forms. + rawSource := strings.NewReplacer( + "PTEST X0, X1", "BYTE $0x66\n\tBYTE $0x0f\n\tBYTE $0x38\n\tBYTE $0x17\n\tBYTE $0xc8", + "VPTEST 32(AX), Y1", "BYTE $0xc4\n\tBYTE $0xe2\n\tBYTE $0x7d\n\tBYTE $0x17\n\tBYTE $0x48\n\tBYTE $0x20", + "VTESTPS 64(AX), X1", "BYTE $0xc4\n\tBYTE $0xe2\n\tBYTE $0x79\n\tBYTE $0x0e\n\tBYTE $0x48\n\tBYTE $0x40", + "VTESTPD 96(AX), Y1", "BYTE $0xc4\n\tBYTE $0xe2\n\tBYTE $0x7d\n\tBYTE $0x0f\n\tBYTE $0x48\n\tBYTE $0x60", + ).Replace(source) + requireX86GoAssemblerResult(t, "amd64", rawSource, true) + rawFile, err := Parse(ArchAMD64, rawSource) + if err != nil { + t.Fatal(err) + } + rawIR, err := Translate(rawFile, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"vectortestsemantics": { + Name: "vectortestsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_vector_test_semantics", triple, rawIR, mainC, runPrefix) +} diff --git a/amd64_lower_vextractps_test.go b/amd64_lower_vextractps_test.go new file mode 100644 index 000000000..6a79ec298 --- /dev/null +++ b/amd64_lower_vextractps_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +func TestAMD64PackedScalarExtractGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64PackedScalarExtractSpecs)) + for op := range amd64PackedScalarExtractSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "EXTRACTPS", "PEXTRB", "PEXTRD", "PEXTRQ", "PEXTRW", + "VEXTRACTPS", "VPEXTRB", "VPEXTRD", "VPEXTRQ", "VPEXTRW", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("packed scalar-extract grammar opcodes = %v, want %v", got, want) + } +} + +func TestTranslateX86ExtractPSCompleteGo127Forms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + highX, highGP := "X15", "R15" + if target.goarch == "386" { + highX, highGP = "X7", "DI" + } + source := "TEXT extractpsforms(SB),$0-0\n" + + "\tEXTRACTPS $0, X0, AX\n" + + "\tEXTRACTPS $3, " + highX + ", (BX)\n" + + "\tEXTRACTPS $2, " + highX + ", " + highGP + "\n" + + "\tVEXTRACTPS $-128, X0, AX\n" + + "\tVPEXTRQ $0, X0, (BX)\n" + + "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"extractpsforms": {Name: "extractpsforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "extractps-"+target.name+".ll", "extractps-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86ExtractPSRejectsFormsOutsideGo127Table(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "EXTRACTPS $-1, X0, AX"}, + {goarch: "amd64", instruction: "EXTRACTPS $4, X0, AX"}, + {goarch: "amd64", instruction: "EXTRACTPS $0, X16, AX"}, + {goarch: "amd64", instruction: "EXTRACTPS $0, X0, M0"}, + {goarch: "386", instruction: "EXTRACTPS $0, X8, AX"}, + {goarch: "386", instruction: "EXTRACTPS $0, X0, R8"}, + {goarch: "386", instruction: "PEXTRQ $0, X0, AX"}, + } { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's scalar-extract tables", test.instruction) + } + }) + } +} + +func TestAMD64ExtractPSRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT extractpssemantics(SB),NOSPLIT,$0-16 + MOVQ in+0(FP), AX + MOVQ out+8(FP), BX + MOVUPS (AX), X0 + EXTRACTPS $0, X0, CX + MOVL CX, 0(BX) + EXTRACTPS $3, X0, 4(BX) + VEXTRACTPS $2, X0, DX + MOVL DX, 8(BX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"extractpssemantics": { + Name: "extractpssemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void extractpssemantics(void *, void *); +int main(void) { + uint32_t in[4] = {0x3f800001,0x40000002,0x40400003,0x40800004}, out[3] = {0}; + extractpssemantics(in, out); + return out[0] == in[0] && out[1] == in[3] && out[2] == in[2] ? 0 : 10; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "extractps_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateAMD64VExtractPSCompleteGo127Forms(t *testing.T) { + const source = `TEXT vextractpsforms(SB),$0-0 + VEXTRACTPS $-128, X0, AX + VEXTRACTPS $255, X1, (AX) + VEXTRACTPS $255, X31, R15 + RET +` + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: target.triple, + Sigs: map[string]FuncSig{"vextractpsforms": {Name: "vextractpsforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "vextractps-"+target.name+".ll", "vextractps-"+target.name+".o", ir) + }) + } +} + +func TestTranslateAMD64VExtractPSRejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VEXTRACTPS $-129, X1, AX", + "VEXTRACTPS $-1, X31, AX", + "VEXTRACTPS $256, X1, AX", + "VEXTRACTPS $1, Y1, AX", + "VEXTRACTPS $1, X1, X2", + } { + t.Run(instruction, func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvextractps table", instruction) + } + }) + } +} diff --git a/amd64_lower_vinserti128_test.go b/amd64_lower_vinserti128_test.go new file mode 100644 index 000000000..86a065293 --- /dev/null +++ b/amd64_lower_vinserti128_test.go @@ -0,0 +1,161 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestTranslateX86VINSERTI128CompleteGo127Forms(t *testing.T) { + const source = `TEXT vinserti128_forms(SB),$0-0 + VINSERTI128 $0, X0, Y1, Y2 + VINSERTI128 $255, 8(BX), Y15, Y15 + VINSERTI128 $1, X15, Y0, Y1 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "darwin-amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "vinserti128_forms": {Name: "vinserti128_forms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "vinserti128-"+target.name+".ll", "vinserti128-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86VINSERTI128RejectsFormsOutsideGo127Table(t *testing.T) { + for _, instruction := range []string{ + "VINSERTI128 X0, Y1, Y2", + "VINSERTI128 $0, X0, Y1", + "VINSERTI128 $-1, X0, Y1, Y2", + "VINSERTI128 $256, X0, Y1, Y2", + "VINSERTI128 AX, X0, Y1, Y2", + "VINSERTI128 $0, Y0, Y1, Y2", + "VINSERTI128 $0, X16, Y1, Y2", + "VINSERTI128 $0, X0, X1, Y2", + "VINSERTI128 $0, X0, Y16, Y2", + "VINSERTI128 $0, X0, Y1, X2", + "VINSERTI128 $0, X0, Y1, Y16", + "VINSERTI128 $0, X0, Y1, 8(BX)", + "VINSERTI128.Z $0, X0, Y1, Y2", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VINSERTI128 table", instruction) + } + }) + } + const source386 = "TEXT bad(SB),$0-0\n\tVINSERTI128 $0, X0, Y1, Y2\n\tRET\n" + requireX86GoAssemblerResult(t, "386", source386, false) + file, err := Parse(ArchAMD64, source386) + if err == nil { + if _, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatal("Translate accepted 386 VINSERTI128 despite Go 1.27's frontend operand limit") + } + } +} + +func TestAMD64VINSERTI128RuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vinserti128Semantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ narrow+8(FP), BX + MOVQ wide+16(FP), CX + VMOVDQU64 (BX), X0 + VMOVDQU64 (CX), Y1 + VINSERTI128 $0, X0, Y1, Y2 + VMOVDQU64 Y2, 0(AX) + VINSERTI128 $255, X0, Y1, Y3 + VMOVDQU64 Y3, 32(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "vinserti128Semantics": { + Name: "vinserti128Semantics", + Args: []LLVMType{Ptr, Ptr, Ptr}, + Ret: Void, + Frame: frame, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void vinserti128Semantics(uint64_t *, const uint64_t *, const uint64_t *); +int main(void) { + const uint64_t narrow[2] = {10, 11}; + const uint64_t wide[4] = {20, 21, 22, 23}; + uint64_t out[8] = {0}; + const uint64_t want[8] = {10, 11, 22, 23, 20, 21, 10, 11}; + vinserti128Semantics(out, narrow, wide); + for (int i = 0; i < 8; i++) if (out[i] != want[i]) return i + 1; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vinserti128", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_vmov_fp_test.go b/amd64_lower_vmov_fp_test.go new file mode 100644 index 000000000..32a0963f2 --- /dev/null +++ b/amd64_lower_vmov_fp_test.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "testing" +) + +func TestX86VEXPackedMoveAggregateFPForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + pointerBytes := 8 + if target.arch == "386" { + pointerBytes = 4 + } + for _, op := range []string{"VMOVDQA", "VMOVDQU"} { + for _, width := range []struct { + name, reg string + bytes int + }{{"x", "X0", 16}, {"y", "Y0", 32}} { + t.Run(op+"/"+width.name, func(t *testing.T) { + name := "packedFrame" + var slots []FrameSlot + args := []LLVMType{Ptr} + slots = append(slots, FrameSlot{Offset: 0, Type: Ptr, Index: 0, Field: -1}) + for i := 0; i < width.bytes/4; i++ { + args = append(args, I32) + slots = append(slots, FrameSlot{ + Offset: int64(pointerBytes + i*4), + Type: I32, + Index: i + 1, + Field: -1, + }) + } + mov := "MOVQ" + if target.arch == "386" { + mov = "MOVL" + } + source := fmt.Sprintf("TEXT %s(SB),4,$0-%d\n%s out+0(FP),AX\n%s key+%d(FP),%s\n%s %s,(AX)\n%s (AX),%s\n%s %s,key+%d(FP)\nRET\n", + name, pointerBytes+width.bytes, mov, op, pointerBytes, width.reg, + op, width.reg, op, width.reg, op, width.reg, pointerBytes) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.arch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{name: { + Name: name, Args: args, Ret: Void, + Frame: FrameLayout{Params: slots}, + }}, + }) + if op == "VMOVDQA" { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("virtual aggregate acquired native alignment: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "vmov-fp.ll", "vmov-fp.o", ir) + }) + } + } + }) + } +} + +func TestAMD64VMOVDQUAggregateFPBytesRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + file, err := Parse(ArchAMD64, `TEXT vmovAggregateFP(SB),4,$0-24 +MOVQ out+0(FP),AX +VMOVDQU key+8(FP),X0 +VMOVDQU X0,(AX) +PXOR X0,X0 +VMOVDQU X0,key+8(FP) +VMOVDQU key+8(FP),X1 +VMOVDQU X1,16(AX) +RET +`) + if err != nil { + t.Fatal(err) + } + sig := FuncSig{ + Name: "vmovAggregateFP", + Args: []LLVMType{Ptr, I32, I32, I32, I32}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I32, Index: 1, Field: -1}, + {Offset: 12, Type: I32, Index: 2, Field: -1}, + {Offset: 16, Type: I32, Index: 3, Field: -1}, + {Offset: 20, Type: I32, Index: 4, Field: -1}, + }}, + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOARCH != "amd64" { + if runtime.GOOS != "darwin" || !rosettaAvailable() { + t.Skip("execution requires amd64 or Rosetta; five-target object tests are separate") + } + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{sig.Name: sig}, + }) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +extern void vmovAggregateFP(uint8_t *, uint32_t, uint32_t, uint32_t, uint32_t); +int main(void) { + uint32_t key[4] = {0x12345678, 0x90abcdef, 0x13579bdf, 0x2468ace0}; + uint8_t out[32]; + memset(out, 0xa5, sizeof out); + vmovAggregateFP(out, key[0], key[1], key[2], key[3]); + for (int i = 0; i < 16; i++) { + if (out[i] != ((uint8_t *)key)[i] || out[16+i] != 0) { + fprintf(stderr, "VMOVDQU FP byte %d: got %u and %u\n", i, out[i], out[16+i]); + return 1; + } + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vmov_aggregate_fp", triple, ir, mainC, prefix) +} + +func TestX86VEXPackedMoveAggregateFPRejectsInvalidForms(t *testing.T) { + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"386", "i386-unknown-linux-gnu"}, + } { + for _, form := range []string{ + "VMOVDQU key+8(FP),X0", // Four-byte slot cannot supply 16 bytes. + "VMOVDQA key+8(FP),Y0", // Four-byte slot cannot supply 32 bytes. + "VMOVDQU key+8(FP),Z0", // No Z form in Go's VEX table. + "VMOVDQU.Z key+8(FP),X0", // No VEX zeroing form. + "VMOVDQU key+8(FP),K1,X0", // No VEX mask form. + } { + t.Run(target.arch+"/"+form, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT invalid(SB),4,$0-12\n"+form+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: target.arch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"invalid": { + Name: "invalid", Args: []LLVMType{Ptr, I32}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: I32, Index: 1, Field: -1}, + }}, + }}, + }) + if err == nil { + t.Fatalf("accepted unsupported form %q", form) + } + }) + } + } +} diff --git a/amd64_lower_vmov_half_test.go b/amd64_lower_vmov_half_test.go new file mode 100644 index 000000000..52206aa09 --- /dev/null +++ b/amd64_lower_vmov_half_test.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestAMD64MoveHalfGrammarCoversCompleteGoFamily(t *testing.T) { + expected := map[Op]amd64MoveHalfSpec{ + "MOVLPS": {}, + "MOVHPS": {high: true}, + "MOVLPD": {}, + "MOVHPD": {high: true}, + "VMOVLPS": {vector: true}, + "VMOVHPS": {high: true, vector: true}, + "VMOVLPD": {vector: true}, + "VMOVHPD": {high: true, vector: true}, + } + if len(amd64MoveHalfSpecs) != len(expected) { + t.Fatalf("move-half grammar has %d entries, want %d", len(amd64MoveHalfSpecs), len(expected)) + } + for op, want := range expected { + if got, ok := amd64MoveHalfSpecs[op]; !ok { + t.Errorf("move-half grammar omitted %s", op) + } else if got != want { + t.Errorf("move-half grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86VEXMoveHalfCompleteFormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + base, last := 30, 31 + if target.goarch == "386" { + base, last = 6, 7 + } + var source strings.Builder + source.WriteString("TEXT vmovhalfforms(SB),$0-0\n") + for _, op := range []string{"VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD"} { + fmt.Fprintf(&source, "\t%s X%d, 0(AX)\n", op, last) + fmt.Fprintf(&source, "\t%s 8(AX), X%d, X%d\n", op, base, last) + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"vmovhalfforms": {Name: "vmovhalfforms", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "extractelement <2 x i64>") || !strings.Contains(ir, "insertelement <2 x i64>") { + t.Fatalf("VEX half-move lowering omitted lane operations:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "vmov-half-"+target.name+".ll", "vmov-half-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86VEXMoveHalfRejectsFormsOutsideGoTable(t *testing.T) { + for _, instruction := range []string{ + "VMOVHPS.Z 0(AX), X1, X2", + "VMOVLPS Y0, 0(AX)", + "VMOVHPD 0(AX), Y1, Y2", + "VMOVLPD X0, X1, X2", + "VMOVHPS 0(AX), X1", + "VMOVLPS X0, X1", + } { + t.Run(strings.NewReplacer(" ", "_", ",", "", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's _yvmovhpd table", instruction) + } + }) + } +} + +func TestAMD64VEXMoveHalfRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT vmovhalfsemantics(SB),$0-24 + MOVQ out+0(FP), AX + MOVQ base+8(FP), CX + MOVQ half+16(FP), DX + MOVUPS 0(CX), X1 + STC + VMOVHPS 0(DX), X1, X2 + MOVUPS X2, 0(AX) + VMOVLPS 0(DX), X1, X3 + MOVUPS X3, 16(AX) + SETCS 32(AX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"vmovhalfsemantics": { + Name: "vmovhalfsemantics", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void vmovhalfsemantics(uint8_t *, const uint64_t *, const uint64_t *); +int main(void) { + uint8_t out[33] = {0}; + const uint64_t base[2] = {UINT64_C(0x1111222233334444), UINT64_C(0x5555666677778888)}; + const uint64_t half[1] = {UINT64_C(0xaaaabbbbccccdddd)}; + uint64_t got[4]; + vmovhalfsemantics(out, base, half); + memcpy(got, out, 32); + if (got[0] != base[0] || got[1] != half[0]) return 10; + if (got[2] != half[0] || got[3] != base[1]) return 11; + if (out[32] != 1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vex_move_half_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_vmov_integer.go b/amd64_lower_vmov_integer.go new file mode 100644 index 000000000..39b6b692a --- /dev/null +++ b/amd64_lower_vmov_integer.go @@ -0,0 +1,113 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Every named GP/X consumer shares exact width and encoding writeback rules. +// Raw VMOVW retains its separate, decoder-bound 16-bit role. +func (c *amd64Ctx) lowerVectorScalarIntegerMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + if !strings.EqualFold(string(ins.Op), string(op)) { + return true, false, fmt.Errorf("contradictory GP/X transfer dispatch %s/%s", op, ins.Op) + } + f, known, err := x86GPSIMDTransferProperties(ins, c.goarch) + if !known || err != nil { + return known, false, err + } + src, dst := ins.Args[0], ins.Args[1] + if f.memory >= 0 && ins.Args[f.memory].Kind == OpFP { + if _, err := c.scalarFPSpan(ins.Args[f.memory].FPOffset, f.bits); err != nil { + return true, false, err + } + } + var low string + if f.sourceX { + low, err = c.loadXLowInteger(src.Reg, f.bits) + } else { + low, err = c.loadGPSIMDTransferSource(src, f.bits) + } + if err != nil { + return true, false, err + } + if f.destinationX { + lanes := 128 / f.bits + inserted, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> zeroinitializer, i%d %s, i32 0\n", inserted, lanes, f.bits, f.bits, low) + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", bytes, lanes, f.bits, inserted) + if f.spec.legacy { + return true, false, c.storeX(dst.Reg, "%"+bytes) + } + return true, false, c.storePackedMoveOperand(dst, 16, "%"+bytes) + } + if f.mmx { + return true, false, c.storeReg(dst.Reg, low) + } + if dst.Kind == OpFP { + return true, false, c.storeScalarFPBits(dst.FPOffset, f.bits, low) + } + return true, false, c.storeVMOVIntegerScalar(dst, f.bits, low) +} + +func (c *amd64Ctx) loadGPSIMDTransferSource(source Operand, bits int) (string, error) { + if source.Kind == OpFP { + return c.loadScalarFPBits(source.FPOffset, bits) + } + var pointer, pointerType string + var err error + switch source.Kind { + case OpMem: + pointer, pointerType, err = c.ptrFromMem(source.Mem) + case OpSym: + pointer, err = c.ptrFromSB(source.Sym) + pointerType = "ptr" + default: + return c.evalIntSized(source, amd64IntegerTypeForBits(bits)) + } + if err != nil { + return "", err + } + value := c.newTmp() + // Unmasked MOVD/Q reads are observable at their original width even + // when the destination is dead. They have no alignment or MXCSR effect. + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %s, align 1\n", value, bits, pointerType, pointer) + return "%" + value, nil +} + +func isAMD64GoXRegister(operand Operand) bool { + if operand.Kind != OpReg { + return false + } + _, ok := amd64ParseXReg(operand.Reg) + return ok +} + +func (c *amd64Ctx) storeVMOVIntegerScalar(dst Operand, bits int, value string) error { + typ := amd64IntegerTypeForBits(bits) + if dst.Kind == OpMem || dst.Kind == OpSym { + var pointer, pointerType string + var err error + if dst.Kind == OpMem { + pointer, pointerType, err = c.ptrFromMem(dst.Mem) + } else { + pointer, err = c.ptrFromSB(dst.Sym) + pointerType = "ptr" + } + if err != nil { + return err + } + // Keep each original unmasked native store, including overwritten + // stores or private symbols whose value becomes otherwise dead. + fmt.Fprintf(c.b, " store volatile %s %s, %s %s, align 1\n", typ, value, pointerType, pointer) + return nil + } + if dst.Kind != OpReg { + return c.storeScalarIntegerOperand(dst, typ, value) + } + if bits == 64 { + return c.storeRegUnchecked(dst.Reg, value) + } + widened := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i%d %s to i64\n", widened, bits, value) + return c.storeRegUnchecked(dst.Reg, "%"+widened) +} diff --git a/amd64_lower_vmov_integer_test.go b/amd64_lower_vmov_integer_test.go new file mode 100644 index 000000000..62e838dd4 --- /dev/null +++ b/amd64_lower_vmov_integer_test.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86VMOVDVMOVQCompleteGoAssemblerForms(t *testing.T) { + // Go 1.27's _yvmovd and _yvmovq tables allow scalar GP/memory transfers + // in both directions. VMOVQ additionally allows X-to-X low-qword moves. + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT vmovintegerscalarforms(SB),4,$0-0 + VMOVD AX, X0 + VMOVD X0, AX + VMOVD SP, X1 + VMOVD X1, SP + VMOVD 8(AX), X20 + VMOVD X20, 12(AX) + VMOVQ AX, X2 + VMOVQ X2, AX + VMOVQ SP, X3 + VMOVQ X3, SP + VMOVQ 16(AX), X30 + VMOVQ X30, 24(AX) + VMOVQ X0, X1 + VMOVQ X20, X31 +` + if target.goarch == "amd64" { + source += "\tVMOVD R8, X31\n\tVMOVD X31, R9\n\tVMOVQ R10, X30\n\tVMOVQ X30, R11\n" + } else { + // Go accepts these high spellings but they lack a bound physical + // mode32 identity. Their independent Context cases live beside + // the shared grammar; ordinary object coverage uses X0-X7. + source = strings.NewReplacer("X20", "X4", "X30", "X5", "X31", "X6").Replace(source) + } + source += "\tRET\n" + goAssemblerOracleOutput(t, target.goarch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "vmovintegerscalarforms": {Name: "vmovintegerscalarforms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "vmov-integer-scalar-"+target.name+".ll", "vmov-integer-scalar-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86VMOVDVMOVQRejectsFormsOutsideGoAssemblerTables(t *testing.T) { + for _, instruction := range []string{ + "VMOVD X0, X1", + "VMOVD Y0, AX", + "VMOVD AX, Y0", + "VMOVD Z0, AX", + "VMOVD AX, Z0", + "VMOVD AX, BX", + "VMOVD (AX), 8(BX)", + "VMOVD.Z AX, X0", + "VMOVD AX, K1, X0", + "VMOVQ Y0, AX", + "VMOVQ AX, Y0", + "VMOVQ Z0, X0", + "VMOVQ AX, BX", + "VMOVQ (AX), 8(BX)", + "VMOVQ.Z X0, X1", + "VMOVQ X0, K1, X1", + } { + t.Run(strings.NewReplacer(" ", "_", "(", "_", ")", "_").Replace(instruction), func(t *testing.T) { + assertX86VMOVIntegerRejected(t, "amd64", "x86_64-unknown-linux-gnu", instruction) + }) + } + for _, instruction := range []string{ + "VMOVD R8, X0", + "VMOVD X0, R8", + "VMOVQ R8, X0", + "VMOVQ X0, R8", + } { + t.Run("386_"+strings.NewReplacer(" ", "_").Replace(instruction), func(t *testing.T) { + assertX86VMOVIntegerRejected(t, "386", "i386-unknown-linux-gnu", instruction) + }) + } +} + +func assertX86VMOVIntegerRejected(t *testing.T, goarch, triple, instruction string) { + t.Helper() + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+instruction+"\n\tRET\n") + if err != nil { + return + } + if _, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: goarch, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's VMOVD/VMOVQ forms for %s", instruction, goarch) + } +} diff --git a/amd64_lower_vmov_scalar.go b/amd64_lower_vmov_scalar.go new file mode 100644 index 000000000..2ec0a7ad3 --- /dev/null +++ b/amd64_lower_vmov_scalar.go @@ -0,0 +1,313 @@ +package plan9asm + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// lowerVectorScalarMove implements all Go 1.27 _yvmovsd forms shared by +// VMOVSD and VMOVSS: scalar loads/stores, three-register upper merges, and +// their EVEX masked variants. +func (c *amd64Ctx) lowerVectorScalarMove(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64ScalarVectorMoveSpecs[Op(baseOp)] + if !recognized || spec.legacy { + return false, false, nil + } + form, _, err := amd64ScalarVectorMoveProperties(ins, c.goarch) + if err != nil { + return true, false, err + } + if form.literal { + if form.legacyZero { + // The actual Go encoder emits legacy XORPS, even for a VMOV + // spelling. Do not zero the wider Y/Z views as VEX would. + var value string + if spec.bits == 32 { + value, err = c.evalF32(ins.Args[0]) + } else { + value, err = c.evalF64(ins.Args[0]) + } + if err != nil { + return true, false, err + } + return true, false, c.storeLegacyScalarMoveRegister(ins.Args[form.destination].Reg, value, spec.bits, false) + } + bits := uint64(ins.Args[0].Imm) + if spec.bits == 32 { + bits = uint64(math.Float32bits(float32(math.Float64frombits(bits)))) + } + mask := "" + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + return true, false, c.storeScalarMoveRegister(ins.Args[form.destination].Reg, spec.bits, + strconv.FormatUint(bits, 10), "zeroinitializer", mask, form.zeroing) + } + if form.upper >= 0 { + mask := Operand{} + if form.mask >= 0 { + mask = ins.Args[form.mask] + } + return c.lowerVectorScalarMoveRegisterMerge(spec.bits, ins.Args[form.low], ins.Args[form.upper], mask, + ins.Args[form.destination], form.zeroing) + } + if form.mask >= 0 { + return c.lowerVectorScalarMoveMaskedMemory(baseOp, spec.bits, form.zeroing, ins) + } + return c.lowerVectorScalarMoveTwoOperand(baseOp, spec.bits, ins) +} + +func (c *amd64Ctx) lowerVectorScalarMoveTwoOperand(baseOp string, elemBits int, ins Instr) (bool, bool, error) { + src, dst := ins.Args[0], ins.Args[1] + if amd64ScalarXOperand(src) && isAMD64MemoryOperand(dst) { + value, err := c.loadXLowInteger(src.Reg, elemBits) + if err != nil { + return true, false, err + } + return true, false, c.storeVectorScalarMemory(dst, elemBits, value, "") + } + if isAMD64MemoryOperand(src) && amd64ScalarXOperand(dst) { + value, err := c.loadScalarMoveMemory(src, elemBits) + if err != nil { + return true, false, err + } + return true, false, c.storeScalarMoveRegister(dst.Reg, elemBits, value, "zeroinitializer", "", false) + } + return true, false, fmt.Errorf("amd64 %s two-operand form requires X-to-memory or memory-to-X: %q", baseOp, ins.Raw) +} + +func (c *amd64Ctx) lowerVectorScalarMoveMaskedMemory(baseOp string, elemBits int, zeroing bool, ins Instr) (bool, bool, error) { + if !amd64NonzeroKOperand(ins.Args[1]) { + return true, false, fmt.Errorf("amd64 %s three-operand masked memory form expects K1-K7 in the middle: %q", baseOp, ins.Raw) + } + mask, err := c.loadK(ins.Args[1].Reg) + if err != nil { + return true, false, err + } + if amd64ScalarXOperand(ins.Args[0]) && isAMD64MemoryOperand(ins.Args[2]) { + if zeroing { + return true, false, fmt.Errorf("amd64 %s cannot use .Z with a memory destination: %q", baseOp, ins.Raw) + } + value, err := c.loadXLowInteger(ins.Args[0].Reg, elemBits) + if err != nil { + return true, false, err + } + return true, false, c.storeVectorScalarMemory(ins.Args[2], elemBits, value, mask) + } + if isAMD64MemoryOperand(ins.Args[0]) && amd64ScalarXOperand(ins.Args[2]) { + value, err := c.loadVectorScalarMemory(ins.Args[0], elemBits, mask) + if err != nil { + return true, false, err + } + return true, false, c.storeScalarMoveRegister(ins.Args[2].Reg, elemBits, value, "zeroinitializer", mask, zeroing) + } + return true, false, fmt.Errorf("amd64 %s masked memory form requires X, K, memory or memory, K, X: %q", baseOp, ins.Raw) +} + +// A masked-off scalar move must not access memory at all. Selecting a loaded +// value afterwards does not suppress faults, and a read/modify/write store +// spuriously reads write-only memory even when the mask is active. +func (c *amd64Ctx) loadVectorScalarMemory(src Operand, elemBits int, mask string) (string, error) { + condition := amd64MaskBitI1(c, mask, 0) + return c.loadX86ScalarMemoryIf(src, elemBits, condition) +} + +func (c *amd64Ctx) loadX86ScalarMemoryIf(src Operand, elemBits int, condition string) (string, error) { + if src.Kind == OpFP && elemBits != 16 { + if _, err := c.scalarFPSpan(src.FPOffset, elemBits); err != nil { + return "", err + } + // Parameter-only loads also need a source-defined mask and merge + // lane. Validate before creating either masked control-flow branch. + if err := c.scalarFPResultInitialized(); err != nil { + return "", err + } + } + return c.loadX86ScalarIf(elemBits, condition, func() (string, error) { + return c.loadScalarMoveMemory(src, elemBits) + }) +} + +func (c *amd64Ctx) loadScalarMoveMemory(src Operand, bits int) (string, error) { + if src.Kind == OpFP && bits != 16 { + return c.loadScalarFPBits(src.FPOffset, bits) + } + return c.evalIntSized(src, amd64IntegerTypeForBits(bits)) +} + +func (c *amd64Ctx) loadX86ScalarIf(elemBits int, condition string, load func() (string, error)) (string, error) { + loadLabel := c.newTmp() + "_scalar_load" + skipLabel := c.newTmp() + "_scalar_skip" + joinLabel := c.newTmp() + "_scalar_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n%s:\n", condition, loadLabel, skipLabel, loadLabel) + value, err := load() + if err != nil { + return "", err + } + fmt.Fprintf(c.b, " br label %%%s\n%s:\n br label %%%s\n%s:\n", joinLabel, skipLabel, joinLabel, joinLabel) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = phi i%d [ %s, %%%s ], [ 0, %%%s ]\n", selected, elemBits, value, loadLabel, skipLabel) + return "%" + selected, nil +} + +func (c *amd64Ctx) lowerVectorScalarMoveRegisterMerge(elemBits int, lowSource, upperSource, maskArg, dst Operand, zeroing bool) (bool, bool, error) { + low, err := c.loadXLowInteger(lowSource.Reg, elemBits) + if err != nil { + return true, false, err + } + upperBytes, err := c.loadX(upperSource.Reg) + if err != nil { + return true, false, err + } + lanes := 128 / elemBits + upper := c.bitcastVectorBytesToIntegerLanes(16, lanes, elemBits, upperBytes) + mask := "" + if maskArg.Kind == OpReg { + mask, err = c.loadK(maskArg.Reg) + if err != nil { + return true, false, err + } + } + return true, false, c.storeScalarMoveRegister(dst.Reg, elemBits, low, upper, mask, zeroing) +} + +func amd64ScalarXOperand(op Operand) bool { + if op.Kind != OpReg { + return false + } + index, ok := amd64ParseXReg(op.Reg) + return ok && index < 32 +} + +func amd64NonzeroKOperand(op Operand) bool { + if op.Kind != OpReg { + return false + } + index, ok := amd64ParseKReg(op.Reg) + return ok && index != 0 +} + +func (c *amd64Ctx) loadXLowInteger(reg Reg, elemBits int) (string, error) { + bytesValue, err := c.loadX(reg) + if err != nil { + return "", err + } + lanes := 128 / elemBits + values := c.bitcastVectorBytesToIntegerLanes(16, lanes, elemBits, bytesValue) + low := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement <%d x i%d> %s, i32 0\n", low, lanes, elemBits, values) + return "%" + low, nil +} + +func (c *amd64Ctx) storeVectorScalarRegister(dst Reg, elemBits int, low, base, mask string, zeroing bool) error { + lanes := 128 / elemBits + if mask != "" { + fallback := "0" + if !zeroing { + old, err := c.loadXLowInteger(dst, elemBits) + if err != nil { + return err + } + fallback = old + } + maskBit := amd64MaskBitI1(c, mask, 0) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %s, i%d %s\n", selected, maskBit, elemBits, low, elemBits, fallback) + low = "%" + selected + } + updated := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement <%d x i%d> %s, i%d %s, i32 0\n", updated, lanes, elemBits, base, elemBits, low) + bytesValue := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %%%s to <16 x i8>\n", bytesValue, lanes, elemBits, updated) + return c.storeX(dst, "%"+bytesValue) +} + +func (c *amd64Ctx) storeScalarMoveRegister(dst Reg, elemBits int, low, base, mask string, zeroing bool) error { + if err := c.storeVectorScalarRegister(dst, elemBits, low, base, mask, zeroing); err != nil { + return err + } + // VEX/EVEX scalar writes clear bits above 127, unlike legacy MOVSS/SD. + // Keep every live wider view coherent with the newly written low X value. + index, _ := amd64ParseXReg(dst) + for _, view := range []struct { + bytes int + slots map[int]string + }{{32, c.yRegSlot}, {64, c.zRegSlot}} { + if slot, live := view.slots[index]; live { + bytesValue, err := c.loadX(dst) + if err != nil { + return err + } + indices := make([]string, view.bytes) + for i := range indices { + lane := i + if lane >= 16 { + lane = 16 // First byte of the zero second source. + } + indices[i] = fmt.Sprintf("i32 %d", lane) + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = shufflevector <16 x i8> %s, <16 x i8> zeroinitializer, <%d x i32> <%s>\n", wide, bytesValue, view.bytes, strings.Join(indices, ", ")) + fmt.Fprintf(c.b, " store <%d x i8> %%%s, ptr %s\n", view.bytes, wide, slot) + } + } + return nil +} + +func (c *amd64Ctx) storeVectorScalarMemory(dst Operand, elemBits int, value, mask string) error { + if dst.Kind == OpFP && elemBits != 16 { + result, err := c.scalarFPSpan(dst.FPOffset, elemBits) + if err != nil { + return err + } + if result && mask != "" { + if err := c.scalarFPResultInitialized(); err != nil { + return err + } + } + } + if mask != "" { + condition := amd64MaskBitI1(c, mask, 0) + storeLabel := c.newTmp() + "_scalar_store" + joinLabel := c.newTmp() + "_scalar_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n%s:\n", condition, storeLabel, joinLabel, storeLabel) + if err := c.storeVectorScalarMemory(dst, elemBits, value, ""); err != nil { + return err + } + fmt.Fprintf(c.b, " br label %%%s\n%s:\n", joinLabel, joinLabel) + return nil + } + switch dst.Kind { + case OpMem: + ptr, ptrType, err := c.ptrFromMem(dst.Mem) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i%d %s, %s %s, align 1\n", elemBits, value, ptrType, ptr) + return nil + case OpSym: + ptr, err := c.ptrFromSB(dst.Sym) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store i%d %s, ptr %s, align 1\n", elemBits, value, ptr) + return nil + case OpFP: + if elemBits != 16 { + return c.storeScalarFPBits(dst.FPOffset, elemBits, value) + } + return c.storeFPResult(dst.FPOffset, amd64IntegerTypeForBits(elemBits), value) + default: + return fmt.Errorf("expected scalar memory destination, got %s", dst.String()) + } +} diff --git a/amd64_lower_vmov_scalar_aggregate_test.go b/amd64_lower_vmov_scalar_aggregate_test.go new file mode 100644 index 000000000..5cad15318 --- /dev/null +++ b/amd64_lower_vmov_scalar_aggregate_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "errors" + "fmt" + "runtime" + "testing" +) + +func TestTranslateX86WideStoresSpanComplex64Result(t *testing.T) { + for _, tc := range []struct { + name string + instruction string + amd64Only bool + needsZeroXMM bool + }{ + {name: "movsd", instruction: "MOVSD X0, ret+0(FP)", needsZeroXMM: true}, + {name: "movq-xmm", instruction: "MOVQ X0, ret+0(FP)", needsZeroXMM: true}, + {name: "vmovq-xmm", instruction: "VMOVQ X0, ret+0(FP)", needsZeroXMM: true}, + {name: "movq-gp", instruction: "MOVQ AX, ret+0(FP)", amd64Only: true}, + {name: "vmovsd", instruction: "VMOVSD X0, ret+0(FP)", amd64Only: true, needsZeroXMM: true}, + } { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + if tc.amd64Only && target.goarch != "amd64" { + continue + } + t.Run(tc.name+"/"+target.name, func(t *testing.T) { + functionName := "widestorecomplex64result" + source := fmt.Sprintf("TEXT %s(SB),$0-8\n", functionName) + if tc.needsZeroXMM { + source += "\tPXOR X0, X0\n" + } + source += "\t" + tc.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + functionName: { + Name: functionName, + Ret: LLVMType("{ float, float }"), + Frame: FrameLayout{Results: []FrameSlot{ + {Offset: 0, Type: LLVMType("float"), Index: 0, Field: -1, Name: "ret"}, + {Offset: 4, Type: LLVMType("float"), Index: 1, Field: -1, Name: "ret"}, + }}, + }, + }, + } + if tc.name == "movq-gp" { + mod, directErr := translateModuleDirect(file, options) + if directErr == nil { + mod.Dispose() + t.Fatal("direct linear prototype unexpectedly accepted a wide aggregate-result store") + } + if !errors.Is(directErr, errDirectModuleUnsupported) { + t.Fatalf("direct linear prototype error = %v, want fallback request", directErr) + } + } + ll, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "wide-store-complex64-result.ll", "wide-store-complex64-result.o", ll) + }) + } + } +} + +func TestAMD64VMOVSDStoresComplex64ResultRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + source := ` +TEXT vmovsdcomplex64result(SB),$0-16 + MOVQ input+0(FP), AX + VMOVSD (AX), X0 + VMOVSD X0, ret+8(FP) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "vmovsdcomplex64result": { + Name: "vmovsdcomplex64result", + Args: []LLVMType{Ptr}, + Ret: LLVMType("{ float, float }"), + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1, Name: "input"}}, + Results: []FrameSlot{ + {Offset: 8, Type: LLVMType("float"), Index: 0, Field: -1, Name: "ret"}, + {Offset: 12, Type: LLVMType("float"), Index: 1, Field: -1, Name: "ret"}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + ll += ` +define void @call_vmovsdcomplex64result(ptr %input, ptr %output) { +entry: + %result = call { float, float } @vmovsdcomplex64result(ptr %input) + %real = extractvalue { float, float } %result, 0 + %imag = extractvalue { float, float } %result, 1 + store float %real, ptr %output, align 4 + %imagptr = getelementptr float, ptr %output, i64 1 + store float %imag, ptr %imagptr, align 4 + ret void +} +` + mainC := ` +extern void call_vmovsdcomplex64result(const float *input, float *output); +int main(void) { + const float input[2] = { 1.25f, -2.5f }; + float got[2] = { 0.0f, 0.0f }; + call_vmovsdcomplex64result(input, got); + if (got[0] != input[0]) return 1; + if (got[1] != input[1]) return 2; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "vmovsd_complex64_result", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_word_madd_family.go b/amd64_lower_word_madd_family.go new file mode 100644 index 000000000..58a332c15 --- /dev/null +++ b/amd64_lower_word_madd_family.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "fmt" +) + +func (c *amd64Ctx) lowerWordMultiplyAddFamily(op Op, ins Instr) (bool, bool, error) { + form, recognized, err := x86PackedMultiplyAddProperties(op, ins, c.goarch) + if err != nil || !recognized { + return recognized, false, err + } + first, second, destination := form.source, ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: %s m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext, op) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + if form.mmx { + if first.Reg != ins.Args[form.first].Reg { + return true, false, fmt.Errorf("%w: %s GP spelling encodes a separate MMX source bank", ErrProbeNeedsContext, op) + } + var a string + if form.memory < 0 { + a, err = c.loadReg(first.Reg) + } else { + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + result := c.emitPackedWordMultiplyAdd(4, c.bitcastI64ToIntegerLanes(4, 16, a), c.bitcastI64ToIntegerLanes(4, 16, b)) + bits := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <2 x i32> %s to i64\n", bits, result) + return true, false, c.storeReg(destination.Reg, "%"+bits) + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, form.spec.inputBits, false) + if err != nil { + return true, false, err + } + var result string + if form.legacy && form.memory >= 0 && first.Kind != OpFP { + result, err = c.emitMultiplyAddNativeMemory(first, b, form.spec) + } else { + // Intel SDM's E4NF.nb is non-fault-suppressing. A plain eager LLVM + // load can disappear under a constant zero mask, so use the reviewed + // complete volatile native reader, never logical masked loads. + a, loadErr := c.loadPackedUnpackSource(first, x86PackedUnpackForm{ + x86PackedBinaryForm: form.x86PackedBinaryForm, spec: x86PackedUnpackSpec{laneBits: form.spec.inputBits}, + }) + if loadErr != nil { + return true, false, loadErr + } + if form.spec.inputBits == 8 { + result = c.emitPackedUnsignedSignedByteMultiplyAdd(form.byteWidth, a, b) + } else { + result = c.emitPackedWordMultiplyAdd(form.byteWidth/2, a, b) + } + } + if err != nil { + return true, false, err + } + lanes := form.byteWidth * 8 / form.spec.outputBits + if form.mask >= 0 { + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, form.spec.outputBits, false) + if err != nil { + return true, false, err + } + } + result = amd64ApplyIntegerLaneMask(c, lanes, form.spec.outputBits, result, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, form.spec.outputBits, result, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +func (c *amd64Ctx) emitMultiplyAddNativeMemory(source Operand, data string, spec x86PackedMultiplyAddSpec) (string, error) { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + switch pointerType { + case "ptr addrspace(256)": + segment = "%gs:" + case "ptr addrspace(257)": + segment = "%fs:" + } + carrier, value := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <4 x i32>\n", carrier, 128/spec.inputBits, spec.inputBits, data) + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, %q(<4 x i32> %%%s, %s %s)\n", + value, spec.nativeOpcode+" "+segment+"($2), $0", "=x,0,r,~{memory}", carrier, pointerType, pointer) + if spec.outputBits != 32 { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <%d x i%d>\n", result, value, 128/spec.outputBits, spec.outputBits) + return "%" + result, nil + } + return "%" + value, nil +} diff --git a/amd64_lower_wrapping_add.go b/amd64_lower_wrapping_add.go new file mode 100644 index 000000000..f8eeb0dfc --- /dev/null +++ b/amd64_lower_wrapping_add.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func (c *amd64Ctx) lowerPackedIntegerAddForm(op Op, ins Instr) (bool, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := amd64PackedAddSpecs[Op(base)]; !known { + return false, false, nil + } + if ins.Op != "" && strings.ToUpper(string(ins.Op)) != strings.ToUpper(string(op)) { + return true, false, fmt.Errorf("packed add dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + ins.Op = op + arch := c.goarch + if arch == "" { + arch = "amd64" + } + form, known, err := x86PackedIntegerAddProperties(ins, arch) + if err != nil || !known { + return known, false, err + } + first, second, destination := ins.Args[form.first], ins.Args[form.second], ins.Args[form.destination] + if form.legacy && !form.mmx && form.memory >= 0 { + virtual := first.Kind == OpFP + if first.Kind == OpMem && !x86CallerMemoryCertified(ins, form.memory) && !c.fixedLocalMemoryCertified(ins, form.memory) { + virtual = !x86OrdinaryNativeMemoryOperand(first, c.goarch) + } + if virtual { + return true, false, fmt.Errorf("%w: legacy packed add m128 needs original native storage, not virtual FP/SP/BP alignment", ErrProbeNeedsContext) + } + } + if form.memory >= 0 && first.Kind == OpFP { + if err := c.validateFPVectorSpan(first.FPOffset, form.memoryBytes); err != nil { + return true, false, err + } + } + bits := form.spec.laneBits + lanes := form.byteWidth * 8 / bits + if form.mmx { + var a string + if first.Kind == OpReg { + a, err = c.loadReg(first.Reg) + } else { + a, err = c.loadPackedUnpackMMXSource(first, 8) + } + if err != nil { + return true, false, err + } + b, err := c.loadReg(second.Reg) + if err != nil { + return true, false, err + } + value := c.emitPackedIntegerAdd(lanes, bits, c.bitcastI64ToIntegerLanes(lanes, bits, a), c.bitcastI64ToIntegerLanes(lanes, bits, b), form.spec.mode) + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to i64\n", out, lanes, bits, value) + return true, false, c.storeReg(destination.Reg, "%"+out) + } + b, err := c.loadPackedCompareLanes(second, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + var mask, value string + if form.mask >= 0 { + mask, err = c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return true, false, err + } + } + if form.legacy && form.memory >= 0 { + mnemonic := strings.ToLower(string(form.spec.canonical)) + if mnemonic == "paddl" { + mnemonic = "paddd" + } + value, err = c.emitPackedBinaryNativeMemory(first, bits, b, mnemonic) + } else { + // PADD's EVEX E4/E4.nb suppresses inactive native source lanes, + // unlike PMADDWD's E4NF. Virtual FP still needs its complete span. + // This shared bit reader keeps native active-lane accesses volatile + // even when outputs are dead, while suppressing exactly inactive E4 + // lanes. It also reads broadcasts across independent checked FP bytes. + // The shift family's count/arithmetic effects are not reused. + a, loadErr := c.loadX86PackedShiftLanes(first, form.byteWidth, bits, form.broadcast, mask) + err = loadErr + if err == nil { + value = c.emitPackedIntegerAdd(lanes, bits, a, b, form.spec.mode) + } + } + if err != nil { + return true, false, err + } + if form.mask >= 0 { + old := "zeroinitializer" + if !form.zeroing { + old, err = c.loadPackedCompareLanes(destination, form.byteWidth, bits, false) + if err != nil { + return true, false, err + } + } + value = amd64ApplyIntegerLaneMask(c, lanes, bits, value, old, mask, form.zeroing) + } + out := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <%d x i8>\n", out, lanes, bits, value, form.byteWidth) + if form.legacy { + return true, false, c.storeLegacyPackedLogical(destination, "%"+out) + } + return true, false, c.storePackedMoveOperand(destination, form.byteWidth, "%"+out) +} + +// The caller supplies a registered legacy mnemonic. Keeping its memory form +// native preserves alignment/fault effects which an unaligned vector load +// followed by portable arithmetic would erase. +func (c *amd64Ctx) emitPackedBinaryNativeMemory(source Operand, bits int, data, mnemonic string) (string, error) { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + carrier, result := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i%d> %s to <4 x i32>\n", carrier, 128/bits, bits, data) + fmt.Fprintf(c.b, " %%%s = call <4 x i32> asm sideeffect %q, %q(<4 x i32> %%%s, %s %s)\n", result, mnemonic+" "+segment+"($2), $0", "=x,0,r,~{memory}", carrier, pointerType, pointer) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <4 x i32> %%%s to <%d x i%d>\n", value, result, 128/bits, bits) + return "%" + value, nil +} diff --git a/amd64_lower_x87.go b/amd64_lower_x87.go index 0ef86ec04..1abef283a 100644 --- a/amd64_lower_x87.go +++ b/amd64_lower_x87.go @@ -1,12 +1,31 @@ package plan9asm -import "fmt" +import ( + "fmt" + "strings" +) func isX87Op(op Op) bool { switch op { - case "FMOVD", "FMOVDP", "FMOVV", "FMOVVP", "FXCHD", "FADDDP", - "FDIVD", "FMULD", "FMULDP", "FSTCW", "FLDCW", "FRNDINT", - "FABS", "FUCOMI", "FTST", "FSTSW", "FLD1", "FSQRT": + case "FMOVB", "FMOVBP", "FMOVD", "FMOVDP", "FMOVF", "FMOVFP", + "FMOVL", "FMOVLP", "FMOVV", "FMOVVP", "FMOVW", "FMOVWP", + "FMOVX", "FMOVXP", "FBLD", "FBSTP", "FXCHD", + "FADDW", "FADDL", "FADDF", "FADDD", "FADDDP", + "FMULW", "FMULL", "FMULF", "FMULD", "FMULDP", + "FSUBW", "FSUBL", "FSUBF", "FSUBD", "FSUBDP", + "FSUBRW", "FSUBRL", "FSUBRF", "FSUBRD", "FSUBRDP", + "FDIVW", "FDIVL", "FDIVF", "FDIVD", "FDIVDP", + "FDIVRW", "FDIVRL", "FDIVRF", "FDIVRD", "FDIVRDP", + "FCOMD", "FCOMDP", "FCOMDPP", "FCOMF", "FCOMFP", "FCOMI", "FCOMIP", + "FCOML", "FCOMLP", "FCOMW", "FCOMWP", + "FUCOM", "FUCOMI", "FUCOMIP", "FUCOMP", "FUCOMPP", + "FCMOVCC", "FCMOVCS", "FCMOVEQ", "FCMOVHI", "FCMOVLS", + "FCMOVB", "FCMOVBE", "FCMOVNB", "FCMOVNBE", "FCMOVE", "FCMOVNE", "FCMOVNU", "FCMOVU", "FCMOVUN", + "FSTCW", "FLDCW", "FLDENV", "FRSTOR", "FSAVE", "FSTENV", "FRNDINT", + "F2XM1", "FABS", "FCHS", "FCLEX", "FCOS", "FDECSTP", "FINCSTP", "FINIT", + "FTST", "FSTSW", "FLD1", "FLDL2E", "FLDL2T", "FLDLG2", "FLDLN2", + "FLDPI", "FLDZ", "FNOP", "FPATAN", "FPREM", "FPREM1", "FPTAN", "FSCALE", + "FSIN", "FSINCOS", "FSQRT", "FXAM", "FXTRACT", "FYL2X", "FYL2XP1": return true default: return false @@ -17,15 +36,71 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err if !isX87Op(op) { return false, false, nil } - if c.goarch != "386" { - return true, false, fmt.Errorf("amd64: x87 instruction %s requires GOARCH=386", op) - } - switch op { + case "FBLD": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FBLD expects memory source: %q", ins.Raw) + } + value, err := c.loadX87PackedBCD(ins.Args[0]) + if err != nil { + return true, false, err + } + return true, false, c.pushX87(value) + + case "FBSTP": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FBSTP expects memory destination: %q", ins.Raw) + } + if err := c.storeX87PackedBCD(ins.Args[0], c.loadX87(0)); err != nil { + return true, false, err + } + c.popX87() + return true, false, nil + + case "FMOVB", "FMOVX": + if len(ins.Args) != 2 || !isX87MemoryOperand(ins.Args[0]) || !isX87F0(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s expects memory source and F0 destination: %q", op, ins.Raw) + } + var value string + var err error + if op == "FMOVB" { + value, err = c.loadX87PackedBCD(ins.Args[0]) + } else { + value, err = c.loadX87Extended(ins.Args[0]) + } + if err != nil { + return true, false, err + } + return true, false, c.pushX87(value) + + case "FMOVBP", "FMOVXP": + if len(ins.Args) != 2 || !isX87F0(ins.Args[0]) || !isX87MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s expects F0 source and memory destination: %q", op, ins.Raw) + } + var err error + if op == "FMOVBP" { + err = c.storeX87PackedBCD(ins.Args[1], c.loadX87(0)) + } else { + err = c.storeX87Extended(ins.Args[1], c.loadX87(0)) + } + if err != nil { + return true, false, err + } + c.popX87() + return true, false, nil + case "FMOVD": if len(ins.Args) != 2 { return true, false, fmt.Errorf("386 FMOVD expects src, dst: %q", ins.Raw) } + srcReg, srcIsReg := x87OperandReg(ins.Args[0]) + dstReg, dstIsReg := x87OperandReg(ins.Args[1]) + valid := (isX87MemoryOperand(ins.Args[0]) || ins.Args[0].Kind == OpImm) && dstIsReg && dstReg == 0 || + srcIsReg && srcReg == 0 && (isX87MemoryOperand(ins.Args[1]) || dstIsReg) || + srcIsReg && dstIsReg && dstReg == 0 + if !valid { + return true, false, fmt.Errorf("386 FMOVD expects memory,F0, Freg,F0, F0,memory, or F0,Freg: %q", ins.Raw) + } v, err := c.evalX87F64(ins.Args[0]) if err != nil { return true, false, err @@ -36,8 +111,8 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.storeX87F64(ins.Args[1], v) case "FMOVDP": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FMOVDP expects src, dst: %q", ins.Raw) + if len(ins.Args) != 2 || !isX87F0(ins.Args[0]) || !(isX87MemoryOperand(ins.Args[1]) || isX87RegisterOperand(ins.Args[1])) { + return true, false, fmt.Errorf("386 FMOVDP expects F0 source and memory or x87-register destination: %q", ins.Raw) } v, err := c.evalX87F64(ins.Args[0]) if err != nil { @@ -49,12 +124,36 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err c.popX87() return true, false, nil - case "FMOVV": + case "FMOVF", "FMOVL", "FMOVW": if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FMOVV expects src, dst: %q", ins.Raw) + return true, false, fmt.Errorf("386 %s expects src, dst: %q", op, ins.Raw) + } + if isX87MemoryOperand(ins.Args[0]) && isX87F0(ins.Args[1]) { + value, err := c.loadX87MoveValue(op, ins.Args[0]) + if err != nil { + return true, false, err + } + return true, false, c.pushX87(value) + } + if isX87F0(ins.Args[0]) && isX87MemoryOperand(ins.Args[1]) { + return true, false, c.storeX87MoveValue(op, ins.Args[1], c.loadX87(0)) + } + return true, false, fmt.Errorf("386 %s expects memory source and F0 destination, or F0 source and memory destination: %q", op, ins.Raw) + + case "FMOVFP", "FMOVLP", "FMOVWP": + if len(ins.Args) != 2 || !isX87F0(ins.Args[0]) || !isX87MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("386 %s expects F0 source and memory destination: %q", op, ins.Raw) } - if idx, ok := x87OperandReg(ins.Args[1]); !ok || idx != 0 { - return true, false, fmt.Errorf("386 FMOVV expects F0 destination: %q", ins.Raw) + baseOp := Op(strings.TrimSuffix(string(op), "P")) + if err := c.storeX87MoveValue(baseOp, ins.Args[1], c.loadX87(0)); err != nil { + return true, false, err + } + c.popX87() + return true, false, nil + + case "FMOVV": + if len(ins.Args) != 2 || !isX87MemoryOperand(ins.Args[0]) || !isX87F0(ins.Args[1]) { + return true, false, fmt.Errorf("386 FMOVV expects memory source and F0 destination: %q", ins.Raw) } v, err := c.evalIntSized(ins.Args[0], I64) if err != nil { @@ -65,11 +164,8 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.pushX87("%" + f) case "FMOVVP": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FMOVVP expects src, dst: %q", ins.Raw) - } - if index, ok := x87OperandReg(ins.Args[0]); !ok || index != 0 { - return true, false, fmt.Errorf("386 FMOVVP expects F0 source: %q", ins.Raw) + if len(ins.Args) != 2 || !isX87F0(ins.Args[0]) || !isX87MemoryOperand(ins.Args[1]) { + return true, false, fmt.Errorf("386 FMOVVP expects F0 source and memory destination: %q", ins.Raw) } var iv string if c.useHardwareX87() { @@ -101,53 +197,26 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err c.storeX87(bv, av) return true, false, nil - case "FDIVD", "FMULD": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 %s expects src, dst: %q", op, ins.Raw) - } - src, err := c.evalX87F64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst, err := c.evalX87F64(ins.Args[1]) - if err != nil { - return true, false, err - } - out := c.newTmp() - if op == "FDIVD" { - fmt.Fprintf(c.b, " %%%s = fdiv double %s, %s\n", out, dst, src) - } else { - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", out, dst, src) - } - return true, false, c.storeX87F64(ins.Args[1], "%"+out) + case "FADDW", "FADDL", "FADDF", "FADDD", "FADDDP", + "FMULW", "FMULL", "FMULF", "FMULD", "FMULDP", + "FSUBW", "FSUBL", "FSUBF", "FSUBD", "FSUBDP", + "FSUBRW", "FSUBRL", "FSUBRF", "FSUBRD", "FSUBRDP", + "FDIVW", "FDIVL", "FDIVF", "FDIVD", "FDIVDP", + "FDIVRW", "FDIVRL", "FDIVRF", "FDIVRD", "FDIVRDP": + return true, false, c.lowerX87Arithmetic(op, ins) - case "FADDDP", "FMULDP": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 %s expects src, dst: %q", op, ins.Raw) - } - src, err := c.evalX87F64(ins.Args[0]) - if err != nil { - return true, false, err - } - dst, err := c.evalX87F64(ins.Args[1]) - if err != nil { - return true, false, err - } - out := c.newTmp() - if op == "FADDDP" { - fmt.Fprintf(c.b, " %%%s = fadd double %s, %s\n", out, dst, src) - } else { - fmt.Fprintf(c.b, " %%%s = fmul double %s, %s\n", out, dst, src) - } - if err := c.storeX87F64(ins.Args[1], "%"+out); err != nil { - return true, false, err - } - c.popX87() - return true, false, nil + case "FCOMD", "FCOMDP", "FCOMDPP", "FCOMF", "FCOMFP", "FCOMI", "FCOMIP", + "FCOML", "FCOMLP", "FCOMW", "FCOMWP", + "FUCOM", "FUCOMI", "FUCOMIP", "FUCOMP", "FUCOMPP": + return true, false, c.lowerX87Compare(op, ins) + + case "FCMOVCC", "FCMOVCS", "FCMOVEQ", "FCMOVHI", "FCMOVLS", + "FCMOVB", "FCMOVBE", "FCMOVNB", "FCMOVNBE", "FCMOVE", "FCMOVNE", "FCMOVNU", "FCMOVU", "FCMOVUN": + return true, false, c.lowerX87ConditionalMove(op, ins) case "FSTCW": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("386 FSTCW expects dst: %q", ins.Raw) + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FSTCW expects memory destination: %q", ins.Raw) } if c.useHardwareX87() { c.storeHardwareX87ControlWord() @@ -157,8 +226,8 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, c.storeX87I16(ins.Args[0], "%"+cw) case "FLDCW": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("386 FLDCW expects src: %q", ins.Raw) + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 FLDCW expects memory source: %q", ins.Raw) } cw, err := c.evalIntSized(ins.Args[0], I16) if err != nil { @@ -170,6 +239,28 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err } return true, false, nil + case "FLDENV", "FRSTOR": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 %s expects memory source: %q", op, ins.Raw) + } + return true, false, c.loadX87Environment(ins.Args[0]) + + case "FSAVE", "FSTENV": + if len(ins.Args) != 1 || !isX87MemoryOperand(ins.Args[0]) { + return true, false, fmt.Errorf("386 %s expects memory destination: %q", op, ins.Raw) + } + if err := c.storeX87Environment(ins.Args[0]); err != nil { + return true, false, err + } + if op == "FSAVE" { + for i := range c.x87Slot { + c.storeX87(i, "0.000000e+00") + } + fmt.Fprintf(c.b, " store i16 895, ptr %s\n", c.x87ControlSlot) + fmt.Fprintf(c.b, " store i16 0, ptr %s\n", c.x87StatusSlot) + } + return true, false, nil + case "FRNDINT": if len(ins.Args) != 0 { return true, false, fmt.Errorf("386 FRNDINT takes no operands: %q", ins.Raw) @@ -180,6 +271,18 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err } return true, false, c.storeX87F64(Operand{Kind: OpReg, Reg: Reg("F0")}, c.roundX87Software(c.loadX87(0))) + case "F2XM1": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 F2XM1 takes no operands: %q", ins.Raw) + } + value := c.loadX87(0) + power := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.exp2.f64(double %s)\n", power, value) + fmt.Fprintf(c.b, " %%%s = fsub double %%%s, 1.000000e+00\n", result, power) + c.storeX87(0, "%"+result) + return true, false, nil + case "FABS", "FSQRT": if len(ins.Args) != 0 { return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) @@ -194,27 +297,177 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err c.storeX87(0, "%"+out) return true, false, nil - case "FUCOMI": - if len(ins.Args) != 2 { - return true, false, fmt.Errorf("386 FUCOMI expects lhs, rhs: %q", ins.Raw) + case "FCHS": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FCHS takes no operands: %q", ins.Raw) } - lhs, err := c.evalX87F64(ins.Args[0]) - if err != nil { + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fneg double %s\n", result, c.loadX87(0)) + c.storeX87(0, "%"+result) + return true, false, nil + + case "FCLEX": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FCLEX takes no operands: %q", ins.Raw) + } + if c.useHardwareX87() { + fmt.Fprintln(c.b, ` call void asm sideeffect "fnclex", "~{fpsr}"()`) + } + status := c.newTmp() + conditionCodes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) + fmt.Fprintf(c.b, " %%%s = and i16 %%%s, 32512\n", conditionCodes, status) // Preserve TOP and C3,C2,C1,C0. + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", conditionCodes, c.x87StatusSlot) + return true, false, nil + + case "FCOS", "FSIN": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + intrinsic := "llvm.cos.f64" + if op == "FSIN" { + intrinsic = "llvm.sin.f64" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @%s(double %s)\n", result, intrinsic, c.loadX87(0)) + c.storeX87(0, "%"+result) + c.clearX87C2() + return true, false, nil + + case "FDECSTP", "FINCSTP": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + c.rotateX87Stack(op == "FINCSTP") + return true, false, nil + + case "FINIT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FINIT takes no operands: %q", ins.Raw) + } + if c.useHardwareX87() { + fmt.Fprintln(c.b, ` call void asm sideeffect "fninit", "~{st},~{st(1)},~{st(2)},~{st(3)},~{st(4)},~{st(5)},~{st(6)},~{st(7)},~{fpsr}"()`) + } + for i := range c.x87Slot { + c.storeX87(i, "0.000000e+00") + } + fmt.Fprintf(c.b, " store i16 895, ptr %s\n", c.x87ControlSlot) + fmt.Fprintf(c.b, " store i16 0, ptr %s\n", c.x87StatusSlot) + return true, false, nil + + case "FNOP": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FNOP takes no operands: %q", ins.Raw) + } + return true, false, nil + + case "FPATAN": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FPATAN takes no operands: %q", ins.Raw) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @atan2(double %s, double %s)\n", result, c.loadX87(1), c.loadX87(0)) + c.storeX87(1, "%"+result) + c.popX87() + return true, false, nil + + case "FPREM", "FPREM1": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + function := "fmod" + if op == "FPREM1" { + function = "remainder" + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @%s(double %s, double %s)\n", result, function, c.loadX87(0), c.loadX87(1)) + c.storeX87(0, "%"+result) + c.clearX87C2() + return true, false, nil + + case "FPTAN": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FPTAN takes no operands: %q", ins.Raw) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @tan(double %s)\n", result, c.loadX87(0)) + c.storeX87(0, "%"+result) + if err := c.pushX87("1.000000e+00"); err != nil { return true, false, err } - rhs, err := c.evalX87F64(ins.Args[1]) - if err != nil { + c.clearX87C2() + return true, false, nil + + case "FSCALE": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FSCALE takes no operands: %q", ins.Raw) + } + exponent := c.newTmp() + scale := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.trunc.f64(double %s)\n", exponent, c.loadX87(1)) + fmt.Fprintf(c.b, " %%%s = call double @llvm.exp2.f64(double %%%s)\n", scale, exponent) + fmt.Fprintf(c.b, " %%%s = fmul double %s, %%%s\n", result, c.loadX87(0), scale) + c.storeX87(0, "%"+result) + return true, false, nil + + case "FSINCOS": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FSINCOS takes no operands: %q", ins.Raw) + } + value := c.loadX87(0) + sine := c.newTmp() + cosine := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.sin.f64(double %s)\n", sine, value) + fmt.Fprintf(c.b, " %%%s = call double @llvm.cos.f64(double %s)\n", cosine, value) + c.storeX87(0, "%"+sine) + if err := c.pushX87("%" + cosine); err != nil { return true, false, err } - z := c.newTmp() - cf := c.newTmp() - fmt.Fprintf(c.b, " %%%s = fcmp ueq double %s, %s\n", z, lhs, rhs) - fmt.Fprintf(c.b, " %%%s = fcmp ult double %s, %s\n", cf, lhs, rhs) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", z, c.flagsZSlot) - fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", cf, c.flagsCFSlot) - // FUCOMI clears both SF and OF. flagsSltSlot models SF xor OF for - // signed conditions, so those conditions must observe false here. - fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsSltSlot) + c.clearX87C2() + return true, false, nil + + case "FXAM": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FXAM takes no operands: %q", ins.Raw) + } + c.classifyX87F0() + return true, false, nil + + case "FXTRACT": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 FXTRACT takes no operands: %q", ins.Raw) + } + value := c.loadX87(0) + abs := c.newTmp() + log := c.newTmp() + exponent := c.newTmp() + scale := c.newTmp() + significand := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.fabs.f64(double %s)\n", abs, value) + fmt.Fprintf(c.b, " %%%s = call double @llvm.log2.f64(double %%%s)\n", log, abs) + fmt.Fprintf(c.b, " %%%s = call double @llvm.floor.f64(double %%%s)\n", exponent, log) + fmt.Fprintf(c.b, " %%%s = call double @llvm.exp2.f64(double %%%s)\n", scale, exponent) + fmt.Fprintf(c.b, " %%%s = fdiv double %s, %%%s\n", significand, value, scale) + c.storeX87(0, "%"+significand) + return true, false, c.pushX87("%" + exponent) + + case "FYL2X", "FYL2XP1": + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + x := c.loadX87(0) + if op == "FYL2XP1" { + incremented := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fadd double %s, 1.000000e+00\n", incremented, x) + x = "%" + incremented + } + log := c.newTmp() + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = call double @llvm.log2.f64(double %s)\n", log, x) + fmt.Fprintf(c.b, " %%%s = fmul double %s, %%%s\n", result, c.loadX87(1), log) + c.storeX87(1, "%"+result) + c.popX87() return true, false, nil case "FTST": @@ -246,22 +499,252 @@ func (c *amd64Ctx) lowerX87(op Op, ins Instr) (ok bool, terminated bool, err err return true, false, nil case "FSTSW": - if len(ins.Args) != 1 { - return true, false, fmt.Errorf("386 FSTSW expects dst: %q", ins.Raw) + if len(ins.Args) != 1 || !(isX87MemoryOperand(ins.Args[0]) || ins.Args[0].Kind == OpReg && ins.Args[0].Reg == AX) { + return true, false, fmt.Errorf("386 FSTSW expects memory destination or AX: %q", ins.Raw) } status := c.newTmp() fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) return true, false, c.storeX87I16(ins.Args[0], "%"+status) - case "FLD1": + case "FLD1", "FLDL2E", "FLDL2T", "FLDLG2", "FLDLN2", "FLDPI", "FLDZ": if len(ins.Args) != 0 { - return true, false, fmt.Errorf("386 FLD1 takes no operands: %q", ins.Raw) + return true, false, fmt.Errorf("386 %s takes no operands: %q", op, ins.Raw) + } + constant := map[Op]string{ + "FLD1": "1.0000000000000000e+00", + "FLDL2E": "1.4426950408889634e+00", + "FLDL2T": "3.3219280948873623e+00", + "FLDLG2": "3.0102999566398120e-01", + "FLDLN2": "6.9314718055994531e-01", + "FLDPI": "3.1415926535897931e+00", + "FLDZ": "0.0000000000000000e+00", } - return true, false, c.pushX87("1.000000e+00") + return true, false, c.pushX87(constant[op]) } return false, false, nil } +func (c *amd64Ctx) lowerX87Arithmetic(op Op, ins Instr) error { + if len(ins.Args) != 2 { + return fmt.Errorf("386 %s expects src, dst: %q", op, ins.Raw) + } + name := string(op) + pop := strings.HasSuffix(name, "DP") + width := name[len(name)-1:] + base := strings.TrimSuffix(name, width) + if pop { + width = "D" + base = strings.TrimSuffix(name, "DP") + } + + if pop { + if !isX87F0(ins.Args[0]) || !isX87RegisterOperand(ins.Args[1]) { + return fmt.Errorf("386 %s expects F0 source and x87-register destination: %q", op, ins.Raw) + } + } else if width == "D" { + srcIndex, srcReg := x87OperandReg(ins.Args[0]) + dstIndex, dstReg := x87OperandReg(ins.Args[1]) + loadToF0 := (isX87MemoryOperand(ins.Args[0]) || ins.Args[0].Kind == OpImm) && dstReg && dstIndex == 0 + regToF0 := srcReg && dstReg && dstIndex == 0 + f0ToReg := srcReg && srcIndex == 0 && dstReg + if !loadToF0 && !regToF0 && !f0ToReg { + return fmt.Errorf("386 %s expects memory,F0, Freg,F0, or F0,Freg: %q", op, ins.Raw) + } + } else if !isX87MemoryOperand(ins.Args[0]) || !isX87F0(ins.Args[1]) { + return fmt.Errorf("386 %s expects memory source and F0 destination: %q", op, ins.Raw) + } + + var src string + var err error + switch width { + case "W", "L": + typ := I16 + if width == "L" { + typ = I32 + } + integer, evalErr := c.evalIntSized(ins.Args[0], typ) + if evalErr != nil { + return evalErr + } + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp %s %s to double\n", converted, typ, integer) + src = "%" + converted + case "F": + value, evalErr := c.evalF32(ins.Args[0]) + if evalErr != nil { + return evalErr + } + converted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", converted, value) + src = "%" + converted + case "D": + src, err = c.evalX87F64(ins.Args[0]) + if err != nil { + return err + } + default: + return fmt.Errorf("386 %s has unknown x87 arithmetic width", op) + } + dst, err := c.evalX87F64(ins.Args[1]) + if err != nil { + return err + } + lhs, rhs := dst, src + if strings.HasPrefix(base, "FSUBR") || strings.HasPrefix(base, "FDIVR") { + lhs, rhs = src, dst + } + operation := "" + switch { + case strings.HasPrefix(base, "FADD"): + operation = "fadd" + case strings.HasPrefix(base, "FMUL"): + operation = "fmul" + case strings.HasPrefix(base, "FSUB"): + operation = "fsub" + case strings.HasPrefix(base, "FDIV"): + operation = "fdiv" + } + if operation == "" { + return fmt.Errorf("386 %s has unknown x87 arithmetic operation", op) + } + result := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s double %s, %s\n", result, operation, lhs, rhs) + if err := c.storeX87F64(ins.Args[1], "%"+result); err != nil { + return err + } + if pop { + c.popX87() + } + return nil +} + +func (c *amd64Ctx) lowerX87Compare(op Op, ins Instr) error { + if len(ins.Args) != 2 { + return fmt.Errorf("386 %s expects lhs, rhs: %q", op, ins.Raw) + } + name := string(op) + directFlags := name == "FCOMI" || name == "FCOMIP" || name == "FUCOMI" || name == "FUCOMIP" + popCount := 0 + switch name { + case "FCOMDP", "FCOMFP", "FCOMIP", "FCOMLP", "FCOMWP", "FUCOMIP", "FUCOMP": + popCount = 1 + case "FCOMDPP", "FUCOMPP": + popCount = 2 + } + + var lhsOperand, rhsOperand Operand + width := "D" + switch { + case name == "FCOMI" || name == "FCOMIP": + if !isX87RegisterOperand(ins.Args[0]) || !isX87F0(ins.Args[1]) { + return fmt.Errorf("386 %s expects x87-register source and F0 destination: %q", op, ins.Raw) + } + lhsOperand, rhsOperand = ins.Args[1], ins.Args[0] + case strings.HasPrefix(name, "FUCOM") || name == "FCOMDPP": + if !isX87F0(ins.Args[0]) || !isX87RegisterOperand(ins.Args[1]) { + return fmt.Errorf("386 %s expects F0 source and x87-register destination: %q", op, ins.Raw) + } + lhsOperand, rhsOperand = ins.Args[0], ins.Args[1] + case name == "FCOMD" || name == "FCOMDP": + srcIndex, srcReg := x87OperandReg(ins.Args[0]) + dstIndex, dstReg := x87OperandReg(ins.Args[1]) + loadToF0 := (isX87MemoryOperand(ins.Args[0]) || ins.Args[0].Kind == OpImm) && dstReg && dstIndex == 0 + regToF0 := srcReg && dstReg && dstIndex == 0 + f0ToReg := srcReg && srcIndex == 0 && dstReg + if !loadToF0 && !regToF0 && !f0ToReg { + return fmt.Errorf("386 %s expects memory,F0, Freg,F0, or F0,Freg: %q", op, ins.Raw) + } + lhsOperand, rhsOperand = ins.Args[1], ins.Args[0] + default: + widthIndex := len("FCOM") + if len(name) <= widthIndex { + return fmt.Errorf("386 %s has unknown x87 compare form", op) + } + width = name[widthIndex : widthIndex+1] + if !isX87MemoryOperand(ins.Args[0]) || !isX87F0(ins.Args[1]) { + return fmt.Errorf("386 %s expects memory source and F0 destination: %q", op, ins.Raw) + } + lhsOperand, rhsOperand = ins.Args[1], ins.Args[0] + } + + lhs, err := c.evalX87F64(lhsOperand) + if err != nil { + return err + } + var rhs string + switch width { + case "D": + rhs, err = c.evalX87F64(rhsOperand) + case "F": + var narrow string + narrow, err = c.evalF32(rhsOperand) + if err == nil { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", wide, narrow) + rhs = "%" + wide + } + case "L", "W": + typ := I16 + if width == "L" { + typ = I32 + } + var integer string + integer, err = c.evalIntSized(rhsOperand, typ) + if err == nil { + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp %s %s to double\n", wide, typ, integer) + rhs = "%" + wide + } + default: + return fmt.Errorf("386 %s has unknown x87 compare width", op) + } + if err != nil { + return err + } + if directFlags { + c.setX87CompareFlags(lhs, rhs) + } else { + c.setX87CompareStatus(lhs, rhs) + } + for i := 0; i < popCount; i++ { + c.popX87() + } + return nil +} + +func (c *amd64Ctx) x87CompareBits(lhs, rhs string) (equal, less, unordered string) { + equalTmp := c.newTmp() + lessTmp := c.newTmp() + unorderedTmp := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fcmp ueq double %s, %s\n", equalTmp, lhs, rhs) + fmt.Fprintf(c.b, " %%%s = fcmp ult double %s, %s\n", lessTmp, lhs, rhs) + fmt.Fprintf(c.b, " %%%s = fcmp uno double %s, %s\n", unorderedTmp, lhs, rhs) + return equalTmp, lessTmp, unorderedTmp +} + +func (c *amd64Ctx) setX87CompareFlags(lhs, rhs string) { + equal, less, unordered := c.x87CompareBits(lhs, rhs) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", equal, c.flagsZSlot) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", less, c.flagsCFSlot) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", unordered, c.flagsPFSlot) + // FCOMI/FUCOMI clear OF, SF, and AF. flagsSltSlot is architectural SF. + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsSltSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsOFSlot) + fmt.Fprintf(c.b, " store i1 false, ptr %s\n", c.flagsAFSlot) +} + +func (c *amd64Ctx) setX87CompareStatus(lhs, rhs string) { + equal, less, unordered := c.x87CompareBits(lhs, rhs) + c3Word := c.x87StatusBit(equal, 16384) + c2Word := c.x87StatusBit(unordered, 1024) + c0Word := c.x87StatusBit(less, 256) + partial := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i16 %s, %s\n", partial, c3Word, c2Word) + fmt.Fprintf(c.b, " %%%s = or i16 %%%s, %s\n", status, partial, c0Word) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", status, c.x87StatusSlot) +} + func x87OperandReg(op Operand) (int, bool) { if op.Kind != OpReg { return 0, false @@ -269,6 +752,244 @@ func x87OperandReg(op Operand) (int, bool) { return amd64ParseX87Reg(op.Reg) } +func isX87RegisterOperand(op Operand) bool { + _, ok := x87OperandReg(op) + return ok +} + +func isX87F0(op Operand) bool { + index, ok := x87OperandReg(op) + return ok && index == 0 +} + +func isX87MemoryOperand(op Operand) bool { + return isAMD64MemoryOperand(op) +} + +func (c *amd64Ctx) loadX87MoveValue(op Op, src Operand) (string, error) { + switch op { + case "FMOVF": + value, err := c.evalF32(src) + if err != nil { + return "", err + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext float %s to double\n", wide, value) + return "%" + wide, nil + case "FMOVL", "FMOVW": + typ := I32 + if op == "FMOVW" { + typ = I16 + } + value, err := c.evalIntSized(src, typ) + if err != nil { + return "", err + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sitofp %s %s to double\n", wide, typ, value) + return "%" + wide, nil + default: + return "", fmt.Errorf("386: unsupported x87 move load %s", op) + } +} + +func (c *amd64Ctx) storeX87MoveValue(op Op, dst Operand, value string) error { + switch op { + case "FMOVF": + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fptrunc double %s to float\n", narrow, value) + return c.storeX87TypedMemory(dst, LLVMType("float"), "%"+narrow) + case "FMOVL", "FMOVW": + typ := I32 + if op == "FMOVW" { + typ = I16 + } + rounded := c.roundX87Software(value) + integer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fptosi double %s to %s\n", integer, rounded, typ) + return c.storeX87TypedMemory(dst, typ, "%"+integer) + default: + return fmt.Errorf("386: unsupported x87 move store %s", op) + } +} + +func (c *amd64Ctx) storeX87TypedMemory(dst Operand, typ LLVMType, value string) error { + if dst.Kind == OpFP { + return c.storeFPResult(dst.FPOffset, typ, value) + } + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + fmt.Fprintf(c.b, " store %s %s, %s %s, align 1\n", typ, value, ptrType, ptr) + return nil +} + +func (c *amd64Ctx) x87MemoryPointer(op Operand) (string, string, error) { + if !isX87MemoryOperand(op) { + return "", "", fmt.Errorf("386 x87 instruction expected memory, got %s", op.String()) + } + ptr, ptrType, err := c.x86DescriptorMemoryPointer(op) + if err != nil { + return "", "", fmt.Errorf("386 x87 memory operand %s: %w", op.String(), err) + } + return ptr, ptrType, nil +} + +func (c *amd64Ctx) loadX87Environment(src Operand) error { + ptr, ptrType, err := c.x87MemoryPointer(src) + if err != nil { + return err + } + control := c.newTmp() + statusPtr := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, %s %s, align 1\n", control, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 4\n", statusPtr, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = load i16, %s %%%s, align 1\n", status, ptrType, statusPtr) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", control, c.x87ControlSlot) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", status, c.x87StatusSlot) + if c.useHardwareX87() { + c.loadHardwareX87ControlWord() + } + return nil +} + +func (c *amd64Ctx) storeX87Environment(dst Operand) error { + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + if c.useHardwareX87() { + c.storeHardwareX87ControlWord() + } + control := c.newTmp() + status := c.newTmp() + statusPtr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", control, c.x87ControlSlot) + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) + fmt.Fprintf(c.b, " store i16 %%%s, %s %s, align 1\n", control, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 4\n", statusPtr, ptrType, ptr) + fmt.Fprintf(c.b, " store i16 %%%s, %s %%%s, align 1\n", status, ptrType, statusPtr) + return nil +} + +func (c *amd64Ctx) loadX87Extended(src Operand) (string, error) { + ptr, ptrType, err := c.x87MemoryPointer(src) + if err != nil { + return "", err + } + extended := c.newTmp() + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load x86_fp80, %s %s, align 1\n", extended, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = fptrunc x86_fp80 %%%s to double\n", value, extended) + return "%" + value, nil +} + +func (c *amd64Ctx) storeX87Extended(dst Operand, value string) error { + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + extended := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fpext double %s to x86_fp80\n", extended, value) + fmt.Fprintf(c.b, " store x86_fp80 %%%s, %s %s, align 1\n", extended, ptrType, ptr) + return nil +} + +func (c *amd64Ctx) loadX87PackedBCD(src Operand) (string, error) { + ptr, ptrType, err := c.x87MemoryPointer(src) + if err != nil { + return "", err + } + accumulator := "0" + for i := 8; i >= 0; i-- { + bytePtr := c.newTmp() + packed := c.newTmp() + low := c.newTmp() + highShifted := c.newTmp() + high := c.newTmp() + low64 := c.newTmp() + high64 := c.newTmp() + digit10 := c.newTmp() + digits := c.newTmp() + scaled := c.newTmp() + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", bytePtr, ptrType, ptr, i) + fmt.Fprintf(c.b, " %%%s = load i8, %s %%%s, align 1\n", packed, ptrType, bytePtr) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, 15\n", low, packed) + fmt.Fprintf(c.b, " %%%s = lshr i8 %%%s, 4\n", highShifted, packed) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, 15\n", high, highShifted) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", low64, low) + fmt.Fprintf(c.b, " %%%s = zext i8 %%%s to i64\n", high64, high) + fmt.Fprintf(c.b, " %%%s = mul i64 %%%s, 10\n", digit10, high64) + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", digits, digit10, low64) + fmt.Fprintf(c.b, " %%%s = mul i64 %s, 100\n", scaled, accumulator) + fmt.Fprintf(c.b, " %%%s = add i64 %%%s, %%%s\n", next, scaled, digits) + accumulator = "%" + next + } + signPtr := c.newTmp() + signByte := c.newTmp() + signBits := c.newTmp() + isNegative := c.newTmp() + negative := c.newTmp() + signed := c.newTmp() + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 9\n", signPtr, ptrType, ptr) + fmt.Fprintf(c.b, " %%%s = load i8, %s %%%s, align 1\n", signByte, ptrType, signPtr) + fmt.Fprintf(c.b, " %%%s = and i8 %%%s, -128\n", signBits, signByte) + fmt.Fprintf(c.b, " %%%s = icmp ne i8 %%%s, 0\n", isNegative, signBits) + fmt.Fprintf(c.b, " %%%s = sub i64 0, %s\n", negative, accumulator) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %s\n", signed, isNegative, negative, accumulator) + fmt.Fprintf(c.b, " %%%s = sitofp i64 %%%s to double\n", value, signed) + return "%" + value, nil +} + +func (c *amd64Ctx) storeX87PackedBCD(dst Operand, value string) error { + ptr, ptrType, err := c.x87MemoryPointer(dst) + if err != nil { + return err + } + rounded := c.roundX87Software(value) + integer := c.newTmp() + isNegative := c.newTmp() + negated := c.newTmp() + absolute := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fptosi double %s to i64\n", integer, rounded) + fmt.Fprintf(c.b, " %%%s = icmp slt i64 %%%s, 0\n", isNegative, integer) + fmt.Fprintf(c.b, " %%%s = sub i64 0, %%%s\n", negated, integer) + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i64 %%%s, i64 %%%s\n", absolute, isNegative, negated, integer) + remaining := "%" + absolute + for i := 0; i < 9; i++ { + digits := c.newTmp() + tens := c.newTmp() + ones := c.newTmp() + tensByte := c.newTmp() + onesByte := c.newTmp() + shifted := c.newTmp() + packed := c.newTmp() + bytePtr := c.newTmp() + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = urem i64 %s, 100\n", digits, remaining) + fmt.Fprintf(c.b, " %%%s = udiv i64 %%%s, 10\n", tens, digits) + fmt.Fprintf(c.b, " %%%s = urem i64 %%%s, 10\n", ones, digits) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i8\n", tensByte, tens) + fmt.Fprintf(c.b, " %%%s = trunc i64 %%%s to i8\n", onesByte, ones) + fmt.Fprintf(c.b, " %%%s = shl i8 %%%s, 4\n", shifted, tensByte) + fmt.Fprintf(c.b, " %%%s = or i8 %%%s, %%%s\n", packed, shifted, onesByte) + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", bytePtr, ptrType, ptr, i) + fmt.Fprintf(c.b, " store i8 %%%s, %s %%%s, align 1\n", packed, ptrType, bytePtr) + fmt.Fprintf(c.b, " %%%s = udiv i64 %s, 100\n", next, remaining) + remaining = "%" + next + } + sign := c.newTmp() + signPtr := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i8 -128, i8 0\n", sign, isNegative) + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 9\n", signPtr, ptrType, ptr) + fmt.Fprintf(c.b, " store i8 %%%s, %s %%%s, align 1\n", sign, ptrType, signPtr) + return nil +} + func (c *amd64Ctx) loadX87(index int) string { t := c.newTmp() fmt.Fprintf(c.b, " %%%s = load double, ptr %s\n", t, c.x87Slot[index]) @@ -294,6 +1015,73 @@ func (c *amd64Ctx) popX87() { c.storeX87(len(c.x87Slot)-1, "0.000000e+00") } +func (c *amd64Ctx) rotateX87Stack(increment bool) { + values := make([]string, len(c.x87Slot)) + for i := range values { + values[i] = c.loadX87(i) + } + if increment { + for i := range values { + c.storeX87(i, values[(i+1)%len(values)]) + } + return + } + for i := range values { + c.storeX87(i, values[(i+len(values)-1)%len(values)]) + } +} + +func (c *amd64Ctx) clearX87C2() { + status := c.newTmp() + cleared := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i16, ptr %s\n", status, c.x87StatusSlot) + fmt.Fprintf(c.b, " %%%s = and i16 %%%s, -1025\n", cleared, status) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", cleared, c.x87StatusSlot) +} + +func (c *amd64Ctx) classifyX87F0() { + value := c.loadX87(0) + isNaN := c.newTmp() + isZero := c.newTmp() + abs := c.newTmp() + isInfinity := c.newTmp() + notNaN := c.newTmp() + notZero := c.newTmp() + c2 := c.newTmp() + c0 := c.newTmp() + bits := c.newTmp() + signBits := c.newTmp() + isNegative := c.newTmp() + fmt.Fprintf(c.b, " %%%s = fcmp uno double %s, %s\n", isNaN, value, value) + fmt.Fprintf(c.b, " %%%s = fcmp oeq double %s, 0.000000e+00\n", isZero, value) + fmt.Fprintf(c.b, " %%%s = call double @llvm.fabs.f64(double %s)\n", abs, value) + fmt.Fprintf(c.b, " %%%s = fcmp oeq double %%%s, 0x7FF0000000000000\n", isInfinity, abs) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notNaN, isNaN) + fmt.Fprintf(c.b, " %%%s = xor i1 %%%s, true\n", notZero, isZero) + fmt.Fprintf(c.b, " %%%s = and i1 %%%s, %%%s\n", c2, notNaN, notZero) + fmt.Fprintf(c.b, " %%%s = or i1 %%%s, %%%s\n", c0, isNaN, isInfinity) + fmt.Fprintf(c.b, " %%%s = bitcast double %s to i64\n", bits, value) + fmt.Fprintf(c.b, " %%%s = and i64 %%%s, -9223372036854775808\n", signBits, bits) + fmt.Fprintf(c.b, " %%%s = icmp ne i64 %%%s, 0\n", isNegative, signBits) + c3Word := c.x87StatusBit(isZero, 16384) + c2Word := c.x87StatusBit(c2, 1024) + c1Word := c.x87StatusBit(isNegative, 512) + c0Word := c.x87StatusBit(c0, 256) + first := c.newTmp() + second := c.newTmp() + status := c.newTmp() + fmt.Fprintf(c.b, " %%%s = or i16 %s, %s\n", first, c3Word, c2Word) + fmt.Fprintf(c.b, " %%%s = or i16 %%%s, %s\n", second, first, c1Word) + fmt.Fprintf(c.b, " %%%s = or i16 %%%s, %s\n", status, second, c0Word) + fmt.Fprintf(c.b, " store i16 %%%s, ptr %s\n", status, c.x87StatusSlot) +} + +func (c *amd64Ctx) x87StatusBit(condition string, mask int) string { + word := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %%%s, i16 %d, i16 0\n", word, condition, mask) + return "%" + word +} + func (c *amd64Ctx) evalX87F64(op Operand) (string, error) { if index, ok := x87OperandReg(op); ok { return c.loadX87(index), nil @@ -332,6 +1120,8 @@ func (c *amd64Ctx) storeX87I16(dst Operand, value string) error { switch dst.Kind { case OpReg: return c.storeRegSized(dst.Reg, I16, value) + case OpFP: + return c.storeFPResult(dst.FPOffset, I16, value) case OpMem: p, ptrType, err := c.ptrFromMem(dst.Mem) if err != nil { diff --git a/amd64_lower_x87_conditional_move.go b/amd64_lower_x87_conditional_move.go new file mode 100644 index 000000000..f0d4d9c6b --- /dev/null +++ b/amd64_lower_x87_conditional_move.go @@ -0,0 +1,56 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerX87ConditionalMove(op Op, ins Instr) error { + if len(ins.Args) != 2 { + return fmt.Errorf("%s %s expects one x87 source and F0 destination: %q", c.goarch, op, ins.Raw) + } + source, sourceOK := x87OperandReg(ins.Args[0]) + destination, destinationOK := x87OperandReg(ins.Args[1]) + if !sourceOK || !destinationOK || destination != 0 { + return fmt.Errorf("%s %s operands are outside Go 1.27's Yrf,F0 row: %q", c.goarch, op, ins.Raw) + } + + cf := func() string { return c.loadFlag(c.flagsCFSlot) } + zf := func() string { return c.loadFlag(c.flagsZSlot) } + pf := func() string { return c.loadFlag(c.flagsPFSlot) } + not := func(value string) string { + inverted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = xor i1 %s, true\n", inverted, value) + return "%" + inverted + } + combine := func(instruction, left, right string) string { + condition := c.newTmp() + fmt.Fprintf(c.b, " %%%s = %s i1 %s, %s\n", condition, instruction, left, right) + return "%" + condition + } + + condition := "" + switch op { + case "FCMOVB", "FCMOVCS": + condition = cf() + case "FCMOVBE", "FCMOVLS": + condition = combine("or", cf(), zf()) + case "FCMOVE", "FCMOVEQ": + condition = zf() + case "FCMOVNB", "FCMOVCC": + condition = not(cf()) + case "FCMOVNBE", "FCMOVHI": + condition = combine("and", not(cf()), not(zf())) + case "FCMOVNE": + condition = not(zf()) + case "FCMOVNU": + condition = not(pf()) + case "FCMOVU", "FCMOVUN": + condition = pf() + default: + return fmt.Errorf("unsupported x87 conditional move %s", op) + } + sourceValue := c.loadX87(source) + destinationValue := c.loadX87(0) + selected := c.newTmp() + fmt.Fprintf(c.b, " %%%s = select i1 %s, double %s, double %s\n", selected, condition, sourceValue, destinationValue) + c.storeX87(0, "%"+selected) + return nil +} diff --git a/amd64_lower_x87_conditional_move_test.go b/amd64_lower_x87_conditional_move_test.go new file mode 100644 index 000000000..7d05c6a94 --- /dev/null +++ b/amd64_lower_x87_conditional_move_test.go @@ -0,0 +1,203 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +var x87ConditionalMoveGo127Ops = []string{ + "FCMOVCC", "FCMOVCS", "FCMOVEQ", "FCMOVHI", "FCMOVLS", + "FCMOVB", "FCMOVBE", "FCMOVNB", "FCMOVNBE", "FCMOVE", + "FCMOVNE", "FCMOVNU", "FCMOVU", "FCMOVUN", +} + +func x86X87ConditionalMoveCompleteFormsSource() string { + var source strings.Builder + source.WriteString("TEXT x87conditionalmoveforms(SB),$0-0\n") + for _, op := range x87ConditionalMoveGo127Ops { + source.WriteString("\t" + op + " F0, F0\n") + source.WriteString("\t" + op + " F7, F0\n") + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86X87ConditionalMoveCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86X87ConditionalMoveCompleteFormsSource() + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"x87conditionalmoveforms": {Name: "x87conditionalmoveforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"select i1", "or i1", "and i1", "xor i1"} { + if !strings.Contains(ir, want) { + t.Fatalf("x87 conditional-move lowering omitted %q:\n%s", want, ir) + } + } + compileLLVMToObject(t, llc, target.triple, "x87-conditional-move-"+target.name+".ll", "x87-conditional-move-"+target.name+".o", ir) + }) + } +} + +func TestTranslateX86X87ConditionalMoveRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, instruction := range []string{ + "FCMOVB 0(AX), F0", + "FCMOVBE F8, F0", + "FCMOVE F1, F2", + "FCMOVNB F1", + "FCMOVNBE F1, F0, F2", + "FCMOVU.Z F1, F0", + } { + for _, goarch := range []string{"amd64", "386"} { + t.Run(goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's yfcmv table", instruction) + } + }) + } + } +} + +func TestAMD64X87ConditionalMoveRuntimeConditions(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT x87conditionalmovesemantics(SB),NOSPLIT,$0-8 + MOVQ data+0(FP), DI + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + STC + FCMOVB F1, F0 + FMOVDP F0, 0(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + STC + FCMOVNB F1, F0 + FMOVDP F0, 8(DI) + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVE F1, F0 + FMOVDP F0, 16(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVNE F1, F0 + FMOVDP F0, 24(DI) + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVBE F1, F0 + FMOVDP F0, 32(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVNBE F1, F0 + FMOVDP F0, 40(DI) + + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVU F1, F0 + FMOVDP F0, 48(DI) + FINIT + FMOVD 64(DI), F0 + FMOVD 72(DI), F0 + XORL AX, AX + FCMOVNU F1, F0 + FMOVDP F0, 56(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "x87conditionalmovesemantics": { + Name: "x87conditionalmovesemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void x87conditionalmovesemantics(uint8_t *); +int main(void) { + double data[10] = {0}; + data[8] = 1.25; + data[9] = 9.5; + x87conditionalmovesemantics((uint8_t *)data); + const double expected[8] = {1.25, 9.5, 1.25, 9.5, 1.25, 9.5, 1.25, 9.5}; + for (int i = 0; i < 8; ++i) if (data[i] != expected[i]) return 10+i; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "x87_conditional_move_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_lower_xadd_test.go b/amd64_lower_xadd_test.go new file mode 100644 index 000000000..0ac37e397 --- /dev/null +++ b/amd64_lower_xadd_test.go @@ -0,0 +1,296 @@ +package plan9asm + +import ( + "runtime" + "strings" + "testing" +) + +func TestTranslateX86XADDCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + source string + }{ + { + name: "darwin-amd64", + goarch: "amd64", + triple: "x86_64-apple-darwin", + source: ` +DATA xadddata+0(SB)/8, $0 +GLOBL xadddata(SB), $8 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB R11, R12 + XADDB DL, 8(BX) + XADDB R11, xadddata(SB) + XADDW DX, BX + XADDW R11, R12 + XADDW DX, 8(BX) + XADDW R11, xadddata(SB) + XADDL DX, BX + XADDL R11, R12 + XADDL DX, 8(BX) + XADDL R11, xadddata(SB) + XADDQ DX, BX + XADDQ R11, R12 + XADDQ DX, 8(BX) + XADDQ R11, xadddata(SB) + RET +`, + }, + { + name: "linux-amd64", + goarch: "amd64", + triple: "x86_64-unknown-linux-gnu", + source: ` +DATA xadddata+0(SB)/8, $0 +GLOBL xadddata(SB), $8 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB R11, R12 + XADDB DL, 8(BX) + XADDB R11, xadddata(SB) + XADDW DX, BX + XADDW R11, R12 + XADDW DX, 8(BX) + XADDW R11, xadddata(SB) + XADDL DX, BX + XADDL R11, R12 + XADDL DX, 8(BX) + XADDL R11, xadddata(SB) + XADDQ DX, BX + XADDQ R11, R12 + XADDQ DX, 8(BX) + XADDQ R11, xadddata(SB) + RET +`, + }, + { + name: "windows-amd64", + goarch: "amd64", + triple: "x86_64-pc-windows-msvc", + source: ` +DATA xadddata+0(SB)/8, $0 +GLOBL xadddata(SB), $8 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB R11, R12 + XADDB DL, 8(BX) + XADDB R11, xadddata(SB) + XADDW DX, BX + XADDW R11, R12 + XADDW DX, 8(BX) + XADDW R11, xadddata(SB) + XADDL DX, BX + XADDL R11, R12 + XADDL DX, 8(BX) + XADDL R11, xadddata(SB) + XADDQ DX, BX + XADDQ R11, R12 + XADDQ DX, 8(BX) + XADDQ R11, xadddata(SB) + RET +`, + }, + { + name: "linux-386", + goarch: "386", + triple: "i386-unknown-linux-gnu", + source: ` +DATA xadddata386+0(SB)/4, $0 +GLOBL xadddata386(SB), $4 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB BP, DI + XADDB DL, 8(BX) + XADDB SI, xadddata386(SB) + XADDW DX, BX + XADDW SI, DI + XADDW DX, 8(BX) + XADDW SI, xadddata386(SB) + XADDL DX, BX + XADDL SI, DI + XADDL DX, 8(BX) + XADDL SI, xadddata386(SB) + RET +`, + }, + { + name: "windows-386", + goarch: "386", + triple: "i686-pc-windows-msvc", + source: ` +DATA xadddata386+0(SB)/4, $0 +GLOBL xadddata386(SB), $4 +TEXT xaddforms(SB),$0-0 + XADDB AH, BL + XADDB BP, DI + XADDB DL, 8(BX) + XADDB SI, xadddata386(SB) + XADDW DX, BX + XADDW SI, DI + XADDW DX, 8(BX) + XADDW SI, xadddata386(SB) + XADDL DX, BX + XADDL SI, DI + XADDL DX, 8(BX) + XADDL SI, xadddata386(SB) + RET +`, + }, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, target.source, true) + file, err := Parse(ArchAMD64, target.source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"xaddforms": {Name: "xaddforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"add i8", "add i16", "add i32", "store i1"} { + if !strings.Contains(ll, want) { + t.Fatalf("XADD lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "add i64") { + t.Fatalf("XADDQ lowering omitted add i64:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "xadd-"+target.name+".ll", "xadd-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86XADDRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "XADDB (BX), DL"}, + {goarch: "amd64", instruction: "XADDW $1, DX"}, + {goarch: "amd64", instruction: "XADDL X0, DX"}, + {goarch: "amd64", instruction: "XADDQ DX, $1"}, + {goarch: "amd64", instruction: "XADDQ DX"}, + {goarch: "amd64", instruction: "XADDL DX, BX, AX"}, + {goarch: "amd64", instruction: "XADDL.Z DX, BX"}, + {goarch: "386", instruction: "XADDB SP, (BX)"}, + {goarch: "386", instruction: "XADDL R8, (BX)"}, + {goarch: "386", instruction: "XADDL DX, 1(R8)"}, + {goarch: "386", instruction: "XADDQ DX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's XADD optab", test.instruction) + } + }) + } +} + +func TestAMD64XADDRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT xaddsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $5, AX + MOVQ $7, BX + XADDQ AX, BX + MOVQ AX, 0(DI) + MOVQ BX, 8(DI) + MOVB $0xff, 16(DI) + MOVB $1, CL + XADDB CL, 16(DI) + MOVB CL, 17(DI) + SETCS 18(DI) + SETEQ 19(DI) + SETPS 20(DI) + SETMI 21(DI) + SETOS 22(DI) + MOVQ $0x1122, AX + XADDB AH, AL + MOVQ AX, 24(DI) + MOVQ $0x1122334455660001, AX + MOVQ $0x887766554433ffff, BX + XADDW AX, BX + MOVQ AX, 32(DI) + MOVQ BX, 40(DI) + MOVQ $3, AX + XADDQ AX, AX + MOVQ AX, 48(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "xaddsemantics": { + Name: "xaddsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void xaddsemantics(uint8_t *); +int main(void) { + uint8_t out[56] = {0}; + xaddsemantics(out); + if (*(uint64_t *)(out+0) != 7 || *(uint64_t *)(out+8) != 12) return 10; + if (out[16] != 0 || out[17] != 0xff) return 11; + if (out[18] != 1 || out[19] != 1 || out[20] != 1) return 12; + if (out[21] != 0 || out[22] != 0) return 13; + if (*(uint64_t *)(out+24) != 0x2233) return 14; + if (*(uint64_t *)(out+32) != UINT64_C(0x112233445566ffff)) return 15; + if (*(uint64_t *)(out+40) != UINT64_C(0x8877665544330000)) return 16; + if (*(uint64_t *)(out+48) != 6) return 17; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "xadd_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_xchg_test.go b/amd64_lower_xchg_test.go new file mode 100644 index 000000000..e846cf994 --- /dev/null +++ b/amd64_lower_xchg_test.go @@ -0,0 +1,222 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func x86XCHGCompleteFormsSource(goarch string) string { + var source strings.Builder + symbol := "xchgdata" + size := 8 + if goarch == "386" { + symbol = "xchgdata386" + size = 4 + } + fmt.Fprintf(&source, "DATA %s+0(SB)/%d, $0\n", symbol, size) + fmt.Fprintf(&source, "GLOBL %s(SB), $%d\n", symbol, size) + source.WriteString("TEXT xchgforms(SB),$0-0\n") + if goarch == "amd64" { + source.WriteString("\tXCHGB AH, BL\n\tXCHGB R11, R12\n") + } else { + source.WriteString("\tXCHGB AH, BL\n\tXCHGB BP, DI\n") + } + source.WriteString("\tXCHGB DL, 8(BX)\n\tXCHGB 8(BX), DL\n") + if goarch == "amd64" { + source.WriteString("\tXCHGB R11, " + symbol + "(SB)\n\tXCHGB " + symbol + "(SB), R11\n") + } else { + source.WriteString("\tXCHGB SI, " + symbol + "(SB)\n\tXCHGB " + symbol + "(SB), SI\n") + } + widths := []string{"W", "L"} + if goarch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + if goarch == "amd64" { + fmt.Fprintf(&source, "\tXCHG%s DX, BX\n\tXCHG%s R11, R12\n", width, width) + } else { + fmt.Fprintf(&source, "\tXCHG%s DX, BX\n\tXCHG%s SI, DI\n", width, width) + } + fmt.Fprintf(&source, "\tXCHG%s DX, 8(BX)\n\tXCHG%s 8(BX), DX\n", width, width) + if goarch == "amd64" { + fmt.Fprintf(&source, "\tXCHG%s R11, %s(SB)\n\tXCHG%s %s(SB), R11\n", width, symbol, width, symbol) + } else { + fmt.Fprintf(&source, "\tXCHG%s SI, %s(SB)\n\tXCHG%s %s(SB), SI\n", width, symbol, width, symbol) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestTranslateX86XCHGCompleteGo127FormsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := x86XCHGCompleteFormsSource(target.goarch) + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"xchgforms": {Name: "xchgforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"asm sideeffect \"xchg", "i8", "i16", "i32"} { + if !strings.Contains(ll, want) { + t.Fatalf("XCHG lowering omitted %q:\n%s", want, ll) + } + } + if target.goarch == "amd64" && !strings.Contains(ll, "i64") { + t.Fatalf("XCHGQ lowering omitted i64 operations:\n%s", ll) + } + compileLLVMToObject(t, llc, target.triple, "xchg-"+target.name+".ll", "xchg-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86XCHGRejectsFormsOutsideGo127Optab(t *testing.T) { + for _, test := range []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "XCHGB X0, DL"}, + {goarch: "amd64", instruction: "XCHGW $1, DX"}, + {goarch: "amd64", instruction: "XCHGL 0(BX), 8(CX)"}, + {goarch: "amd64", instruction: "XCHGQ DX, X0"}, + {goarch: "amd64", instruction: "XCHGW DX"}, + {goarch: "amd64", instruction: "XCHGL DX, BX, AX"}, + {goarch: "amd64", instruction: "XCHGL.Z DX, BX"}, + {goarch: "386", instruction: "XCHGB SP, (BX)"}, + {goarch: "386", instruction: "XCHGL R8, (BX)"}, + {goarch: "386", instruction: "XCHGL 1(R8), DX"}, + {goarch: "386", instruction: "XCHGQ DX, BX"}, + } { + t.Run(test.goarch+"/"+strings.NewReplacer(" ", "_", ",", "").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's XCHG optab", test.instruction) + } + }) + } +} + +func TestAMD64XCHGRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT xchgsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x1122, AX + MOVQ $0x3344, BX + XCHGB AH, BL + MOVQ AX, 0(DI) + MOVQ BX, 8(DI) + MOVQ $0x1122334455667788, AX + MOVQ $0x887766554433aabb, BX + XCHGW AX, BX + MOVQ AX, 16(DI) + MOVQ BX, 24(DI) + MOVQ $0x1122334455667788, AX + MOVQ $0x88776655aabbccdd, BX + XCHGL AX, BX + MOVQ AX, 32(DI) + MOVQ BX, 40(DI) + MOVQ $5, AX + MOVQ $7, BX + XCHGQ AX, BX + MOVQ AX, 48(DI) + MOVQ BX, 56(DI) + MOVB $0x5a, 64(DI) + MOVQ $0x11223344556677c3, AX + CMPQ AX, AX + STC + XCHGB 64(DI), AL + MOVQ AX, 72(DI) + SETCS 80(DI) + SETEQ 81(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "xchgsemantics": { + Name: "xchgsemantics", + Args: []LLVMType{Ptr}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void xchgsemantics(uint8_t *); +int main(void) { + uint8_t out[88] = {0}; + xchgsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0x4422)) return 10; + if (*(uint64_t *)(out+8) != UINT64_C(0x3311)) return 11; + if (*(uint64_t *)(out+16) != UINT64_C(0x112233445566aabb)) return 12; + if (*(uint64_t *)(out+24) != UINT64_C(0x8877665544337788)) return 13; + if (*(uint64_t *)(out+32) != UINT64_C(0xaabbccdd)) return 14; + if (*(uint64_t *)(out+40) != UINT64_C(0x55667788)) return 15; + if (*(uint64_t *)(out+48) != 7 || *(uint64_t *)(out+56) != 5) return 16; + if (out[64] != 0xc3 || *(uint64_t *)(out+72) != UINT64_C(0x112233445566775a)) return 17; + if (out[80] != 1 || out[81] != 1) return 18; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "xchg_semantics", triple, ll, mainC, runPrefix) +} diff --git a/amd64_lower_xlat.go b/amd64_lower_xlat.go new file mode 100644 index 000000000..0633719e0 --- /dev/null +++ b/amd64_lower_xlat.go @@ -0,0 +1,40 @@ +package plan9asm + +import "fmt" + +func (c *amd64Ctx) lowerXLAT(op Op, ins Instr) (ok bool, terminated bool, err error) { + if op != "XLAT" { + return false, false, nil + } + if len(ins.Args) != 0 { + return true, false, fmt.Errorf("%s XLAT takes no operands: %q", c.goarch, ins.Raw) + } + base := "" + if c.goarch == "386" { + value, err := c.evalIntSized(Operand{Kind: OpReg, Reg: BX}, I32) + if err != nil { + return true, false, err + } + wide := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i32 %s to i64\n", wide, value) + base = "%" + wide + } else { + value, err := c.loadReg(BX) + if err != nil { + return true, false, err + } + base = value + } + index, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AL}, I8) + if err != nil { + return true, false, err + } + wideIndex := c.newTmp() + fmt.Fprintf(c.b, " %%%s = zext i8 %s to i64\n", wideIndex, index) + basePointer := c.ptrFromAddrI64(base) + pointer := c.newTmp() + fmt.Fprintf(c.b, " %%%s = getelementptr i8, ptr %s, i64 %%%s\n", pointer, basePointer, wideIndex) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load i8, ptr %%%s, align 1\n", value, pointer) + return true, false, c.storeRegSized(AL, I8, "%"+value) +} diff --git a/amd64_lower_xstate.go b/amd64_lower_xstate.go new file mode 100644 index 000000000..da007ebbf --- /dev/null +++ b/amd64_lower_xstate.go @@ -0,0 +1,155 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64XStateFamily uint8 + +const ( + amd64XStateFX amd64XStateFamily = iota + amd64XStateXSAVE + amd64XStateXSAVEOPT + amd64XStateXSAVEC + amd64XStateXSAVES + amd64XStateXRSTOR + amd64XStateXRSTORS +) + +type amd64XStateDirection uint8 + +const ( + amd64XStateSave amd64XStateDirection = iota + amd64XStateRestore +) + +type amd64XStateSpec struct { + family amd64XStateFamily + direction amd64XStateDirection + mode64 bool + implicitMask bool + memoryBytes int +} + +// amd64XStateSpecs is the complete Go 1.27 extended-state memory grammar. +// Opcode spelling selects only the state family, transfer direction, and +// encoding mode; one parser owns the shared Ym memory operand rule. +var amd64XStateSpecs = map[Op]amd64XStateSpec{ + "FXSAVE": {family: amd64XStateFX, direction: amd64XStateSave, memoryBytes: 512}, + "FXSAVE64": {family: amd64XStateFX, direction: amd64XStateSave, mode64: true, memoryBytes: 512}, + "FXRSTOR": {family: amd64XStateFX, direction: amd64XStateRestore, memoryBytes: 512}, + "FXRSTOR64": {family: amd64XStateFX, direction: amd64XStateRestore, mode64: true, memoryBytes: 512}, + "XSAVE": {family: amd64XStateXSAVE, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVE64": {family: amd64XStateXSAVE, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT64": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEC": {family: amd64XStateXSAVEC, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEC64": {family: amd64XStateXSAVEC, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVES": {family: amd64XStateXSAVES, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVES64": {family: amd64XStateXSAVES, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTOR": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTOR64": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTORS": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTORS64": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, +} + +type amd64XStateForm struct { + memory Operand +} + +func (c *amd64Ctx) lowerXState(op Op, ins Instr) (ok bool, terminated bool, err error) { + rawOp := strings.ToUpper(string(op)) + baseOp := rawOp + if dot := strings.IndexByte(rawOp, '.'); dot >= 0 { + baseOp = rawOp[:dot] + } + spec, recognized := amd64XStateSpecs[Op(baseOp)] + if !recognized { + return false, false, nil + } + if rawOp != baseOp { + return true, false, fmt.Errorf("%s %s does not accept instruction suffixes: %q", c.goarch, baseOp, ins.Raw) + } + if c.goarch == "386" && spec.mode64 { + return true, false, fmt.Errorf("386 %s is absent from Go 1.27's 32-bit encoding table: %q", baseOp, ins.Raw) + } + form, err := c.parseXStateForm(baseOp, ins) + if err != nil { + return true, false, err + } + return c.emitXState(spec, form) +} + +func (c *amd64Ctx) parseXStateForm(op string, ins Instr) (amd64XStateForm, error) { + var form amd64XStateForm + if len(ins.Args) != 1 { + return form, fmt.Errorf("%s %s expects one Go 1.27 Ym memory operand: %q", c.goarch, op, ins.Raw) + } + memory := ins.Args[0] + if !isAMD64MemoryOperand(memory) { + return form, fmt.Errorf("%s %s expects Go 1.27 Ym memory: %q", c.goarch, op, ins.Raw) + } + if memory.Kind == OpMem && !x86MemoryRegistersValidForArch(memory.Mem, c.goarch) { + return form, fmt.Errorf("%s %s address uses an out-of-range register: %q", c.goarch, op, ins.Raw) + } + return amd64XStateForm{memory: memory}, nil +} + +func (spec amd64XStateSpec) mnemonic() string { + var mnemonic string + switch spec.family { + case amd64XStateFX: + mnemonic = "fxsave" + if spec.direction == amd64XStateRestore { + mnemonic = "fxrstor" + } + case amd64XStateXSAVE: + mnemonic = "xsave" + case amd64XStateXSAVEOPT: + mnemonic = "xsaveopt" + case amd64XStateXSAVEC: + mnemonic = "xsavec" + case amd64XStateXSAVES: + mnemonic = "xsaves" + case amd64XStateXRSTOR: + mnemonic = "xrstor" + case amd64XStateXRSTORS: + mnemonic = "xrstors" + default: + panic("unknown amd64 xstate family") + } + if spec.mode64 { + mnemonic += "64" + } + return mnemonic +} + +func (c *amd64Ctx) emitXState(spec amd64XStateSpec, form amd64XStateForm) (bool, bool, error) { + pointer, pointerType, err := c.x86DescriptorMemoryPointer(form.memory) + if err != nil { + return true, false, err + } + memoryConstraint := "=*m" + if spec.direction == amd64XStateRestore { + memoryConstraint = "*m" + } + constraints := memoryConstraint + arguments := fmt.Sprintf("%s elementtype([%d x i8]) %s", pointerType, spec.memoryBytes, pointer) + if spec.implicitMask { + ax, err := c.evalIntSized(Operand{Kind: OpReg, Reg: AX}, I32) + if err != nil { + return true, false, err + } + dx, err := c.evalIntSized(Operand{Kind: OpReg, Reg: DX}, I32) + if err != nil { + return true, false, err + } + constraints += ",{ax},{dx}" + arguments += fmt.Sprintf(", i32 %s, i32 %s", ax, dx) + } + constraints += ",~{memory},~{dirflag},~{fpsr},~{flags}" + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", + spec.mnemonic()+" $0", constraints, arguments) + return true, false, nil +} diff --git a/amd64_lower_xstate_test.go b/amd64_lower_xstate_test.go new file mode 100644 index 000000000..d8d37f09a --- /dev/null +++ b/amd64_lower_xstate_test.go @@ -0,0 +1,211 @@ +package plan9asm + +import ( + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +func TestAMD64XStateGrammarIsComplete(t *testing.T) { + got := make([]string, 0, len(amd64XStateSpecs)) + for op := range amd64XStateSpecs { + got = append(got, string(op)) + } + sort.Strings(got) + want := []string{ + "FXRSTOR", "FXRSTOR64", "FXSAVE", "FXSAVE64", + "XRSTOR", "XRSTOR64", "XRSTORS", "XRSTORS64", + "XSAVE", "XSAVE64", "XSAVEC", "XSAVEC64", "XSAVEOPT", "XSAVEOPT64", "XSAVES", "XSAVES64", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("xstate grammar opcodes = %v, want %v", got, want) + } + for op, want := range map[Op]amd64XStateSpec{ + "FXSAVE": {family: amd64XStateFX, direction: amd64XStateSave, memoryBytes: 512}, + "FXSAVE64": {family: amd64XStateFX, direction: amd64XStateSave, mode64: true, memoryBytes: 512}, + "FXRSTOR": {family: amd64XStateFX, direction: amd64XStateRestore, memoryBytes: 512}, + "FXRSTOR64": {family: amd64XStateFX, direction: amd64XStateRestore, mode64: true, memoryBytes: 512}, + "XSAVE": {family: amd64XStateXSAVE, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVE64": {family: amd64XStateXSAVE, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEOPT64": {family: amd64XStateXSAVEOPT, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVEC": {family: amd64XStateXSAVEC, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVEC64": {family: amd64XStateXSAVEC, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XSAVES": {family: amd64XStateXSAVES, direction: amd64XStateSave, implicitMask: true, memoryBytes: 1}, + "XSAVES64": {family: amd64XStateXSAVES, direction: amd64XStateSave, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTOR": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTOR64": {family: amd64XStateXRSTOR, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, + "XRSTORS": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, implicitMask: true, memoryBytes: 1}, + "XRSTORS64": {family: amd64XStateXRSTORS, direction: amd64XStateRestore, mode64: true, implicitMask: true, memoryBytes: 1}, + } { + if got := amd64XStateSpecs[op]; got != want { + t.Errorf("xstate grammar %s = %+v, want %+v", op, got, want) + } + } +} + +func TestTranslateX86XStateCompleteGoAssemblerForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + source := `TEXT xstateforms(SB),$0-0 + FXSAVE (BX) + FXRSTOR 8(BP) + XSAVE 16(SI) + XRSTOR 24(DI) + XSAVEOPT 32(BX) + XSAVEC 40(BP) + XSAVES 48(SI) + XRSTORS 56(DI) +` + if target.goarch == "amd64" { + source += ` FXSAVE64 64(R11) + FXRSTOR64 72(R12) + XSAVE64 80(R13) + XRSTOR64 88(R14) + XSAVEOPT64 96(R15) + XSAVEC64 104(R11) + XSAVES64 112(R12) + XRSTORS64 120(R13) +` + } + source += "\tRET\n" + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"xstateforms": {Name: "xstateforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"fxsave", "fxrstor", "xsave", "xrstor", "xsaveopt", "xsavec", "xsaves", "xrstors"} { + if !strings.Contains(ir, want+" $0") { + t.Errorf("IR is missing %s:\n%s", want, ir) + } + } + if want := `"target-features"="+fxsr,+xsave,+xsavec,+xsaveopt,+xsaves"`; !strings.Contains(ir, want) { + t.Errorf("IR is missing complete xstate feature set %s:\n%s", want, ir) + } + compileLLVMToObject(t, llc, target.triple, "xstate-"+target.name+".ll", "xstate-"+target.name+".o", ir) + }) + } +} + +func TestAMD64FXStateRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT fxstatesemantics(SB),NOSPLIT,$0-8 + MOVQ area+0(FP), DI + FXSAVE (DI) + FXRSTOR (DI) + FXSAVE64 512(DI) + FXRSTOR64 512(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"fxstatesemantics": { + Name: "fxstatesemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void fxstatesemantics(uint8_t *); +int main(void) { + __attribute__((aligned(16))) uint8_t area[1024]; + memset(area, 0xa5, sizeof(area)); + fxstatesemantics(area); + uint16_t control0 = 0, control1 = 0; + memcpy(&control0, area, sizeof(control0)); + memcpy(&control1, area + 512, sizeof(control1)); + if (control0 == UINT16_C(0xa5a5)) return 10; + if (control1 == UINT16_C(0xa5a5)) return 11; + if (control0 != control1) return 12; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "fxstate_semantics", triple, ir, mainC, runPrefix) +} + +func TestTranslateX86XStateRejectsFormsOutsideGoTables(t *testing.T) { + tests := []struct { + goarch string + instruction string + }{ + {goarch: "amd64", instruction: "FXSAVE AX"}, + {goarch: "amd64", instruction: "FXRSTOR $1"}, + {goarch: "amd64", instruction: "XSAVE"}, + {goarch: "amd64", instruction: "XRSTOR (BX), (CX)"}, + {goarch: "amd64", instruction: "XSAVEOPT.Z (BX)"}, + {goarch: "amd64", instruction: "XSAVEC X0"}, + {goarch: "amd64", instruction: "XSAVES AX"}, + {goarch: "amd64", instruction: "XRSTORS $1"}, + {goarch: "386", instruction: "XSAVE (R11)"}, + } + for _, op := range []string{"FXSAVE64", "FXRSTOR64", "XSAVE64", "XRSTOR64", "XSAVEOPT64", "XSAVEC64", "XSAVES64", "XRSTORS64"} { + tests = append(tests, struct { + goarch string + instruction string + }{goarch: "386", instruction: op + " (BX)"}) + } + for _, test := range tests { + t.Run(test.goarch+"_"+strings.NewReplacer(" ", "_", "$", "", "(", "_", ")", "_").Replace(test.instruction), func(t *testing.T) { + source := "TEXT bad(SB),$0-0\n\t" + test.instruction + "\n\tRET\n" + requireX86GoAssemblerResult(t, test.goarch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + triple := "x86_64-unknown-linux-gnu" + if test.goarch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: test.goarch, TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted %q outside Go 1.27's xstate tables", test.instruction) + } + }) + } +} diff --git a/amd64_madd_typed_forms_test.go b/amd64_madd_typed_forms_test.go new file mode 100644 index 000000000..8cfeac7ff --- /dev/null +++ b/amd64_madd_typed_forms_test.go @@ -0,0 +1,409 @@ +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86MADDOfficialRows(t *testing.T, arch string) []string { + t.Helper() + byOpcode := x86OfficialPackedBinaryWitnesses(t) + var rows []string + for _, name := range []string{"VPMADDWD", "VPMADDUBSW"} { + if len(byOpcode[name]) == 0 { + t.Fatal("official MADD table missing", name) + } + for _, row := range byOpcode[name] { + if arch != "386" || strings.Count(row, ",") != 3 { + rows = append(rows, row) + } + } + } + return rows +} + +// Full Go source classes, including valid forms lacking native backing or +// physical mode32 identity. These are not all object/runtime positives. +func x86MADDWholeFormats(t *testing.T, arch string) []string { + rows := x86MADDOfficialRows(t, arch) + for _, name := range []string{"PMADDWL", "PMADDUBSW"} { + for _, source := range []string{"X0", "X7", "(AX)", "16(AX)(CX*2)", "value+0(FP)", "local-16(SP)", "data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+",X1") + } + if arch == "amd64" { + rows = append(rows, name+" X15,X0") + } + } + if arch == "amd64" { + for _, source := range []string{"M0", "M7", "(AX)", "value+0(FP)", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + rows = append(rows, "PMADDWL "+source+",M1") + } + } + for _, name := range []string{"VPMADDWD", "VPMADDUBSW"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{"value+0(FP)", "local-64(SP)", "data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+","+bank+"1,"+bank+"2") + } + if arch != "386" || bank != "Z" { + rows = append(rows, name+" "+bank+"20,"+bank+"21,"+bank+"22") + } + } + } + return rows +} + +func TestX86MADDTypedCompleteGoAttributes(t *testing.T) { + if len(x86PackedMultiplyAddSpecs) != 4 { + t.Fatal("MADD source family must have four canonical Go names") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APMADDWL, ymm, Py1, opBytes{0xf5, Pe, 0xf5}}", + "{APMADDUBSW, yxm_q4, Pq4, opBytes{0x04}}", + "{Zm_r_xm, 1, argList{Ymm, Ymr}}", "{Zm_r_xm, 2, argList{Yxm, Yxr}}", + "{Zm_r, 1, argList{Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual legacy opcode/class row changed", row) + } + } + for _, class := range []string{"Yax", "Ycx", "Yrx", "Yrl", "Yrl32", "Ym", "Ymr"} { + if !strings.Contains(string(legacy), "ycover["+class+"*Ymax+Ymm] = 1") { + t.Fatal("actual MMX source class changed", class) + } + } + for _, item := range []struct{ name, mapName, opcode string }{ + {"VPMADDWD", "0F", "F5"}, {"VPMADDUBSW", "0F38", "04"}, + } { + start := strings.Index(string(avx), "{as: A"+item.name+",") + if start < 0 { + t.Fatal("actual AVX opcode missing", item.name) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvandnpd, prefix: Pavx") || strings.Contains(row, "Bcst") || strings.Contains(row, "Sae") { + t.Fatal("actual opcode-specific attributes changed", row) + } + for _, width := range []string{"128", "256"} { + want := fmt.Sprintf("avxEscape | vex%s | vex66 | vex%s | vexW0, 0x%s", width, item.mapName, item.opcode) + if !strings.Contains(row, want) { + t.Fatal("actual VEX opcode attributes changed", want) + } + } + for i, width := range []string{"128", "256", "512"} { + want := fmt.Sprintf("avxEscape | evex%s | evex66 | evex%s | evexW0, evexN%d | evexZeroingEnabled, 0x%s", width, item.mapName, 16<= 3 { + physical = (index - 3) & 7 + } + if len(code) == 5 && code[0] == 0x41 { + code = code[1:] // Actual Go's ignored MMX REX.B annotation. + } + if want := []byte{0x0f, 0xf5, byte(0xc8 | physical), 0xc3}; !bytes.Equal(code, want) { + t.Fatalf("actual MMX physical bank changed: %s %x want%x", register, code, want) + } + if index >= 3 { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + context, ir := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := ir.String() + if _, _, err := context.lowerPackedWordMultiplyAdd("PMADDWL", file.Funcs[0].Instrs[1]); !errors.Is(err, ErrProbeNeedsContext) || ir.String() != before { + t.Fatal("GP spelling fabricated an initialized MMX source", register, err) + } + } + } +} + +func TestX86MADDTypedRetainsRejectedFormsAndVirtualMemory(t *testing.T) { + for _, row := range []string{ + "PMADDWD X0,X1", "PMADDWL X0,Y1", "PMADDWL M0,AX", "PMADDUBSW M0,M1", + "PMADDUBSW X0,(AX)", "PMADDUBSW.Z X0,X1", "VPMADDWD X0,Y1,Y2", + "VPMADDUBSW.BCST (AX),X0,K1,X1", "VPMADDWD.SAE Z0,Z1,Z2", + "VPMADDUBSW X0,X1,K0,X2", "VPMADDWD $1,X1,X2", "VPMADDUBSW X0,X1,(AX)", + } { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if err := ProbeInstruction(ArchAMD64, "amd64", file.Funcs[0].Instrs[1]); err == nil { + t.Fatal("Go-rejected MADD form obtained lowering", row) + } + } + for _, row := range []string{"PMADDUBSW source+0(FP),X1", "PMADDUBSW (SP),X1", "PMADDUBSW (BP),X1", "PMADDUBSW (AX)(BP*2),X1"} { + source := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + context, ir := newAMD64CtxWithFuncForTest(t, file.Funcs[0], x86WordMaddFPType(t), nil) + before := ir.String() + if _, _, err := context.lowerPackedWordMultiplyAdd("PMADDUBSW", file.Funcs[0].Instrs[1]); !errors.Is(err, ErrProbeNeedsContext) || ir.String() != before { + t.Fatal("virtual storage became an original native m128 pointer", row, err) + } + } +} + +func TestX86MADDTypedByteCallerRetainsUnknownAndPartialInputs(t *testing.T) { + for name, body := range map[string]string{ + "unknown source": "VMOVUPS (DI),Z1\nVPMADDUBSW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "unknown second": "VMOVUPS (DI),Z0\nVPMADDUBSW Z0,Z1,Z2\nVMOVSS X2,(DX)", + "partial pair": "MOVSS (DI),X0\nMOVSS (SI),X1\nVPMADDUBSW Y0,Y1,Y2\nVEXTRACTF128 $1,Y2,X3\nVMOVSS X3,(DX)", + "unknown mask": "VMOVUPS (DI),Z0\nVPMADDUBSW.Z Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown merge": "VMOVUPS (DI),Z0\nMOVL $1,AX\nKMOVQ AX,K1\nVPMADDUBSW Z0,Z0,K1,Z2\nVMOVSS X2,(DX)", + "unknown address": "VMOVUPS (DI),Z0\nVPMADDUBSW (BX),Z0,Z2\nVMOVSS X2,(DX)", + "zero still full read": "VMOVUPS (DI),Z0\nMOVL $0,AX\nKMOVQ AX,K1\nVPMADDUBSW.Z (DX),Z0,K1,Z2\nVMOVSS X2,(DX)", + "private SP": "VMOVUPS (DI),Z0\nMOVQ SP,BX\nPMADDUBSW (BX),X0\nVMOVSS X0,(DX)", + "join missing source": "VMOVUPS (DI),Z1\nCMPQ AX,$0\nJE joined\nVMOVUPS (SI),Z0\njoined:\nVPMADDUBSW Z0,Z1,Z2\nVMOVSS X2,(DX)", + } { + t.Run(name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("typed byte-MADD grammar became input/address permission", err) + } + }) + } +} + +func TestX86MADDTypedMalformedIdentityAndPhysicalBanks(t *testing.T) { + for _, name := range []string{"PMADDWL", "PMADDUBSW", "VPMADDWD", "VPMADDUBSW"} { + operands := "X0,X1" + if strings.HasPrefix(name, "V") { + operands = "X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", original.Op, Op(strings.ToLower(name)), "MOVQ", Op(name + ".BAD")} { + ins := original + ins.Op = identity + before := fmt.Sprintf("%#v", ins) + context, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ir, temporary := out.String(), context.tmp + known, terminated, err := context.lowerPackedWordMultiplyAdd(original.Op, ins) + bad := identity != "" && !strings.EqualFold(string(identity), string(original.Op)) + if !known || terminated || (err != nil) != bad || bad && (out.String() != ir || context.tmp != temporary) || before != fmt.Sprintf("%#v", ins) { + t.Fatal("dispatch identity changed input or emitted before rejection", name, identity, err) + } + } + } + for _, opcode := range []string{"VPMADDWD", "VPMADDUBSW"} { + for _, bank := range []string{"X", "Y", "Z"} { + row := opcode + " " + bank + "20," + bank + "21," + bank + "22" + accepted := bank != "Z" + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", accepted) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, _, err = x86PackedMultiplyAddProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "386") + if accepted && !errors.Is(err, ErrProbeNeedsContext) || !accepted && err == nil { + t.Fatal("Go syntax is not physical mode32 permission", row, err) + } + } + code := encodeX86RawEVEXPackedMADD(Op(opcode), 2, 1, 2, 0, 1, true, true) + decoded, _, _, err := decodedX86EVEXPackedMADDInstruction(code, 32) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(decoded) + if err != nil { + t.Fatal(err) + } + var detached Instr + if err := json.Unmarshal(data, &detached); err != nil { + t.Fatal(err) + } + if _, _, err := x86PackedMultiplyAddProperties(detached.Op, detached, "386"); err == nil { + t.Fatal("JSON/raw text fabricated physical mode32 mask provenance") + } + for _, address := range []int{16, 32, 64} { + changed := decoded + changed.x86AddressBits = address + if _, _, err := x86PackedMultiplyAddProperties(changed.Op, changed, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound address-size marker became memory permission", err) + } + } + } +} + +func TestX86MADDTypedNamedAndPhysicalThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, raw := range []bool{false, true} { + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(fmt.Sprintf("raw=%t/%s", raw, target.triple), func(t *testing.T) { + rows := x86MADDOfficialRows(t, target.arch) + if raw { + rows = nil + for _, row := range x86MADDOfficialRows(t, "amd64") { + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + code = code[:len(code)-1] + for w := 0; w < 2; w++ { + physical := append([]byte(nil), code...) + if physical[0] == 0xc5 { + physical = append([]byte{0xc4, physical[1]&0x80 | 0x61, physical[1] & 0x7f}, physical[2:]...) + } + physical[2] |= byte(w << 7) + var bytes strings.Builder + for _, value := range physical { + fmt.Fprintf(&bytes, "BYTE $%d\n", value) + } + rows = append(rows, bytes.String()) + } + } + } else { + rows = append(rows, "PMADDWL X0,X1", "PMADDWL (AX),X1", "PMADDUBSW X0,X1", "PMADDUBSW (AX),X1") + } + var source, declarations strings.Builder + declarations.WriteString("package madd\n") + for index, row := range rows { + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", index, row) + fmt.Fprintf(&declarations, "func F%d()\n", index) + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + pkg := mustGoPackage(t, "example.invalid/madd", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for index := range rows { + name := fmt.Sprintf("F%d", index) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), target.arch, types.SizesFor("gc", target.arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "madd-text.ll", "madd-text.o", ir) + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "madd-module.ll", "madd-module.o", module.String()) + result, err := translateGoModuleInContext(context, pkg, []byte(source.String()), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve, + }) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "madd-go.ll", "madd-go.o", result.Module.String()) + t.Logf("%d named/actual-Go-derived physical forms, three APIs/object axes; no caller or native runtime grant", len(rows)) + }) + } + } +} diff --git a/amd64_madd_typed_mode32_test.go b/amd64_madd_typed_mode32_test.go new file mode 100644 index 000000000..573334385 --- /dev/null +++ b/amd64_madd_typed_mode32_test.go @@ -0,0 +1,45 @@ +package plan9asm + +import "testing" + +func TestX86MADDTypedMode32LegacySourceRules(t *testing.T) { + for _, opcode := range []string{"PMADDWL", "PMADDUBSW"} { + for _, operands := range []string{"X0,X1", "X0,X7", "(AX),X7"} { + row := opcode + " " + operands + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + code := assembleX87ControlBytes(t, "386", source) + if len(code) == 0 { + t.Fatal("accepted legacy source has no original Go machine bytes", row) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, known, err := x86PackedMultiplyAddProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "386") + if !known || err != nil { + t.Errorf("accepted low mode32 source lost its typed shape: %s bytes%x: %v", row, code, err) + } + } + // Unlike the AVX name rows, the legacy encoder rejects these high + // source spellings; do not label them accepted physical Context rows. + for _, operands := range []string{"X8,X1", "X0,X15", "(AX),X8", "X16,X0", "X0,X16"} { + source := "TEXT f(SB),4,$0-0\n" + opcode + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, known, err := x86PackedMultiplyAddProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "386") + if !known || err == nil { + t.Fatal("Go-rejected high legacy source obtained typed admission", source) + } + } + } + // The shared ymm is not a blanket 386 MMX row: Py1's original row + // selection rejects its z<1 MMX forms in 32-bit mode. + for _, source := range []string{"M0", "(AX)", "AX", "CX", "SI"} { + row := "PMADDWL " + source + ",M1" + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + } +} diff --git a/amd64_madd_typed_native_linux_amd64_test.go b/amd64_madd_typed_native_linux_amd64_test.go new file mode 100644 index 000000000..e9cf1301f --- /dev/null +++ b/amd64_madd_typed_native_linux_amd64_test.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The filename is the Linux/amd64 build role and the cross-runtime prefix is +// selected by required CI. Native AVX must be available: no Skip, Rosetta +// fallback or object-only substitution satisfies this original-Go axis. +func TestCrossLinuxRuntimeMatrixX86MADDTypedOriginalVEX(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.requireAVX(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, row := range x86MADDOracleRows() { + if row.legacy || row.masking != 0 || row.width > 32 { + continue + } + name := fmt.Sprintf("V%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[48]byte,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-36\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nSTMXCSR saved-4(SP)\nLDMXCSR csr+32(FP)\nVMOVDQU (CX),Y0\nVMOVDQU (DX),Y1\nVMOVDQU (DX),Y2\nPUSHQ flags+24(FP)\nPOPFQ\n", name) + instruction, destination := row.instruction() + fmt.Fprintf(&assembly, "%s\nVMOVDQU Y%s,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,32(AX)\nSTMXCSR 40(AX)\nVZEROUPPER\nLDMXCSR saved-4(SP)\nRET\n", instruction, destination[1:]) + fmt.Fprintf(&table, "{%s,%d,%d},\n", name, row.width, row.inputBytes) + } + tests := `package oracle +import ( + "testing" + "encoding/binary" + "runtime" + "unsafe" +) + +var cases = []struct { + run func(*[48]byte, *[64]byte, *[64]byte, uint64, uint32) + width, bytes int +}{ +` + table.String() + `} + +func TestOriginal(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + edges := [...]int32{0, 1, -1, 127, -128, 255, 256, -257, 32767, -32768, 16384, -16384} + profiles := []uint32{0x1f80, 0x3fc1, 0x5f82, 0x9fe5} + observations := 0 + for row, entry := range cases { + for sample := 0; sample < 32; sample++ { + for _, csr := range profiles { + for seed := 0; seed < 2; seed++ { + var a, b [64]byte + var out, want [50]byte + for i := 0; i < 64; i += entry.bytes { + x, y := edges[(sample+i*3)%len(edges)], edges[(sample+7+i*3)%len(edges)] + a[i], b[i] = byte(x), byte(y) + if entry.bytes == 2 { + a[i+1], b[i+1] = byte(uint32(x)>>8), byte(uint32(y)>>8) + } + } + if entry.bytes == 2 && sample == 0 { + for i := 0; i < 64; i += 2 { + a[i] = 0 + b[i] = 0 + a[i+1] = 0x80 + b[i+1] = 0x80 + } + } + if entry.bytes == 1 && sample < 2 { + for i := range a { + a[i] = 0x7f + if sample == 1 { + a[i] = 0x80 + } + b[i] = 0xff + } + } + beforeA, beforeB := a, b + for i := range out { + out[i] = 0xc3 + want[i] = 0xc3 + } + + flags := uint64(2) + if seed != 0 { + flags |= 0x8d5 + } + entry.run((*[48]byte)(unsafe.Pointer(&out[1])), &a, &b, flags, csr) + for i := 0; i < 32; i++ { + want[i+1] = 0 + } + for i := 0; i < entry.width; i += entry.bytes * 2 { + var sum int64 + if entry.bytes == 2 { + sum = int64(int16(binary.LittleEndian.Uint16(a[i:]))) * int64(int16(binary.LittleEndian.Uint16(b[i:]))) + + int64(int16(binary.LittleEndian.Uint16(a[i+2:]))) * int64(int16(binary.LittleEndian.Uint16(b[i+2:]))) + } else { + sum = int64(int8(a[i]))*int64(b[i]) + int64(int8(a[i+1]))*int64(b[i+1]) + if sum < -32768 { + sum = -32768 + } + if sum > 32767 { + sum = 32767 + } + } + for j := 0; j < entry.bytes * 2; j++ { + want[i+j+1] = byte(uint32(sum) >> uint(j*8)) + } + } + + actualFlags := binary.LittleEndian.Uint64(out[33:41]) + actualCSR := binary.LittleEndian.Uint32(out[41:45]) + copy(want[33:45], out[33:45]) + observations++ + if out != want || a != beforeA || b != beforeB || actualFlags&0x8d5 != flags&0x8d5 || actualCSR != csr { + t.Fatalf("original VEX MADD row%d sample%d flags%x CSR%x got%x want%x", + row, sample, actualFlags, actualCSR, out, want) + } + } + } + } + } + if observations != len(cases)*32*4*2 { + t.Fatal("incomplete original VEX matrix", observations) + } +} +` + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests) + t.Logf("%d original VEX X/Y kernels, register/memory/all destination aliases ×32 samples ×4 CSR profiles ×2 six-flag seeds; full Y upper-zero/input/canaries. No original EVEX execution claim.", count) +} diff --git a/amd64_madd_typed_rules_test.go b/amd64_madd_typed_rules_test.go new file mode 100644 index 000000000..c0ea8286b --- /dev/null +++ b/amd64_madd_typed_rules_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +// Intel's PMADDWD and PMADDUBSW tables specify WIG, unlike EVEX PACK*DW. +// Bind W0 to stock Go's actual instruction, then independently compare the +// complete LLVM MC opcode/operands of the W1 physical counterpart. +func TestX86MADDTypedPhysicalWIGCompleteOperands(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 MC required") + } + for _, opcode := range []string{"VPMADDWD", "VPMADDUBSW"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "1", "16(AX)(CX*2)"} { + row := opcode + ".Z " + source + "," + bank + "2,K3," + bank + "4" + code := assembleX87ControlBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + code = code[:len(code)-1] + if code[0] != 0x62 || code[2]&0x80 != 0 { + t.Fatal("Go original is not the expected EVEX W0 row", row, code) + } + for _, mode := range []int{32, 64} { + t.Run(fmt.Sprintf("%d/%s", mode, row), func(t *testing.T) { + var texts [2]string + for w := 0; w < 2; w++ { + physical := append([]byte(nil), code...) + physical[2] |= byte(w << 7) + var input strings.Builder + for _, value := range physical { + fmt.Fprintf(&input, "0x%02x ", value) + } + triple := "x86_64" + if mode == 32 { + triple = "i386" + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, + "--mattr=+avx2,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(input.String() + "\n") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || strings.Contains(string(output), "warning:") || + !strings.Contains(string(output), strings.ToLower(opcode)) { + t.Fatalf("independent MC rejected %x: %v\n%s", physical, err, output) + } + texts[w] = strings.TrimSpace(string(output)) + instruction, size, known, err := decodedX86EVEXPackedMADDInstruction(physical, mode) + if err != nil || !known || size != len(physical) { + t.Errorf("Go-bound WIG physical form rejected: %x %d/%t/%v", physical, size, known, err) + continue + } + arch := map[int]string{32: "386", 64: "amd64"}[mode] + if form, known, err := x86PackedMultiplyAddProperties(instruction.Op, instruction, arch); err != nil || !known || form.mask != 2 { + t.Errorf("physical MADD missing complete typed grammar: %+v %t/%v", form, known, err) + } + if width, write, valid := x86InstructionMemoryShape(instruction, 0, arch); source[0] == '1' && + (!valid || write || width != int64(amd64VectorByteWidth(Reg(bank+"0")))) { + t.Errorf("complete physical memory envelope missing: %d/%t/%t", width, write, valid) + } + } + if texts[0] != texts[1] { + t.Fatalf("WIG changed complete disassembled operands: W0=%q W1=%q", texts[0], texts[1]) + } + }) + } + } + } + } +} + +func TestX86MADDTypedCallerByteFamily(t *testing.T) { + rows := []string{"PMADDUBSW X0,X1", "PMADDUBSW X1,X1", "PMADDUBSW (DI),X1"} + for _, bank := range []string{"X", "Y", "Z"} { + for _, first := range []string{bank + "0", "(DI)"} { + rows = append(rows, + "VPMADDUBSW "+first+","+bank+"1,"+bank+"2", + "VPMADDUBSW "+first+","+bank+"1,K1,"+bank+"2", + "VPMADDUBSW.Z "+first+","+bank+"1,K1,"+bank+"2") + } + } + for _, row := range rows { + t.Run(row, func(t *testing.T) { + result := "X2" + if strings.HasPrefix(row, "PMADD") { + result = "X1" + } + body := "VMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\nVMOVUPS (DI),Z2\nMOVL $1,AX\nKMOVQ AX,K1\n" + row + "\nVMOVSS " + result + ",(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal("Go-accepted byte-MADD is missing caller effects", triple, err) + } + } + }) + } +} + +func TestCrossLinuxRuntimeMatrixX86MADDTypedXUpperZero(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + assembly := "TEXT ·upper(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (DX),Z2\nVPMADDUBSW X0,X1,X2\nVMOVUPS Z2,(AX)\nRET\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + ir := x86NarrowOracleIR(t, assembly, "package narroworacle\nfunc upper(out,a,b *[64]byte)\n", execution.triple) + main := `#include +#include +#include +extern void upper(void*,const void*,const void*); +int main(void) { + uint8_t a[64],b[64],out[66];memset(a,2,64);memset(b,3,64);memset(out,0xc3,66); + upper(out+1,a,b); + for(int i=0;i<64;++i) { + int want=i<16?(i&1?0:12):0; + if(out[i+1]!=want) {fprintf(stderr,"MADD alias byte%d got=%u want=%u\n",i,out[i+1],want);return 1;} + } + if(out[0]!=0xc3||out[65]!=0xc3)return 2; + return 0; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "madd-x-upper", ir, main, level) + }) + } +} diff --git a/amd64_madd_typed_runtime_test.go b/amd64_madd_typed_runtime_test.go new file mode 100644 index 000000000..2a83aa6a2 --- /dev/null +++ b/amd64_madd_typed_runtime_test.go @@ -0,0 +1,429 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +type x86MADDOracleRow struct { + width, inputBytes, source, alias, masking int + legacy, mmx bool +} + +func x86MADDOracleRows() []x86MADDOracleRow { + var rows []x86MADDOracleRow + for _, inputBytes := range []int{1, 2} { + for _, width := range []int{8, 16, 32, 64} { + for _, legacy := range []bool{true, false} { + if legacy && width > 16 || !legacy && width == 8 || width == 8 && inputBytes == 1 { + continue + } + for source := 0; source < 2; source++ { + for alias := 0; alias < 3; alias++ { + if legacy && alias == 2 { + continue + } + for mask := 0; mask < 3; mask++ { + if legacy && mask != 0 { + continue + } + rows = append(rows, x86MADDOracleRow{width, inputBytes, source, alias, mask, legacy, width == 8}) + } + } + } + } + } + } + return rows +} + +func (row x86MADDOracleRow) instruction() (string, string) { + bank := map[int]string{8: "M", 16: "X", 32: "Y", 64: "Z"}[row.width] + first, destination := bank+"0", fmt.Sprintf("%s%d", bank, row.alias) + if row.legacy { + destination = bank + "1" + if row.alias == 1 { + first = destination + } + } + if row.source == 1 { + first = "(CX)" + if row.legacy && row.alias == 1 { + first = "(DX)" + } + } + name := "PMADDWL" + if row.inputBytes == 1 { + name = "PMADDUBSW" + } + if row.legacy { + return name + " " + first + "," + destination, destination + } + name = "VPMADDWD" + if row.inputBytes == 1 { + name = "VPMADDUBSW" + } + if row.masking == 2 { + name += ".Z" + } + mask := "" + if row.masking != 0 { + mask = "K1," + } + return name + " " + first + "," + bank + "1," + mask + destination, destination +} + +// Independent widened integer arithmetic, not a production helper/intrinsic. +// Signed word pairs wrap modulo 2^32; signed-byte * unsigned-byte pairs +// saturate to signed16. Every pair stays in its source byte position. +const x86MADDIndependentCReference = ` +static const int32_t edges[] = { + 0, 1, -1, 2, -2, 127, -128, 255, 256, -257, 32767, -32768, 16384, -16384 +}; + +static void fill(uint8_t *out, unsigned sample, unsigned salt, int bytes) { + for (int i = 0; i < 64; i += bytes) { + int32_t v = edges[(sample + salt + i * 3) % 14]; + if (sample >= 14) v = (int32_t)(sample * 7919 + i * 104729 + salt * 65537); + out[i] = (uint8_t)v; + if (bytes == 2) out[i + 1] = (uint8_t)(v >> 8); + } +} + +static int64_t signed_word(const uint8_t *p) { + uint32_t bits = (uint32_t)p[0] | (uint32_t)p[1] << 8; + return bits < 32768 ? (int64_t)bits : (int64_t)bits - 65536; +} + +static void independent_madd(uint8_t *out, const uint8_t *first, + const uint8_t *second, int width, int bytes) { + int group = bytes * 2; + for (int i = 0; i < width; i += group) { + int64_t sum; + if (bytes == 2) sum = signed_word(first + i) * signed_word(second + i) + + signed_word(first + i + 2) * signed_word(second + i + 2); + else { + int32_t a = first[i] < 128 ? first[i] : (int32_t)first[i] - 256; + int32_t b = first[i + 1] < 128 ? first[i + 1] : (int32_t)first[i + 1] - 256; + sum = (int64_t)a * second[i] + (int64_t)b * second[i + 1]; + if (sum < -32768) sum = -32768; + if (sum > 32767) sum = 32767; + } + uint32_t bits = (uint32_t)sum; + for (int j = 0; j < group; ++j) out[i + j] = (uint8_t)(bits >> (j * 8)); + } +} +` + +func TestCrossLinuxRuntimeMatrixX86MADDTypedIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + rows := x86MADDOracleRows() + var assembly, declarations, table, main strings.Builder + declarations.WriteString("package narroworacle\n") + main.WriteString("#include \n#include \n#include \n") + for index, row := range rows { + name := fmt.Sprintf("madd%d", index) + fmt.Fprintf(&declarations, "func %s(out *[80]byte,a,b,old *[64]byte,mask,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\n", name) + if row.mmx { + assembly.WriteString("MOVQ (CX),M0\nMOVQ (DX),M1\n") + } else { + assembly.WriteString("VMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\n") + } + instruction, destination := row.instruction() + fmt.Fprintf(&assembly, "LDMXCSR csr+48(FP)\nPUSHQ flags+40(FP)\nPOPFQ\n%s\n", instruction) + if row.mmx { + fmt.Fprintf(&assembly, "MOVQ %s,(AX)\nEMMS\n", destination) + } else { + fmt.Fprintf(&assembly, "VMOVUPS Z%s,(AX)\n", destination[1:]) + } + assembly.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nSTMXCSR 72(AX)\nRET\n") + fmt.Fprintf(&main, "extern void %s(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t);\n", name) + fmt.Fprintf(&table, "{%s,%d,%d,%d,%d,%d,%d},\n", name, row.width, row.inputBytes, row.source, row.alias, row.masking, x86FloatAddSubBoolInt(row.legacy)) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + main.WriteString("static const struct {void (*run)(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t);int width,bytes,source,alias,masking,legacy;} cases[]={\n" + table.String() + "};\n") + main.WriteString(x86MADDIndependentCReference) + main.WriteString(` +int main(void) { + unsigned saved; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + const uint32_t profiles[] = {0x1f80, 0x3fc1, 0x5f82, 0x9fe5}; + unsigned long long observations = 0; + for (unsigned row = 0; row < sizeof(cases) / sizeof(cases[0]); ++row) { + int lanes = cases[row].width / (cases[row].bytes * 2); + for (unsigned sample = 0; sample < 32; ++sample) + for (int select = 0; select < lanes + 4; ++select) + for (unsigned profile = 0; profile < 4; ++profile) + for (unsigned seed = 0; seed < 2; ++seed) { + _Alignas(64) uint8_t a[64], b[64], old[64], beforeA[64], beforeB[64], beforeOld[64]; + uint8_t out[82], want[82], packed[64]; + fill(a, sample, 0, cases[row].bytes); + fill(b, sample, 7, cases[row].bytes); + for (int i = 0; i < 64; ++i) old[i] = (uint8_t)(0x91 + i * 11); + if (cases[row].bytes == 2 && sample == 0) + for (int i = 0; i < 64; i += 2) { + a[i] = b[i] = 0; + a[i + 1] = b[i + 1] = 0x80; + } + if (cases[row].bytes == 1 && sample < 2) { + memset(a, sample ? 0x80 : 0x7f, 64); + memset(b, 0xff, 64); + } + memcpy(beforeA, a, 64); + memcpy(beforeB, b, 64); + memcpy(beforeOld, old, 64); + memset(out, 0xc3, sizeof(out)); + memcpy(want, out, sizeof(want)); + + uint64_t mask = select == 0 ? 0 : select == 1 ? UINT64_MAX : + select == 2 ? UINT64_C(0xaaaaaaaaaaaaaaaa) : + select == 3 ? UINT64_MAX << lanes : UINT64_C(1) << (select - 4); + uint64_t flags = 2 | (seed ? 0x8d5 : 0); + cases[row].run(out + 1, a, b, old, mask, flags, profiles[profile]); + const uint8_t *first = cases[row].legacy && cases[row].alias ? b : a; + independent_madd(packed, first, b, cases[row].width, cases[row].bytes); + const uint8_t *previous = cases[row].legacy ? b : + cases[row].alias == 0 ? a : cases[row].alias == 1 ? b : old; + for (int i = 0; i < 64; ++i) { + uint8_t value = cases[row].width == 8 ? 0xc3 : cases[row].legacy ? previous[i] : 0; + if (i < cases[row].width) { + value = packed[i]; + if (cases[row].masking && !(mask >> (i / (cases[row].bytes * 2)) & 1)) + value = cases[row].masking == 2 ? 0 : previous[i]; + } + want[i + 1] = value; + } + + uint64_t actualFlags; + uint32_t actualCSR; + memcpy(&actualFlags, out + 65, 8); + memcpy(&actualCSR, out + 73, 4); + memcpy(want + 65, out + 65, 12); + ++observations; + if ((actualFlags & 0x8d5) != (flags & 0x8d5) || actualCSR != profiles[profile] || + memcmp(out, want, sizeof(out)) || memcmp(a, beforeA, 64) || + memcmp(b, beforeB, 64) || memcmp(old, beforeOld, 64)) { + fprintf(stderr, "MADD row%u sample%u width%d bytes%d source%d alias%d mask%llx flags%llx csr%x\n", + row, sample, cases[row].width, cases[row].bytes, cases[row].source, + cases[row].alias, (unsigned long long)mask, (unsigned long long)actualFlags, actualCSR); + for (unsigned i = 0; i < sizeof(out); ++i) + if (out[i] != want[i]) fprintf(stderr, "byte%u got%u want%u\n", i, out[i], want[i]); + return 1; + } + } + } + __asm__ volatile("ldmxcsr %0" :: "m"(saved)); + printf("MADD kernels=%zu observations=%llu aliases64 sixFlags fullCSR canaries\n", + sizeof(cases) / sizeof(cases[0]), observations); + return 0; +} +`) + observations := 0 + for _, row := range rows { + observations += (row.width/(row.inputBytes*2) + 4) * 32 * 4 * 2 + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + ir := ir + if level == "-O2" { + ir = x86MulDivRoundOptimize(t, ir) + } + output := execution.compileAndRunLLVM(t, "madd-independent", ir, main.String(), level) + want := fmt.Sprintf("MADD kernels=%d observations=%d aliases64 sixFlags fullCSR canaries\n", len(rows), observations) + if string(output) != want { + t.Fatalf("incomplete MADD denominator: %q want %q", output, want) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + t.Log("software VEX/EVEX is not original native AVX/AVX512 execution") +} + +func TestCrossLinuxRuntimeMatrixX86MADDTypedOriginalLegacy(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, row := range x86MADDOracleRows() { + if !row.legacy { + continue + } + name := fmt.Sprintf("original%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[32]byte,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-36\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nSTMXCSR saved-4(SP)\nLDMXCSR csr+32(FP)\n", name) + move, bank := "MOVOU", "X" + if row.mmx { + move, bank = "MOVQ", "M" + } + fmt.Fprintf(&assembly, "%s (CX),%s0\n%s (DX),%s1\nPUSHQ flags+24(FP)\nPOPFQ\n", move, bank, move, bank) + instruction, destination := row.instruction() + fmt.Fprintf(&assembly, "%s\n%s %s,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,16(AX)\nSTMXCSR 24(AX)\n", instruction, move, destination) + if row.mmx { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&table, "{%s,%d,%d,%d,-1},\n", name, row.width, row.inputBytes, row.alias) + } + // These accepted GP spellings encode MMX ModRM source banks. Initialize + // each physical M register independently; do not interpret GP contents as + // numeric MMX inputs or grant this source spelling LLVM admission. + for index, register := range []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + name := fmt.Sprintf("original%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out *[32]byte,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-36\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nSTMXCSR saved-4(SP)\nLDMXCSR csr+32(FP)\n", name) + for bank := 0; bank < 8; bank++ { + fmt.Fprintf(&assembly, "MOVQ %d(CX),M%d\n", bank*8, bank) + } + assembly.WriteString("MOVQ (DX),M1\nPUSHQ flags+24(FP)\nPOPFQ\n") + fmt.Fprintf(&assembly, "PMADDWL %s,M1\nMOVQ M1,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,16(AX)\nSTMXCSR 24(AX)\nEMMS\nLDMXCSR saved-4(SP)\nRET\n", register) + fmt.Fprintf(&table, "{%s,8,2,0,%d},\n", name, index&7) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + tests := `package oracle +import ( + "testing" + "encoding/binary" + "runtime" + "unsafe" +) + +var cases = []struct { + run func(*[32]byte, *[64]byte, *[64]byte, uint64, uint32) + width, bytes, alias, bank int +}{ +` + table.String() + `} + +func aligned(storage *[128]byte) *[64]byte { + offset := int((-uintptr(unsafe.Pointer(&storage[0]))) & 63) + return (*[64]byte)(unsafe.Pointer(&storage[offset])) +} + +func TestOriginal(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + edges := [...]int32{0, 1, -1, 2, -2, 127, -128, 255, 256, -257, 32767, -32768, 16384, -16384} + profiles := []uint32{0x1f80, 0x3fc1, 0x5f82, 0x9fe5} + var aa, bb [128]byte + a, b := aligned(&aa), aligned(&bb) + observations := 0 + + for row, entry := range cases { + for sample := 0; sample < 32; sample++ { + for _, csr := range profiles { + for seed := 0; seed < 2; seed++ { + for i := range aa { + aa[i] = 0xa7 + bb[i] = 0xb9 + } + for i := 0; i < 64; i += entry.bytes { + x, y := edges[(sample+i*3)%14], edges[(sample+7+i*3)%14] + if sample >= 14 { + x = int32(sample*7919+i*104729) + y = int32(sample*7919+i*104729+7*65537) + } + a[i], b[i] = byte(x), byte(y) + if entry.bytes == 2 { + a[i+1], b[i+1] = byte(uint32(x)>>8), byte(uint32(y)>>8) + } + } + if entry.bytes == 2 && sample == 0 { + for i := 0; i < 64; i += 2 { + a[i] = 0 + b[i] = 0 + a[i+1] = 0x80 + b[i+1] = 0x80 + } + } + if entry.bytes == 1 && sample < 2 { + for i := range a { + a[i] = 0x7f + if sample == 1 { + a[i] = 0x80 + } + b[i] = 0xff + } + } + beforeA, beforeB := aa, bb + var out, want [34]byte + for i := range out { + out[i] = 0xc3 + want[i] = 0xc3 + } + + flags := uint64(2) + if seed != 0 { + flags |= 0x8d5 + } + entry.run((*[32]byte)(unsafe.Pointer(&out[1])), a, b, flags, csr) + first := a[:] + if entry.alias == 1 { + first = b[:] + } + if entry.bank >= 0 { + first = a[entry.bank*8:] + if entry.bank == 1 { + first = b[:] + } + } + for i := 0; i < entry.width; i += entry.bytes * 2 { + var sum int64 + if entry.bytes == 2 { + sum = int64(int16(binary.LittleEndian.Uint16(first[i:]))) * int64(int16(binary.LittleEndian.Uint16(b[i:]))) + + int64(int16(binary.LittleEndian.Uint16(first[i+2:]))) * int64(int16(binary.LittleEndian.Uint16(b[i+2:]))) + } else { + sum = int64(int8(first[i]))*int64(b[i]) + int64(int8(first[i+1]))*int64(b[i+1]) + if sum < -32768 { + sum = -32768 + } + if sum > 32767 { + sum = 32767 + } + } + for j := 0; j < entry.bytes * 2; j++ { + want[i+j+1] = byte(uint32(sum) >> uint(j*8)) + } + } + + actualFlags := binary.LittleEndian.Uint64(out[17:25]) + actualCSR := binary.LittleEndian.Uint32(out[25:29]) + copy(want[17:29], out[17:29]) + observations++ + if out != want || aa != beforeA || bb != beforeB || actualFlags&0x8d5 != flags&0x8d5 || actualCSR != csr { + t.Fatalf("original MADD row%d sample%d width%d bytes%d flags%x CSR%x got%x want%x", + row, sample, entry.width, entry.bytes, actualFlags, actualCSR, out, want) + } + } + } + } + } + t.Logf("original MADD kernels=%d observations=%d sixFlags fullCSR canaries", len(cases), observations) +} +` + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original") + command := exec.Command("go", "test", "-c", "-p=2", "-vet=off", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOMAXPROCS=2") + x86FloatCompareCaptureCommand(t, command) + output := execution.run(t, executable, "-test.v", "-test.run=^TestOriginal$", "-test.count=1") + want := fmt.Sprintf("original MADD kernels=%d observations=%d sixFlags fullCSR canaries", count, count*32*4*2) + if !strings.Contains(string(output), want) || !strings.Contains(string(output), "--- PASS: TestOriginal") || strings.Contains(string(output), "SKIP") { + t.Fatalf("incomplete original-Go denominator: %q", output) + } + t.Log(strings.TrimSpace(string(output))) + t.Log("original SSE/MMX including all16 GP-to-MMX encoded bank spellings only; this gate does not execute VEX/EVEX") +} diff --git a/amd64_maddword_context_test.go b/amd64_maddword_context_test.go new file mode 100644 index 000000000..8f14a2810 --- /dev/null +++ b/amd64_maddword_context_test.go @@ -0,0 +1,28 @@ +package plan9asm + +import ( + "errors" + "strings" + "testing" +) + +func TestX86WordMultiplyAddLegacyVirtualMemoryCannotProveAlignment(t *testing.T) { + for _, operand := range []string{"source+0(FP)", "(SP)", "(BP)"} { + source := "TEXT f(SB),4,$0-64\nPMADDWL " + operand + ",X1\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + _, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"f": x86WordMaddFPType(t)}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s virtual%s obtained original m128 alignment/fault semantics: %v", triple, operand, err) + } + } + } +} diff --git a/amd64_maddword_fault_runtime_test.go b/amd64_maddword_fault_runtime_test.go new file mode 100644 index 000000000..2b4e0d3da --- /dev/null +++ b/amd64_maddword_fault_runtime_test.go @@ -0,0 +1,118 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddNativeMemoryStatus(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for i, row := range []struct { + instruction string + width int + }{ + {"PMADDWL (BX),X0", 16}, + {"VPMADDWD.Z (BX),X0,K1,X1", 16}, + {"VPMADDWD.Z (BX),Y0,K1,Y1", 32}, + {"VPMADDWD.Z (BX),Z0,K1,Z1", 64}, + {"PMADDUBSW (BX),X0", 16}, + {"VPMADDUBSW.Z (BX),X0,K1,X1", 16}, + {"VPMADDUBSW.Z (BX),Y0,K1,Y1", 32}, + {"VPMADDUBSW.Z (BX),Z0,K1,Z1", 64}, + } { + name := fmt.Sprintf("wordmaddfault%d", i) + // Deliberately discard the computed result. Sideeffect/volatile reads + // must remain after O2, even with the constant zero K mask. + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-8\nMOVQ input+0(FP),BX\nVPXORD Z0,Z0,Z0\nMOVL $0,CX\nKMOVQ CX,K1\n%s\nRET\n", name, row.instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*);\n", name) + reference := fmt.Sprintf("read%d", row.width) + if i == 0 { + reference = "originalPMADDWD" + } else if i == 4 { + reference = "originalPMADDUBSW" + } + fmt.Fprintf(&checks, "for(int offset=0;offset<16;++offset) if(compareStatus(%s,%s,memory+offset)) return 1;\n", reference, name) + fmt.Fprintf(&checks, "if(compareStatus(%s,%s,0) || compareStatus(%s,%s,boundary-%d+1) || compareStatus(%s,%s,boundary-%d)) return 1;\n", reference, name, reference, name, row.width, reference, name, row.width) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +__attribute__((noinline)) static void originalPMADDWD(const uint8_t *p) { + typedef int32_t V __attribute__((vector_size(16))); + V carrier={0,0,0,0}; + // This is the original PMADDWD memory opcode, not MOVDQA plus PMADDWD. + __asm__ volatile("pmaddwd (%1),%0" : "+x"(carrier) : "r"(p) : "memory"); +} +__attribute__((noinline)) static void originalPMADDUBSW(const uint8_t *p) { + typedef int32_t V __attribute__((vector_size(16))); + V carrier={0,0,0,0}; + __asm__ volatile("pmaddubsw (%1),%0" : "+x"(carrier) : "r"(p) : "memory"); +} +__attribute__((noinline)) static void read16(const uint8_t *p) {for(int i=0;i<16;++i) (void)((const volatile uint8_t*)p)[i];} +__attribute__((noinline)) static void read32(const uint8_t *p) {for(int i=0;i<32;++i) (void)((const volatile uint8_t*)p)[i];} +__attribute__((noinline)) static void read64(const uint8_t *p) {for(int i=0;i<64;++i) (void)((const volatile uint8_t*)p)[i];} +typedef struct {int signal;int code;} Status; +static int observe(void (*program)(const uint8_t*),const uint8_t *input,Status *out) { + pid_t child=fork(); if(child<0) return 1; + if(child==0) { + struct rlimit limit={0,0}; if(setrlimit(RLIMIT_CORE,&limit)) _exit(3); + program(input); _exit(0); + } + int status=0; if(waitpid(child,&status,0)!=child) return 1; + if(WIFSIGNALED(status)) {*out=(Status){WTERMSIG(status),0};return 0;} + if(WIFEXITED(status)) {*out=(Status){0,WEXITSTATUS(status)};return 0;} + return 1; +} +static int compareStatus(void (*original)(const uint8_t*),void (*translated)(const uint8_t*),const uint8_t *input) { + Status a,b; + if(observe(original,input,&a) || observe(translated,input,&b)) return 1; + if(a.signal!=b.signal || a.code!=b.code) { + fprintf(stderr,"input=%p mod16=%lu original=(%d,%d) translated=(%d,%d)\n",input,(unsigned long)((uintptr_t)input%16),a.signal,a.code,b.signal,b.code); return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); if(page<=0 || page%64) return 1; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED) return 1; + memset(memory,0x83,(size_t)page); + if(mprotect(memory,(size_t)page,PROT_READ) || mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary=memory+page; +` + checks.String() + ` + if(munmap(memory,(size_t)page*2)) return 1; + return 0; +} +` + for _, opt := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", opt), func(t *testing.T) { + text := ir + if opt { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd-read-status", triple, text, main, prefix) + }) + } + t.Log("legacy exact opcode original/translated typed status at each mod16 and real guard-page boundaries; EVEX zero-mask full-read contract uses independent volatile reader, not unexecuted AVX512 hardware as oracle") +} diff --git a/amd64_maddword_forms.go b/amd64_maddword_forms.go new file mode 100644 index 000000000..e93388b71 --- /dev/null +++ b/amd64_maddword_forms.go @@ -0,0 +1,99 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go calls Intel's legacy PMADDWD instruction PMADDWL. The raw decoder keeps +// that canonical spelling. ymm supplies MMX/m64 and X/m128; _yvandnpd supplies +// all VEX/EVEX widths. Each VPMADDWD row permits zeroing, never broadcasting. +var x86WordMultiplyAddSpecs = map[Op]x86PackedBinarySpec{ + "PMADDWL": {encoding: x86PackedLogicalLegacyInteger}, + "VPMADDWD": {encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, +} + +type x86PackedMultiplyAddSpec struct { + x86PackedBinarySpec + inputBits, outputBits int + nativeOpcode string +} + +var x86PackedMultiplyAddSpecs = map[Op]x86PackedMultiplyAddSpec{ + "PMADDWL": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, 32, "pmaddwd"}, + "PMADDUBSW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 8, 16, "pmaddubsw"}, + "VPMADDWD": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, 32, ""}, + "VPMADDUBSW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, + 8, 16, ""}, +} + +type x86PackedMultiplyAddForm struct { + x86PackedBinaryForm + spec x86PackedMultiplyAddSpec + source Operand +} + +// This describes Go's actual ymm/yxm_q4/_yvandnpd operand rows only. WIG +// and the decoder's private physical marker do not grant a native address, +// initialized input, caller frame, or a new MMX transport. +func x86PackedMultiplyAddProperties(op Op, ins Instr, arch string) (x86PackedMultiplyAddForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedMultiplyAddSpecs[Op(base)] + if !known { + return x86PackedMultiplyAddForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedMultiplyAddForm{}, true, fmt.Errorf("packed multiply-add dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + local.Args = append([]Operand(nil), ins.Args...) + if spec.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return x86PackedMultiplyAddForm{}, true, err + } + } + grammarArch := arch + if arch == "386" && local.x86Encoded && spec.encoding == x86PackedLogicalVEXOrEVEX { + grammarArch = "amd64" + } + binary, err := x86PackedBinaryProperties(local, grammarArch, spec.x86PackedBinarySpec) + form := x86PackedMultiplyAddForm{x86PackedBinaryForm: binary, spec: spec} + if err == nil { + form.source = local.Args[binary.first] + if arch == "386" && grammarArch != arch && binary.memory >= 0 && !x86PackedShiftMemoryValid(form.source, arch) { + err = fmt.Errorf("386 %s physical source has an unbound address", op) + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && binary.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: packed multiply-add has no bound address-size override", ErrProbeNeedsContext) + } + } + return form, true, err +} + +func x86WordMultiplyAddProperties(ins Instr, goarch string) (x86PackedBinaryForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + _, recognized := x86WordMultiplyAddSpecs[Op(base)] + if !recognized { + return x86PackedBinaryForm{}, false, nil + } + // Keep this word-only consumer boundary for the existing frame proof. + form, known, err := x86PackedMultiplyAddProperties(ins.Op, ins, goarch) + return form.x86PackedBinaryForm, known, err +} + +func x86TypedPackedMultiplyAddByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedMultiplyAddProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: form.first, operand: form.source, width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), read: true, write: true}, + }}, true, nil +} diff --git a/amd64_maddword_forms_test.go b/amd64_maddword_forms_test.go new file mode 100644 index 000000000..1342f37c5 --- /dev/null +++ b/amd64_maddword_forms_test.go @@ -0,0 +1,282 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86WordMaddFormats(arch string) []string { + rows := []string{"PMADDWL X0,X7", "PMADDWL (AX),X7"} + if arch == "amd64" { + rows = append(rows, "PMADDWL M0,M7", "PMADDWL (AX),M7", "PMADDWL source+0(FP),M7", "PMADDWL X15,X0", "PMADDWL X7,X7", "PMADDWL (FS),X1", "PMADDWL (GS),X1", "PMADDWL data(SB),X1") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "16(AX)(BX*2)", "source+0(FP)"} { + rows = append(rows, fmt.Sprintf("VPMADDWD %s,%s1,%s7", source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("VPMADDWD %s,%s31,K1,%s30", source, bank, bank), fmt.Sprintf("VPMADDWD.Z %s,%s31,K7,%s30", source, bank, bank)) + } + } + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("VPMADDWD %s31,%s30,%s29", bank, bank, bank)) + } + } + return rows +} + +func x86WordMaddFPType(t *testing.T) FuncSig { + t.Helper() + pkg := mustGoPackage(t, "oracle", "package oracle\nfunc f(source [64]byte)\n") + sig, err := DeriveGoFuncABI0Signature("f", pkg.Types.Scope().Lookup("f").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + if len(sig.Frame.Params) != 64 { + t.Fatalf("actual Go 64-byte field layout changed: %+v", sig.Frame) + } + return sig +} + +func TestX86WordMultiplyAddActualGoCompleteAttributes(t *testing.T) { + if len(x86WordMultiplyAddSpecs) != 2 { + t.Fatal("word multiply-add family must contain exactly the two Go spellings") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "{APMADDWL, ymm, Py1, opBytes{0xf5, Pe, 0xf5}}") { + t.Fatal("actual Go legacy ymm registration changed") + } + start := strings.Index(string(avx), "{as: AVPMADDWD,") + if start < 0 { + t.Fatal("actual Go omits VPMADDWD") + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvandnpd") || strings.Contains(row, "Bcst") || strings.Contains(row, "Sae") { + t.Fatalf("actual Go operand attributes changed: %s", row) + } + for _, width := range []string{"vex128", "vex256"} { + if !strings.Contains(row, "avxEscape | "+width+" | vex66 | vex0F | vexW0, 0xF5") { + t.Fatalf("actual Go VEX attribute missing: %s", width) + } + } + for i, width := range []string{"evex128", "evex256", "evex512"} { + if !strings.Contains(row, fmt.Sprintf("avxEscape | %s | evex66 | evex0F | evexW0, evexN%d | evexZeroingEnabled, 0xF5", width, 16<= 0 { + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := out.String() + _, _, err := ctx.lowerPackedWordMultiplyAdd(ins.Op, ins) + if missing < 0 { + if err != nil { + t.Fatal(err) + } + } else if err == nil || out.String() != before { + t.Fatalf("%s missing FP byte%d emitted or invented backing: %v", row, missing, err) + } + } + } +} diff --git a/amd64_maddword_go_oracle_test.go b/amd64_maddword_go_oracle_test.go new file mode 100644 index 000000000..0704e0bc3 --- /dev/null +++ b/amd64_maddword_go_oracle_test.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddOriginalGoBits(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import "testing" +func signed(p []byte) int64 { v:=int64(uint16(p[0])|uint16(p[1])<<8); if v>=32768 {v-=65536};return v } +func TestOriginalPairWrap(t *testing.T) { + var patterns [8][16]byte + state:=uint32(0x12345678) + for p:=range patterns {for i:=range patterns[p] {state=state*1664525+1013904223;patterns[p][i]=byte(state>>24)}} + for i:=0;i<16;i+=2 {patterns[0][i]=0;patterns[0][i+1]=0x80;patterns[1][i]=0;patterns[1][i+1]=0x80} + for i:=range patterns[2] {patterns[2][i]=0xff} +`) + count := 0 + for _, bank := range []string{"X", "M"} { + forms := []string{"register"} + if bank == "M" { + forms = append(forms, "memory") + } + for _, form := range forms { + name := fmt.Sprintf("wordmadd%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n", name) + move, width := "MOVOU", 16 + if bank == "M" { + move, width = "MOVQ", 8 + } + fmt.Fprintf(&assembly, "%s (BX),%s0\n", move, bank) + first := "(AX)" + if form == "register" { + first = bank + "1" + fmt.Fprintf(&assembly, "%s (AX),%s1\n", move, bank) + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\nPMADDWL %s,%s0\n%s %s0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nSTMXCSR 18(CX)\n", first, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&tests, ` + for _,a:=range patterns {for _,b:=range patterns {for mode:=0;mode<4;mode++ { + var out [24]byte;for i:=range out {out[i]=0xc3} + csr:=uint32(0x1f80|mode<<13|mode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0]!=0xc3 || out[23]!=0xc3 || out[17]!=1 || out[18]!=0 {t.Fatalf("%[1]s flags/canary: %%x",out)} + gotCSR:=uint32(out[19])|uint32(out[20])<<8|uint32(out[21])<<16|uint32(out[22])<<24 + if gotCSR!=csr {t.Fatalf("%[1]s CSR %%x != %%x",gotCSR,csr)} + for i:=0;i<%[2]d;i+=4 { + want:=uint32(signed(a[i:])*signed(b[i:])+signed(a[i+2:])*signed(b[i+2:])) + for j:=0;j<4;j++ {if out[i+j+1]!=byte(want>>uint(8*j)) {t.Fatalf("%[1]s pair %%d got %%x want %%x",i,out,want)}} + } + for i:=%[2]d;i<16;i++ {if out[i+1]!=0xc3 {t.Fatalf("%[1]s touched upper MMX bytes")}} + }}} +`, name, width) + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go signed-pair oracle: %v\n%s", err, output) + } + t.Logf("%d actual original SSE/MMX Go functions × 256 independent signed-pair/CSR/flags/canary cases", count) +} diff --git a/amd64_maddword_runtime_test.go b/amd64_maddword_runtime_test.go new file mode 100644 index 000000000..0d998b031 --- /dev/null +++ b/amd64_maddword_runtime_test.go @@ -0,0 +1,226 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" +) + +const x86WordMaddIndependentRuntimeOracle = ` +static uint8_t patterns[8][64]; +static const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; +static int32_t word(const uint8_t *p) { + uint32_t bits=(uint32_t)p[0]|(uint32_t)p[1]<<8; + return bits<32768 ? (int32_t)bits : (int32_t)bits-65536; +} +__attribute__((noinline)) static int checkMadd( + void (*program)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t), + const char *name,int width,int mode,int legacy,int destination,int shift) { + for(int round=0;round<8;++round) for(int m=0;m<4;++m) { + _Alignas(64) uint8_t aStorage[96],bStorage[96],output[68]; + uint8_t *a=aStorage+shift,*b=bStorage+3; + memset(aStorage,0xa7,sizeof(aStorage)); memset(bStorage,0xb9,sizeof(bStorage)); + memset(output,0xc3,sizeof(output)); + memcpy(a,patterns[round],64); memcpy(b,patterns[(round+1)%8],64); + uint64_t mask=masks[m]; + uint32_t saved,before=0x1f80|((round%4)<<13)|round,after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + program(output+1,a,b,mask); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if(after!=before || output[0]!=0xc3 || output[67]!=0xc3 || output[65]!=1 || output[66]!=0) { + fprintf(stderr,"%s round=%d CSR/flags/canary failure\n",name,round); return 1; + } + for(int i=0;i<64;i+=4) { + // Widen independently to signed int64 before summing. Conversion to + // uint32 implements the specified pair wrap, including 0x80000000. + uint32_t result=(uint32_t)((int64_t)word(a+i)*word(b+i)+(int64_t)word(a+i+2)*word(b+i+2)); + const uint8_t *old=destination==1 ? a : b; + int active=mode==0 || ((mask>>(i/4))&1); + for(int j=0;j<4;++j) { + uint8_t want=i>(j*8)) : (mode==2 ? 0 : old[i+j])) : (legacy ? b[i+j] : 0); + if(output[i+j+1]!=want) { + fprintf(stderr,"%s round=%d mask=%llx byte=%d got=%x want=%x\n",name,round,(unsigned long long)mask,i+j,output[i+j+1],want); return 1; + } + } + } + for(int i=0;i<96;++i) { + if((i=shift+64) && aStorage[i]!=0xa7) return 1; + if((i<3 || i>=67) && bStorage[i]!=0xb9) return 1; + } + if(memcmp(a,patterns[round],64) || memcmp(b,patterns[(round+1)%8],64)) return 1; + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, legacy := range []bool{true, false} { + widths := []int{16} + if !legacy { + widths = []int{16, 32, 64} + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, memory := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + if legacy && mode != 0 { + continue + } + for destination := 0; destination < 3; destination++ { + if legacy && destination != 0 { + continue + } + name := fmt.Sprintf("wordmadd%d", count) + count++ + first := bank + "1" + if memory { + first = "(BX)" + } + row := "PMADDWL " + first + ",X0" + if !legacy { + opcode := "VPMADDWD" + if mode == 2 { + opcode += ".Z" + } + row = opcode + " " + first + "," + bank + "0" + if mode != 0 { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", name, row, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + shift := 1 + if legacy && memory { + shift = 16 + } + fmt.Fprintf(&checks, "if(checkMadd(%s,\"%s\",%d,%d,%d,%d,%d)) return 1;\n", name, name, width, mode, x86FloatAddSubBoolInt(legacy), destination, shift) + } + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n#include \n" + declarations.String() + x86WordMaddIndependentRuntimeOracle + ` +int main(void) { + uint32_t state=0x12345678; + for(int p=0;p<8;++p) for(int i=0;i<64;++i) {state=state*1664525+1013904223; patterns[p][i]=(uint8_t)(state>>24);} + // Explicit signed extrema and the unique overflowing pair, not merely random data. + memset(patterns[0],0,64); memset(patterns[1],0,64); + for(int i=1;i<64;i+=2) {patterns[0][i]=0x80;patterns[1][i]=0x80;} + memset(patterns[2],0xff,64); + for(int i=0;i<64;i+=2) {patterns[3][i]=0xff;patterns[3][i+1]=0x7f;} +` + checks.String() + "return 0;\n}\n" + for _, opt := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", opt), func(t *testing.T) { + text := ir + if opt { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd", triple, text, main, prefix) + }) + } + t.Logf("%d opcode/width/memory/mask/alias programs × 32 signed-pair/MXCSR/flags/canary cases at O0 and O2; translated wide-vector execution is not native AVX512 evidence", count) +} + +func TestCrossLinuxRuntimeMatrixX86WordMultiplyAddLogicalFPFields(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var definitions, wrappers, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, width := range []int{8, 16, 32, 64} { + bank := map[int]string{8: "M", 16: "X", 32: "Y", 64: "Z"}[width] + for mode := 0; mode < 3; mode++ { + if width == 8 && mode != 0 { + continue + } + name := fmt.Sprintf("wordmaddfp%d", index) + index++ + declaration := fmt.Sprintf("func %s(source [64]byte,out *byte,mask uint64)\n", name) + pkg := mustGoPackage(t, "oracle", "package oracle\n"+declaration) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + row := "PMADDWL source+0(FP),M1" + load, store := "MOVQ (AX),M1", "MOVQ M1,(AX)" + if width != 8 { + row = "VPMADDWD source+0(FP)," + bank + "1" + if mode == 2 { + row = strings.Replace(row, "VPMADDWD ", "VPMADDWD.Z ", 1) + } + if mode != 0 { + row += ",K1" + } + row += "," + bank + "1" + load = "VMOVUPS (AX),Z1" + store = "VMOVUPS " + bank + "1,(AX)" + } + fmt.Fprintf(&definitions, "TEXT %s(SB),4,$0-80\nMOVQ out+64(FP),AX\nMOVQ mask+72(FP),DX\nKMOVQ DX,K1\n%s\n%s\n%s\nRET\n", name, load, row, store) + fmt.Fprintf(&wrappers, "define void @check_%s(ptr %%input,ptr %%output,i64 %%mask){\n", name) + packed := "poison" + for i := 0; i < 64; i++ { + fmt.Fprintf(&wrappers, " %%address%d=getelementptr i8,ptr %%input,i64 %d\n %%byte%d=load i8,ptr %%address%d,align 1\n %%pack%d=insertvalue [64 x i8] %s,i8 %%byte%d,%d\n", i, i, i, i, i, packed, i, i) + packed = fmt.Sprintf("%%pack%d", i) + } + fmt.Fprintf(&wrappers, " call void @%s([64 x i8] %s,ptr %%output,i64 %%mask)\n ret void\n}\n", name, packed) + fmt.Fprintf(&declarations, "extern void check_%s(const uint8_t*,uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, ` + for(int round=0;round<16;++round) for(unsigned m=0;m<4;++m) { + uint8_t input[66],output[66];memset(input,0xa7,66);memset(output,0xc3,66); + for(int i=0;i<64;++i) input[i+1]=(uint8_t)(i*17+round*31); + for(int i=0;i<64;i+=2) {output[i+1]=1;output[i+2]=0;} + check_%[1]s(input+1,output+1,masks[m]); + for(int i=0;i<64;i+=4) { + int32_t pair=word(input+i+1)+word(input+i+3); + int active=%[2]d==0 || ((masks[m]>>(i/4))&1); + uint32_t want=i<%[3]d ? (active ? (uint32_t)pair : (%[2]d==2 ? 0 : UINT32_C(0x00010001))) : UINT32_C(0x00010001); + for(int j=0;j<4;++j) if(output[i+j+1]!=(uint8_t)(want>>(8*j))) return 1; + } + if(input[0]!=0xa7 || input[65]!=0xa7 || output[0]!=0xc3 || output[65]!=0xc3) return 1; + for(int i=0;i<64;++i) if(input[i+1]!=(uint8_t)(i*17+round*31)) return 1; + } +`, name, mode, width) + } + } + requireX86GoAssemblerResult(t, "amd64", definitions.String(), true) + file, err := Parse(ArchAMD64, definitions.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n#include \n" + declarations.String() + x86WordMaddIndependentRuntimeOracle + "int main(void){\n" + checks.String() + "return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + wrappers.String() + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "word-madd-logical-fp", triple, text, main, prefix) + }) + } + t.Logf("%d logical FP programs × 64 independent signed-pair cases; signatures derived from actual Go [64]byte declarations; explicit LLVM array carrier is not native stack/alignment or MMX tag proof", index) +} diff --git a/amd64_mask_move_dispatch_test.go b/amd64_mask_move_dispatch_test.go new file mode 100644 index 000000000..a21fff65a --- /dev/null +++ b/amd64_mask_move_dispatch_test.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestAMD64MaskMoveDispatchBindsCompleteFamily(t *testing.T) { + // The internal lowerer takes the dispatched opcode separately. Older + // helper callers intentionally leave Instr.Op unset; parsed instructions + // retain their own opcode. Both must use the same complete _ykmovb grammar. + forms := []struct { + name string + src Operand + dst Operand + }{ + {"mask_to_mask", Operand{Kind: OpReg, Reg: "K1"}, Operand{Kind: OpReg, Reg: "K2"}}, + {"mask_to_gp", Operand{Kind: OpReg, Reg: "K1"}, Operand{Kind: OpReg, Reg: AX}}, + {"gp_to_mask", Operand{Kind: OpReg, Reg: AX}, Operand{Kind: OpReg, Reg: "K2"}}, + {"mask_to_memory", Operand{Kind: OpReg, Reg: "K1"}, Operand{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}}, + {"memory_to_mask", Operand{Kind: OpMem, Mem: MemRef{Base: BX, Off: 8}}, Operand{Kind: OpReg, Reg: "K2"}}, + } + for _, arch := range []string{"amd64", "386"} { + for _, op := range []Op{"KMOVB", "KMOVW", "KMOVD", "KMOVQ"} { + for _, form := range forms { + for _, instructionOp := range []Op{"", op, Op(strings.ToLower(string(op)))} { + name := fmt.Sprintf("%s/%s/%s/op=%q", arch, op, form.name, instructionOp) + t.Run(name, func(t *testing.T) { + ins := Instr{Op: instructionOp, Args: []Operand{form.src, form.dst}} + registered := ins + registered.Op = op + c, _ := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{registered}}, FuncSig{Ret: Void}, nil) + c.goarch = arch + if ok, terminated, err := c.lowerVec(op, ins); !ok || terminated || err != nil { + t.Fatalf("lowerVec = (%v, %v, %v)", ok, terminated, err) + } + if ins.Op != instructionOp { + t.Fatal("dispatch mutated its caller's instruction") + } + }) + } + } + } + } +} + +func TestAMD64MaskMoveDispatchRejectsContradictoryIdentity(t *testing.T) { + for _, tc := range []struct { + name string + dispatch Op + source Op + }{ + {"different_width", "KMOVB", "KMOVQ"}, + {"different_family", "KMOVB", "MOVQ"}, + {"unregistered_suffix", "KMOVB", "KMOVB.Z"}, + {"unknown_source", "KMOVB", "UNKNOWN"}, + {"unknown_dispatch", "UNKNOWN", "KMOVB"}, + {"both_unknown", "UNKNOWN", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + ins := Instr{Op: tc.source, Args: []Operand{{Kind: OpReg, Reg: "K1"}, {Kind: OpReg, Reg: "K2"}}} + c, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + if err := c.lowerMaskMove(tc.dispatch, ins); err == nil { + t.Fatal("accepted an unknown or contradictory mask-move opcode") + } + if output.String() != before { + t.Fatal("invalid opcode emitted IR before rejection") + } + }) + } +} diff --git a/amd64_mask_vector_forms.go b/amd64_mask_vector_forms.go new file mode 100644 index 000000000..0270b8d41 --- /dev/null +++ b/amd64_mask_vector_forms.go @@ -0,0 +1,75 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86MaskVectorSpec struct { + laneBits int + maskToVector bool +} + +// Go's complete _yvpbroadcastmb2q/_yvpmovb2m rows are EVEX register-only +// moves. K0 is data, not the absence of writemasking; no suffix is permitted. +var x86MaskVectorSpecs = map[Op]x86MaskVectorSpec{ + "VPMOVM2B": {8, true}, "VPMOVM2W": {16, true}, + "VPMOVM2D": {32, true}, "VPMOVM2Q": {64, true}, + "VPMOVB2M": {8, false}, "VPMOVW2M": {16, false}, + "VPMOVD2M": {32, false}, "VPMOVQ2M": {64, false}, +} + +type x86MaskVectorForm struct { + spec x86MaskVectorSpec + vector, mask int + byteWidth int +} + +// Syntax and physical register roles are shared by lowering and caller effects. +// The form supplies no mask value, initialized vector bytes or entry contract. +func x86MaskVectorProperties(ins Instr, arch string) (x86MaskVectorForm, bool, error) { + base, _, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86MaskVectorSpecs[Op(base)] + if !recognized { + return x86MaskVectorForm{}, false, nil + } + form := x86MaskVectorForm{spec: spec, vector: 0, mask: 1} + if spec.maskToVector { + form.vector, form.mask = 1, 0 + } + fail := func() (x86MaskVectorForm, bool, error) { + return form, true, fmt.Errorf("%s %s requires register-only K0-K7 and X/Y/Z operands without suffix: %q", arch, base, ins.Raw) + } + if arch == "" { + arch = "amd64" + } + if arch != "amd64" && arch != "386" || dotted || len(ins.Args) != 2 || !amd64IsKOperand(ins.Args[form.mask]) { + return fail() + } + vector := ins.Args[form.vector] + form.byteWidth = amd64VectorByteWidth(vector.Reg) + if !amd64GoPackedVectorMoveRegister(vector, form.byteWidth, arch) { + return fail() + } + return form, true, x86Mode32VectorRegisterContext(ins, arch) +} + +func x86CallerMaskVectorStep(ins *Instr, form x86MaskVectorForm, state *x86SourceStackState) bool { + mask := ins.Args[form.mask].Reg + vector := x86CallerVectorRoot(ins.Args[form.vector].Reg) + if form.spec.maskToVector { + if !state.masks[mask] { + return false + } + // EVEX writes the selected vector and clears every wider alias byte. + state.vectors[vector] = ^uint64(0) + return true + } + width := x86CallerVectorByteMask(int64(form.byteWidth)) + if state.vectors[vector]&width != width { + return false // An entry/partially initialized vector is not numeric data. + } + // Sign bits fill the low mask lanes and all remaining K bits become zero. + state.masks[mask] = true + return true +} diff --git a/amd64_mask_vector_runtime_test.go b/amd64_mask_vector_runtime_test.go new file mode 100644 index 000000000..d1428f16a --- /dev/null +++ b/amd64_mask_vector_runtime_test.go @@ -0,0 +1,132 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Observe both directions independently: a round trip alone could let two +// inverse mistakes agree. The reference does not consult the production spec. +func x86MaskVectorRuntimeFixture(t *testing.T) (string, map[string]FuncSig, string) { + t.Helper() + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, row := range x86OfficialMaskVectorRows(t) { + for _, mask := range []string{"K0", "K7"} { + name := fmt.Sprintf("maskvector%d", count) + count++ + instruction := strings.ReplaceAll(row, "K1", mask) + toVector := strings.HasPrefix(instruction, "VPMOVM2") + fields := strings.Fields(instruction) + vector := strings.TrimPrefix(strings.TrimSuffix(fields[1], ","+mask), mask+",") + width := map[byte]int{'X': 16, 'Y': 32, 'Z': 64}[vector[0]] + lane := strings.TrimPrefix(fields[0], "VPMOV") + lane = strings.TrimPrefix(strings.TrimSuffix(lane, "2M"), "M2") + laneBytes := map[string]int{"B": 1, "W": 2, "D": 4, "Q": 8}[lane] + if width == 0 || laneBytes == 0 { + t.Fatal("unmodeled independent mask/vector runtime spelling", row) + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ in+8(FP),BX\nMOVQ mask+16(FP),DX\n", name) + fmt.Fprintf(&source, "KMOVQ DX,%s\nMOVQ $0x8123456789abcdef,DI\nKMOVQ DI,K2\nVMOVUPS (BX),Z1\nCMPQ AX,AX\nSTC\n", mask) + fmt.Fprintln(&source, instruction) + fmt.Fprintf(&source, "VMOVUPS Z1,(AX)\nKMOVQ %s,DX\nMOVQ DX,64(AX)\nKMOVQ K2,DX\nMOVQ DX,72(AX)\nSETCS 80(AX)\nSETNE 81(AX)\nRET\n", mask) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkMaskVector(%s,\"%s\",%d,%d,%d)) return 1;\n", name, name, width, laneBytes, x86FloatAddSubBoolInt(toVector)) + } + } + if count != 48 { + t.Fatalf("independent complete runtime program denominator: %d", count) + } + oracle := `#include +#include +#include +` + declarations.String() + ` +static int checkMaskVector(void (*program)(uint8_t*,const uint8_t*,uint64_t), + const char *name,unsigned width,unsigned lane,int toVector) { + const uint64_t masks[]={0,1,UINT64_MAX,UINT64_C(0x55aa55aa55aa55aa),UINT64_C(1)<<63}; + for(unsigned round=0;round<4;round++) for(unsigned m=0;m<5;m++) { + uint8_t input[66],original[66],output[84]; + for(unsigned i=0;i<66;i++) input[i]=(uint8_t)(i*19+round*83+129); + memcpy(original,input,sizeof(input)); + memset(output,0xc3,sizeof(output)); + uint32_t saved,before=0x1f80|(round<<13)|round,after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + program(output+1,input+1,masks[m]); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + uint64_t gotMask,otherMask,wantMask=masks[m]; + memcpy(&gotMask,output+65,8); + memcpy(&otherMask,output+73,8); + if(!toVector) { + wantMask=0; + for(unsigned element=0;element>7)<=width ? 0 : (masks[m]>>(i/lane)&1) ? 255 : 0; + if(output[1+i]!=want) { + fprintf(stderr,"%s byte=%u got=%x want=%x\n",name,i,output[1+i],want); + return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + return source.String(), sigs, oracle +} + +func runX86MaskVectorRuntimeFixture(t *testing.T, llc, clang, triple string, prefix []string) { + t.Helper() + source, sigs, oracle := x86MaskVectorRuntimeFixture(t) + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "mask-vector-complete-bytes", triple, ir, oracle, prefix) + t.Log("48 independent direction/lane/width/K0-K7 programs; 20 complete byte/alias/mask/flags/MXCSR/canary observations each") +} + +func TestX86MaskVectorCompleteByteRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the mandatory cross-runtime counterpart") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang required") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + runX86MaskVectorRuntimeFixture(t, llc, clang, triple, prefix) +} + +func TestCrossLinuxRuntimeMatrixX86MaskVectorCompleteByteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + runX86MaskVectorRuntimeFixture(t, llc, clang, triple, prefix) +} diff --git a/amd64_memory_address_role.go b/amd64_memory_address_role.go new file mode 100644 index 000000000..6b6781838 --- /dev/null +++ b/amd64_memory_address_role.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "math" + "strings" +) + +// NAME_EXTERN/NAME_STATIC in the non-shared Go 386 encoder discards Reg, +// including a nonscaled SP/BP annotation. Index remains an actual address +// input. AMD64 uses RIP and rejects either annotation. This describes address +// roles, not backing, initialized bytes, alignment or caller permissions. +func x86SBMemoryAddressRole(operand Operand, arch string) (MemRef, bool) { + var memory MemRef + switch operand.Kind { + case OpSym: + memory.Sym = operand.Sym + case OpMem: + memory = operand.Mem + if x86SymbolIndexedMemoryError(memory, arch) != nil { + return MemRef{}, false + } + default: + return MemRef{}, false + } + if arch != "386" && arch != "amd64" || + !strings.HasSuffix(strings.TrimSpace(memory.Sym), "(SB)") { + return MemRef{}, false + } + if arch == "386" { + memory.Base = "" + } + return memory, true +} + +// Native emitters may additionally possess a checked physical SP/BP token. +// This ordinary branch does not create one, including for an effective Index. +// It classifies already validated operands; the typed instruction grammar must +// first reject a bare name or malformed address instead of inventing storage. +func x86OrdinaryNativeMemoryOperand(operand Operand, arch string) bool { + if operand.Kind == OpSym { + return true + } + if operand.Kind != OpMem { + return false + } + memory := operand.Mem + if named, valid := x86SBMemoryAddressRole(operand, arch); valid { + memory = named + } + return memory.Base != SP && memory.Base != BP && memory.Index != SP && memory.Index != BP +} + +// The key includes the complete effective displacement, not Operand.String: +// raw AST memory can hold a displacement in addition to its symbolic offset. +type x86StaticSBAddress struct { + symbol string + offset int64 +} + +func x86StaticSBReadAddress(operand Operand, arch string) (x86StaticSBAddress, bool) { + memory, valid := x86SBMemoryAddressRole(operand, arch) + if !valid || memory.Index != "" || memory.Scale != 0 || memory.IndexExt != "" || + memory.OffRaw != "" || memory.Segment != "" { + return x86StaticSBAddress{}, false + } + base, offset, valid := parseSBRef(memory.Sym) + if !valid || base == "" || strings.IndexAny(base, "$+(),[] \t") >= 0 || + memory.Off > 0 && offset > math.MaxInt64-memory.Off || + memory.Off < 0 && offset < math.MinInt64-memory.Off { + return x86StaticSBAddress{}, false + } + offset += memory.Off + if offset < 0 { + return x86StaticSBAddress{}, false + } + return x86StaticSBAddress{symbol: base, offset: offset}, true +} diff --git a/amd64_minmax_runtime_test.go b/amd64_minmax_runtime_test.go new file mode 100644 index 000000000..1c5855c53 --- /dev/null +++ b/amd64_minmax_runtime_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86MinMaxMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" { + t.Skip("execution requires amd64 or Rosetta; five-target objects remain required") + } + + var source, declarations strings.Builder + checks := map[string]*strings.Builder{"views": {}, "memory": {}} + sigs := map[string]FuncSig{} + index := 0 + for _, op := range x86MinMaxTestOps { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + signed, minimum := 0, 0 + if op[5] == 'S' { + signed = 1 + } + if strings.HasPrefix(op, "VPMIN") { + minimum = 1 + } + + for w, width := range []string{"X", "Y", "Z"} { + // Register/memory unmasked, register/memory masked, masked broadcast. + for mode := 0; mode < 5; mode++ { + if mode == 4 && bits < 32 { + continue + } + for _, zero := range []bool{false, true} { + if zero && mode < 2 { + continue + } + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("minmax%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\n") + source.WriteString("VMOVUPS (AX),Z0\n") + + input := "(BX)" + registerInput := mode == 0 || mode == 2 + if registerInput { + source.WriteString("VMOVUPS (BX),Z1\n") + input = width + "1" + } else if raw { + source.WriteString("VMOVUPS (AX),Z1\n") + } + suffix, mask := "", "" + if mode == 4 { + suffix = ".BCST" + } + if zero { + suffix += ".Z" + } + if mode >= 2 { + source.WriteString("KMOVQ mask+24(FP),K1\n") + mask = "K1," + } + + destination, oldB := 0, 0 + if raw { + destination = 1 + if registerInput { + oldB = 1 + } + } + line := fmt.Sprintf("%s%s %s,%s0,%s%s%d", op, suffix, input, width, mask, width, destination) + emitX86GoEncodedTestInstruction(t, &source, line, raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\n", destination) + fmt.Fprintf(&source, "VMOVUPS Y%d,64(DI)\n", destination) + fmt.Fprintf(&source, "VMOVUPS X%d,96(DI)\nRET\n", destination) + + sigs[name] = FuncSig{ + Name: name, + Args: []LLVMType{Ptr, Ptr, Ptr, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void*, const void*, void*, uint64_t);\n", name) + kind, zeroing := "views", 0 + if mode >= 2 { + kind = "memory" + } + if zero { + zeroing = 1 + } + fmt.Fprintf(checks[kind], " if (check(%s, %d, %d, %d, %d, %d, %d, %d)) {\n", name, 16<> (n % 32); + uint64_t active = mask & low_mask(lanes); + for (int i = 0; i < lanes; i++) { + uint64_t av = lane(a + i * bytes, bytes); + uint64_t bv = lane(b + (mode == 4 ? 0 : i * bytes), bytes); + uint64_t result; + if (mode < 2 || ((mask >> i) & 1)) { + result = minmax(av, bv, bytes * 8, sign, minimum); + } else { + result = zero ? 0 : old_b ? lane(b + i * bytes, bytes) : av; + } + memcpy(want + i * bytes, &result, bytes); + } + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + fn(a, mode >= 3 && !active ? NULL : b, out, mask); + if (memcmp(out, want, sizeof(out))) { + fprintf(stderr, "width=%d bytes=%d sign=%d min=%d mode=%d zero=%d n=%d\n", + width, bytes, sign, minimum, mode, zero, n); + return 1; + } + } + + if (mode >= 3) { + for (int enabled = 1; enabled <= lanes; enabled++) { + uint8_t a[64], out[112], want[112] = {0}; + memset(a, 255, sizeof(a)); + int readable = mode == 4 ? bytes : enabled * bytes; + uint8_t *b = guard + page - readable; + memset(b, 255, (size_t)readable); + + for (int i = 0; i < lanes; i++) { + if (i < enabled || !zero) { + memset(want + i * bytes, 255, (size_t)bytes); + } + } + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + fn(a, b, out, low_mask(enabled)); + if (memcmp(out, want, sizeof(out))) { + return 1; + } + } + } + return 0; +} +` diff --git a/amd64_minpos_runtime_test.go b/amd64_minpos_runtime_test.go new file mode 100644 index 000000000..31d86e622 --- /dev/null +++ b/amd64_minpos_runtime_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86MinimumPositionMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" { + t.Skip("execution requires amd64 or Rosetta; native amd64 CI and five-target objects are required") + } + + for _, raw := range []bool{false, true} { + t.Run(fmt.Sprintf("raw=%t", raw), func(t *testing.T) { + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for vex, op := range []string{"PHMINPOSUW", "VPHMINPOSUW"} { + for _, operand := range []string{"(BX)", "X1", "X0"} { + name := fmt.Sprintf("minpos%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\n", name) + source.WriteString("MOVQ old+0(FP),AX\nMOVQ input+8(FP),BX\nMOVQ out+16(FP),DI\n") + source.WriteString("VMOVUPS (AX),Z0\nVMOVUPS (AX),Z1\n") + if operand != "(BX)" { + fmt.Fprintf(&source, "MOVOU (BX),%s\n", operand) + } + emitX86GoEncodedTestInstruction(t, &source, op+" "+operand+",X0", raw) + source.WriteString("VMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n") + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void*, const void*, void*);\n", name) + fmt.Fprintf(&checks, " if (check(%s, %d)) {\n", name, vex) + fmt.Fprintf(&checks, " fprintf(stderr, \"%s failed\\n\");\n return 1;\n }\n", name) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + body := "#include \n#include \n#include \n" + + x86TestGuardPagesC + declarations.String() + x86MinimumPositionOracleC + + "\nint main(void) {\n if (setup_guard()) return 2;\n" + + checks.String() + " return free_guard();\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "minpos", triple, ir, body, prefix) + }) + } +} + +const x86MinimumPositionOracleC = ` +static int check(void (*fn)(const void*, const void*, void*), int vex) { + const uint16_t values[] = {0, 1, 0x7fff, 0x8000, 0x8001, 0xfffe, 0xffff}; + for (int n = 0; n < 256; n++) { + uint8_t old[65], storage[17], out[112], want[112] = {0}; + uint8_t *input = storage + 1; + for (int i = 0; i < 65; i++) old[i] = (uint8_t)(i * 3 + 31); + + uint16_t minimum = 0xffff, position = 0; + for (int i = 0; i < 8; i++) { + uint16_t value = n < 8 ? (i == n ? 0 : 0xffff) : + values[(n + i * (n % 5)) % 7]; + memcpy(input + i * 2, &value, 2); + if (value < minimum) { + minimum = value; + position = (uint16_t)i; + } + } + memcpy(want, &minimum, 2); + memcpy(want + 2, &position, 2); + if (!vex) memcpy(want + 16, old + 1 + 16, 48); + memcpy(want + 64, want, 32); + memcpy(want + 96, want, 16); + + for (int edge = 0; edge < 2; edge++) { + const uint8_t *source = input; + if (edge) { + source = guard + page - 16; + memcpy(guard + page - 16, input, 16); + } + memset(out, 0xa5, sizeof(out)); + fn(old + 1, source, out); + if (memcmp(out, want, sizeof(out))) { + fprintf(stderr, "vex=%d n=%d edge=%d\n", vex, n, edge); + return 1; + } + } + } + return 0; +} +` diff --git a/amd64_mode32_object_source_test.go b/amd64_mode32_object_source_test.go new file mode 100644 index 000000000..6375c1a54 --- /dev/null +++ b/amd64_mode32_object_source_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "errors" + "regexp" + "strconv" + "strings" + "testing" +) + +var x86Mode32ObjectHighBank = regexp.MustCompile(`\b[XY](?:[89]|[12][0-9]|3[01])\b`) + +// Keep the complete original source acceptance and its missing physical +// binding observable. The returned low-bank fixture tests object generation, +// not equivalence to the original high-bank machine code or runtime behavior. +func x86PhysicalMode32ObjectSource(t *testing.T, source, arch string) string { + t.Helper() + if arch != "386" { + return source + } + goSource := strings.ReplaceAll(source, ",NOSPLIT", ",4") + requireX86GoAssemblerResult(t, arch, goSource, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + highRows := 0 + for _, function := range file.Funcs { + for _, instruction := range function.Instrs { + if !errors.Is(x86Mode32VectorRegisterContext(instruction, arch), ErrProbeNeedsContext) { + continue + } + highRows++ + one, err := Parse(ArchAMD64, "TEXT highbank(SB),4,$0-0\n"+instruction.Raw+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(one, Options{ + Goarch: arch, TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"highbank": {Name: "highbank", Ret: Void}}, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("original high bank acquired an unproved physical binding: %s: %v", instruction.Raw, err) + } + } + } + if highRows == 0 { + t.Fatal("mode32 fixture lost its original high-bank source coverage") + } + physical := x86Mode32ObjectHighBank.ReplaceAllStringFunc(source, func(register string) string { + index, err := strconv.Atoi(register[1:]) + if err != nil { + t.Fatal(err) + } + return register[:1] + strconv.Itoa(index%8) + }) + requireX86GoAssemblerResult(t, arch, strings.ReplaceAll(physical, ",NOSPLIT", ",4"), true) + return physical +} diff --git a/amd64_mode32_vector_bindings_test.go b/amd64_mode32_vector_bindings_test.go new file mode 100644 index 000000000..b1255e01a --- /dev/null +++ b/amd64_mode32_vector_bindings_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "errors" + "fmt" + "sort" + "strings" + "testing" +) + +type mode32VectorFamilyCase struct { + name string + rows []string + properties func(Instr, string) (bool, error) + lower func(*amd64Ctx, Instr) (bool, bool, error) +} + +// Existing SDK-table completeness tests independently bind these families to +// Go. The actual Go object here observes syntax acceptance, not register state. +func mode32VectorFamilyCases() []mode32VectorFamilyCase { + logical := mode32VectorFamilyCase{ + name: "logical", + properties: func(ins Instr, arch string) (bool, error) { + _, recognized, err := x86PackedLogicalProperties(ins, arch) + return recognized, err + }, + lower: func(ctx *amd64Ctx, ins Instr) (bool, bool, error) { + return ctx.lowerPackedLogicalFamily(ins.Op, ins) + }, + } + for _, opcode := range x86LogicalOpcodeNames() { + if x86PackedLogicalSpecs[opcode].encoding > x86PackedLogicalLegacyInteger { + logical.rows = append(logical.rows, x86LogicalAcceptedForms(opcode, "386")...) + } + } + move := mode32VectorFamilyCase{ + name: "move", + properties: func(ins Instr, arch string) (bool, error) { + _, recognized, err := amd64PackedVectorMoveProperties(ins, arch) + return recognized, err + }, + lower: func(ctx *amd64Ctx, ins Instr) (bool, bool, error) { + return ctx.lowerPackedVectorMove(ins.Op, ins) + }, + } + var moves []string + for opcode := range amd64PackedVectorMoveSpecs { + moves = append(moves, string(opcode)) + } + sort.Strings(moves) + for _, opcode := range moves { + spec := amd64PackedVectorMoveSpecs[Op(opcode)] + high := 31 + if spec.encoding == amd64PackedMoveVEX { + high = 15 + } + for _, bank := range []string{"X", "Y"} { + reg := fmt.Sprintf("%s%d", bank, high) + for _, args := range []string{"(AX)," + reg, reg + ",(AX)", reg + "," + bank + "0"} { + move.rows = append(move.rows, opcode+" "+args) + } + if spec.encoding != amd64PackedMoveVEX { + for _, args := range []string{"(AX),K7," + reg, reg + ",K7,(AX)", reg + ",K7," + bank + "0"} { + move.rows = append(move.rows, opcode+" "+args, opcode+".Z "+args) + } + } + } + } + fma := mode32VectorFamilyCase{ + name: "FMA3", + properties: func(ins Instr, arch string) (bool, error) { + _, recognized, err := amd64FMA3Properties(ins, arch) + return recognized, err + }, + lower: func(ctx *amd64Ctx, ins Instr) (bool, bool, error) { + return ctx.lowerFMA3(ins.Op, ins) + }, + } + // Scalar rounding, memory and packed broadcast axes remain in the source. + // Z0-Z7 are physically available; this regression targets the X/Y banks. + for _, row := range x86FMA3NamedFormats("386") { + changed := strings.NewReplacer("X2", "X16", "Y2", "Y16").Replace(row) + if changed != row { + fma.rows = append(fma.rows, changed) + } + } + return []mode32VectorFamilyCase{logical, move, fma} +} + +func TestX86Mode32VectorFamiliesRequirePhysicalBindings(t *testing.T) { + checked := 0 + for _, family := range mode32VectorFamilyCases() { + for _, row := range family.rows { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + high := false + for _, operand := range instruction.Args { + width := amd64VectorByteWidth(operand.Reg) + index, valid := amd64VectorRegisterIndex(operand.Reg, width) + high = high || operand.Kind == OpReg && valid && index >= 8 + } + if !high { + continue + } + checked++ + t.Run(family.name+"/"+strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + requireX86GoAssemblerResult(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", true) + if recognized, err := family.properties(instruction, "386"); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go-accepted high bank became a physical mode32 binding: recognized=%v err=%v", recognized, err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := output.String() + if recognized, _, err := family.lower(ctx, instruction); !recognized || + !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("unproved high-bank source emitted IR: recognized=%v err=%v", recognized, err) + } + }) + } + } + if checked != 664 { + t.Fatalf("Go-accepted high-bank inventory changed: checked %d rows, want 664", checked) + } +} + +// This checks only the shared identity predicate. Family grammar tests above +// independently prove source acceptance, and the object tests retain low-bank +// success; this helper cannot manufacture a supported source operand. +func TestX86Mode32VectorRegisterContextBindsEveryOperand(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, index := range []int{0, 7, 8, 15, 16, 31} { + for position := 0; position < 4; position++ { + name := fmt.Sprintf("%s/%s%d/operand%d", arch, bank, index, position) + t.Run(name, func(t *testing.T) { + args := make([]Operand, 4) + for i := range args { + args[i] = Operand{Kind: OpReg, Reg: Reg(bank + "0")} + } + args[position].Reg = Reg(fmt.Sprintf("%s%d", bank, index)) + instruction := Instr{Op: "VECTOR_IDENTITY", Args: args} + err := x86Mode32VectorRegisterContext(instruction, arch) + if arch == "386" && index >= 8 { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved physical identity accepted: %v", err) + } + } else if err != nil { + t.Fatalf("physical register bank rejected: %v", err) + } + }) + } + } + } + } +} diff --git a/amd64_mov_extension_caller_test.go b/amd64_mov_extension_caller_test.go new file mode 100644 index 000000000..7115f0e8d --- /dev/null +++ b/amd64_mov_extension_caller_test.go @@ -0,0 +1,468 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" +) + +var x86ExtensionTestOperations = []string{ + "MOVBWSX", "MOVBWZX", "MOVBLSX", "MOVBLZX", "MOVBQSX", "MOVBQZX", + "MOVWLSX", "MOVWLZX", "MOVWQSX", "MOVWQZX", "MOVLQSX", "MOVLQZX", + "MOVSWW", "MOVZWW", +} + +func TestX86ExtensionCompletePrimaryTables(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86", "asm6.go") + bytes, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + table := string(bytes) + for _, name := range []string{"ymb_rl", "yml_rl"} { + pattern := regexp.MustCompile(`(?s)var ` + name + ` = \[\]ytab\{\s*\{(Zmb_r|Zm_r), 1, argList\{(Ymb|Yml), Yrl\}\},\s*\}`) + if !pattern.MatchString(table) { + t.Fatalf("actual Go table changed: %s", name) + } + } + rows := regexp.MustCompile(`\{A(MOV[A-Z]+), (ymb_rl|yml_rl), (Pm|Pw|Pq|Pe|Px),`).FindAllStringSubmatch(table, -1) + var names []string + for _, row := range rows { + names = append(names, row[1]) + } + want := append([]string(nil), x86ExtensionTestOperations...) + sort.Strings(names) + sort.Strings(want) + if strings.Join(names, ",") != strings.Join(want, ",") { + t.Fatalf("complete extension rows changed: got %v, want %v", names, want) + } +} + +func x86ExtensionGoListing(t *testing.T, arch, instruction string) (string, error) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "extension.s") + source := "TEXT ·extension(SB),4,$0-0\n" + instruction + "\nRET\n" + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "extension", + "-o", filepath.Join(dir, "extension.o"), path) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + output, err := runtimeOracleCombinedOutput(ctx, command) + return string(output), err +} + +func TestX86ExtensionActualGoAliasAudit(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, instruction := range []string{ + "MOVSWW AL,AX", "MOVSWW AH,AX", "MOVSWW BP,AX", + "MOVZWW AL,AX", "MOVZWW AH,AX", "MOVZWW BP,AX", + "MOVBLSX AH,AX", "MOVBLSX BP,AX", "MOVLQZX AX,BX", + } { + t.Run(arch+"/"+strings.ReplaceAll(instruction, " ", "_"), func(t *testing.T) { + listing, err := x86ExtensionGoListing(t, arch, instruction) + if err != nil { + t.Fatalf("actual %s assembler: %v\n%s", runtime.Version(), err, listing) + } + t.Log(listing) + }) + } + } +} + +func TestX86ExtensionGoPhysicalRegisterViews(t *testing.T) { + for _, test := range []struct { + instruction, bytes string + source Reg + }{ + {"MOVSWW AL,AX", "66 0f bf c0 c3", AX}, + {"MOVSWW AH,AX", "66 0f bf c4 c3", SP}, + {"MOVZWW CH,AX", "66 0f b7 c5 c3", BP}, + {"MOVSWW DH,AX", "66 0f bf c6 c3", SI}, + {"MOVZWW BH,AX", "66 0f b7 c7 c3", DI}, + {"MOVBLSX AH,AX", "0f be c4 c3", AH}, + {"MOVBLSX AH,R8", "44 0f be c4 c3", SP}, + {"MOVBQSX BH,AX", "48 0f be c7 c3", DI}, + } { + t.Run(test.instruction, func(t *testing.T) { + listing, err := x86ExtensionGoListing(t, "amd64", test.instruction) + if err != nil || !strings.Contains(listing, test.bytes) { + t.Fatalf("actual Go machine view: %v\n%s", err, listing) + } + file, err := Parse(ArchAMD64, "TEXT v(SB),4,$0-0\n"+test.instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := append([]Operand(nil), ins.Args...) + form, recognized, err := x86ScalarExtensionProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || form.source.Reg != test.source || !reflect.DeepEqual(before, ins.Args) { + t.Fatalf("Go ModRM view/source mutation: %+v recognized=%t err=%v", form, recognized, err) + } + }) + } +} + +func TestX86ExtensionCompleteFormatsThreeAPIFiveObjects(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var body strings.Builder + for _, operation := range x86ExtensionTestOperations { + if target.arch == "386" && (operation == "MOVBQSX" || operation == "MOVBQZX" || + operation == "MOVWQSX" || operation == "MOVWQZX" || operation == "MOVLQSX") { + continue + } + for _, source := range []string{"AX", "8(BX)", "-1(BX)(SI*2)", "x+0(FP)", "@WORD(SP)", "·payload(SB)"} { + fmt.Fprintf(&body, "%s %s,CX\n", operation, source) + } + if strings.HasPrefix(operation, "MOVB") || operation == "MOVSWW" || operation == "MOVZWW" { + fmt.Fprintf(&body, "%s BL,CX\n", operation) + if target.arch == "386" { + fmt.Fprintf(&body, "%s DI,CX\n", operation) + } + } + } + body.WriteString("@MOV CX,ret+@WORD(FP)\nRET") + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body.String()) + source += "GLOBL ·payload(SB),8,$16\nDATA ·payload+0(SB)/8,$0x1122334455667788\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions(target.arch, target.triple, sig) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "extension-form.ll", "extension-form.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, target.triple, "extension-form-module.ll", "extension-form-module.o", module.String()) + ctx = llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, + }) + if err != nil { + t.Fatal(err) + } + defer translation.Module.Dispose() + compileLLVMToObject(t, llc, target.triple, "extension-form-go.ll", "extension-form-go.o", translation.Module.String()) + }) + } +} + +func TestX86ExtensionSourceAndPhysicalDecoderShareEffects(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + for _, body := range []string{operation + " BX,AX", x86ExtensionNumericalRawBytes(operation)} { + for _, defined := range []bool{false, true} { + prefix := "" + if defined { + prefix = "MOVQ $0xff80,BX\n" + } + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", prefix+body+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if defined && err != nil || !defined && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("source/physical input availability defined=%t: %v\n%s", defined, err, body) + } + } + } + } +} + +func TestX86ExtensionSharedFormIsOrderedAndImmutable(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + file, err := Parse(ArchAMD64, "TEXT v(SB),4,$0-0\n"+operation+" AX,BX\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, spelling := range []Op{original.Op, "", "MOVQ"} { + instruction := original + instruction.Op = spelling + before := append([]Operand(nil), instruction.Args...) + _, recognized, err := x86ScalarExtensionProperties(original.Op, instruction, "amd64") + if !recognized || (err == nil) != (spelling != "MOVQ") || !reflect.DeepEqual(before, instruction.Args) { + t.Fatalf("%s spelling %s: recognized=%t err=%v source changed=%t", operation, spelling, recognized, err, !reflect.DeepEqual(before, instruction.Args)) + } + } + } +} + +func TestX86ExtensionActualGoRejectedFormats(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, operation := range x86ExtensionTestOperations { + if target.arch == "386" && amd64ScalarExtensionMoveSpecs[Op(operation)].rejectedOn386 { + requireX86GoAssemblerResult(t, target.arch, "TEXT ·extension(SB),4,$0-0\n"+operation+" AX,BX\nRET\n", false) + continue + } + for _, operands := range []string{"$1,AX", "AX,(BX)", "X1,AX", "AX,AL", "8(BAD),AX"} { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, operation+" "+operands+"\nRET") + requireX86GoAssemblerResult(t, target.arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue // The public parser itself rejected the original format. + } + options := x86SourceStackOptions(target.arch, target.triple, sig) + if _, err := Translate(file, options); err == nil { + t.Fatalf("Translate accepted Go-rejected %s %s", operation, operands) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatalf("TranslateModule accepted Go-rejected %s %s", operation, operands) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatalf("TranslateGoModule accepted Go-rejected %s %s", operation, operands) + } + } + } + } +} + +func TestX86ExtensionRawMOVSXDRequiresKnownNativeWidth(t *testing.T) { + // Register and default-32 memory widths have independent native contracts. + // The 66 memory footprint still differs across vendor descriptions. + for _, encoding := range []string{"BYTE $0x66\nWORD $0x0763", "BYTE $0x66\nWORD $0x0f63"} { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVQ $0x80000000,BX\n"+encoding+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown raw MOVSXD width Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown raw MOVSXD width module: %v", err) + } + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unknown raw MOVSXD width Go module: %v", err) + } + } + } +} + +func TestX86GoCallerRegionExtensionSourceInputs(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + for _, input := range []string{"register", "memory"} { + t.Run(operation+"/"+input, func(t *testing.T) { + operand := "AX" + if input == "memory" { + operand = "(DI)" + } + body := fmt.Sprintf("%s %s,AX\nMOVL AX,(DX)", operation, operand) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } + }) + } + } +} + +func TestX86GoCallerRegionExtensionDefinedBytesAndPointerNegatives(t *testing.T) { + for _, test := range []struct { + name, body string + valid bool + }{ + {"one initialized low byte", "MOVB $128,BX\nMOVBLZX BL,AX", true}, + {"one initialized high byte", "MOVB $128,BH\nMOVBLSX BH,AX", true}, + {"complete low word via aliases", "MOVB $128,BL\nMOVB $255,BH\nMOVSWW BL,AX", true}, + {"word low byte incomplete", "MOVB $128,BL\nMOVSWW BL,AX", false}, + {"unknown source", "MOVLQSX BX,AX", false}, + {"partial source long", "MOVW $7,BX\nMOVLQSX BX,AX", false}, + {"caller pointer sign extension", "MOVLQSX DI,AX", false}, + {"caller pointer byte extension", "MOVBQZX DI,AX", false}, + {"caller pointer apparent high-byte REX", "MOVBQZX BH,AX", false}, + {"implicit SP source", "MOVSWW AH,AX", false}, + {"implicit BP source", "MOVZWW CH,AX", false}, + {"partial pointer destination", "MOVQ DI,BX\nMOVBWZX AL,BX", false}, + {"partial pointer replaced completely", "MOVQ DI,BX\nMOVBLZX AL,BX", true}, + {"unaligned pointer slot", "MOVLQSX a+1(FP),AX", false}, + {"partial pointer FP slot", "MOVLQSX a+0(FP),AX", false}, + {"return PC load", "MOVLQSX 8(BP),AX", false}, + {"saved BP load", "MOVWLZX 0(BP),AX", false}, + {"different defined bytes join", "CMPQ AX,$0\nJE other\nMOVB $7,BL\nother:\nMOVBLZX BL,AX", false}, + } { + t.Run(test.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", test.body+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu") + if test.valid && err != nil || !test.valid && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("valid=%t: %v", test.valid, err) + } + }) + } +} + +func TestX86GoCallerRegionExtensionThreeAPIs(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVLQSX (DI),AX\nMOVL AX,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "extension.ll", "extension.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "extension-module.ll", "extension-module.o", module.String()) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestX86SourceStackExtensionRequiresDefinedLowBytes(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, operation := range x86ExtensionTestOperations { + if target.arch == "386" && amd64ScalarExtensionMoveSpecs[Op(operation)].rejectedOn386 { + continue // Covered separately as actual Go-rejected formats, not Context forms. + } + for _, prefix := range []string{"", "MOVB $7,BX\n", "@MOV x+0(FP),BX\n"} { + body := prefix + operation + " BX,AX\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3" + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions(target.arch, target.triple, sig) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller source acquired extension proof: %v\n%s", err, source) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller source acquired module proof: %v\n%s", err, source) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller source acquired Go module proof: %v\n%s", err, source) + } + } + } + } +} + +func TestX86ExtensionCannotBorrowStaleCallerMemoryEffects(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "MOVLQSX (DI),AX\nMOVL AX,(DX)", 1) + for _, change := range []string{"width", "opcode", "source", "missing caller"} { + t.Run(change, func(t *testing.T) { + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs, ResolveSym: resolve} + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + for index := range normalized.Funcs[0].Instrs { + instruction := &normalized.Funcs[0].Instrs[index] + if instruction.Op != "MOVLQSX" { + continue + } + switch change { + case "width": + access := instruction.x86CallerMemory[0] + access.width = 8 // Actual read width is 4, never the result width. + instruction.x86CallerMemory[0] = access + case "opcode": + instruction.Op = "MOVWQSX" + case "source": + instruction.Args[0].Mem.Base = DX + } + } + if change == "missing caller" { + sig := sigs["caller.kernel"] + sig.x86CallerRegions = nil + sigs["caller.kernel"] = sig + } + if change == "width" { + if x86RawReturnMemoryBound(normalized.Funcs[0], normalized, options, true) { + t.Fatal("extension accepted a stale destination-width memory certificate") + } + return // A final consumer legitimately recomputes unchanged source effects. + } + if _, err := Translate(normalized, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed %s borrowed an extension caller proof: %v", change, err) + } + }) + } +} diff --git a/amd64_mov_extension_effects.go b/amd64_mov_extension_effects.go new file mode 100644 index 000000000..57377eb9f --- /dev/null +++ b/amd64_mov_extension_effects.go @@ -0,0 +1,139 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64ScalarExtensionMoveSpec struct { + sourceBits int + destinationBits int + signed bool + rejectedOn386 bool + byteClass bool +} + +// These fourteen rows use Go's ymb_rl or yml_rl tables. MOVSWW/MOVZWW +// read a word despite using ymb_rl and its byte-register ModRM spellings. +var amd64ScalarExtensionMoveSpecs = map[Op]amd64ScalarExtensionMoveSpec{ + "MOVBWSX": {sourceBits: 8, destinationBits: 16, signed: true, byteClass: true}, + "MOVBWZX": {sourceBits: 8, destinationBits: 16, byteClass: true}, + "MOVBLSX": {sourceBits: 8, destinationBits: 32, signed: true, byteClass: true}, + "MOVBLZX": {sourceBits: 8, destinationBits: 32, byteClass: true}, + "MOVBQSX": {sourceBits: 8, destinationBits: 64, signed: true, rejectedOn386: true, byteClass: true}, + "MOVBQZX": {sourceBits: 8, destinationBits: 64, rejectedOn386: true, byteClass: true}, + "MOVWLSX": {sourceBits: 16, destinationBits: 32, signed: true}, + "MOVWLZX": {sourceBits: 16, destinationBits: 32}, + "MOVWQSX": {sourceBits: 16, destinationBits: 64, signed: true, rejectedOn386: true}, + "MOVWQZX": {sourceBits: 16, destinationBits: 64, rejectedOn386: true}, + "MOVLQSX": {sourceBits: 32, destinationBits: 64, signed: true, rejectedOn386: true}, + "MOVLQZX": {sourceBits: 32, destinationBits: 64}, + "MOVSWW": {sourceBits: 16, destinationBits: 16, signed: true, byteClass: true}, + "MOVZWW": {sourceBits: 16, destinationBits: 16, byteClass: true}, +} + +type x86ScalarExtensionForm struct { + spec amd64ScalarExtensionMoveSpec + source Operand + result Reg +} + +func x86ScalarExtensionProperties(op Op, ins Instr, arch string) (x86ScalarExtensionForm, bool, error) { + op = Op(strings.ToUpper(string(op))) + spec, recognized := amd64ScalarExtensionMoveSpecs[op] + if !recognized { + return x86ScalarExtensionForm{}, false, nil + } + fail := func(reason string) (x86ScalarExtensionForm, bool, error) { + return x86ScalarExtensionForm{}, true, fmt.Errorf("%s %s %s: %q", arch, op, reason, ins.Raw) + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) || len(ins.Args) != 2 { + return fail("requires matching opcode and two operands") + } + if arch == "386" && spec.rejectedOn386 { + return fail("is outside Go's 386 extension tables") + } + if arch == "386" && op == "MOVLQZX" { + // Px is an ordinary MOVL in 386 mode, not a 64-bit write. + spec.destinationBits = 32 + } + source, destination := ins.Args[0], ins.Args[1] + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, arch) { + return fail("expects a Yrl destination register") + } + if source.Kind == OpReg { + valid := isX86YrlRegisterForArch(source.Reg, arch) + if spec.byteClass { + valid = isGoYmbRegisterForArch(source.Reg, arch) + } + if !valid { + return fail("source register is outside its Go class") + } + if spec.byteClass && spec.sourceBits == 16 { + source.Reg = amd64YmbEffectiveRegister(source.Reg, 16) + } else if spec.sourceBits == 8 && arch != "386" && + (spec.destinationBits == 64 || x86ExtensionHighRegister(destination.Reg)) { + // A REX prefix changes ModRM 4..7 from AH/CH/DH/BH to the + // low bytes of SP/BP/SI/DI. Go emits those actual bytes too. + switch source.Reg { + case AH, CH, DH, BH: + source.Reg = amd64YmbEffectiveRegister(source.Reg, 16) + } + } + } else if !isAMD64MemoryOperand(source) || + source.Kind == OpSym && !x86PackedShiftMemoryValid(source, arch) { + return fail("expects a register or memory source") + } + if source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, arch) { + return fail("memory source has an invalid address register") + } + return x86ScalarExtensionForm{spec: spec, source: source, result: destination.Reg}, true, nil +} + +func x86ExtensionHighRegister(register Reg) bool { + switch register { + case "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15": + return true + default: + return false + } +} + +func x86ScalarMoveMemoryBytes(ins Instr, operand int, arch string) int64 { + if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + if err == nil && operand == 0 && isAMD64MemoryOperand(form.source) { + return int64(form.bits / 8) + } + return 0 + } + op := Op(normalizeInstructionOpcode(ins.Op)) + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); recognized { + if err == nil && operand == 0 { + return 1 + } + return 0 + } + if width := x86RawReturnScalarMoveBytes(op); width != 0 { + return width + } + form, recognized, err := x86ScalarExtensionProperties(op, ins, arch) + if !recognized || err != nil || operand != 0 { + return 0 + } + return int64(form.spec.sourceBits / 8) +} + +func x86CallerExtensionStep(ins *Instr, state *x86SourceStackState, proof *x86GoCallerRegionProof) (bool, bool) { + form, recognized, err := x86ScalarExtensionProperties(ins.Op, *ins, "amd64") + if !recognized || err != nil { + return recognized, false + } + width := int64(form.spec.sourceBits / 8) + if !x86CallerMemoryStep(ins, state, proof, width) { + return true, false + } + if form.source.Kind != OpMem && !x86CallerNumericRead(form.source, width, state) { + return true, false + } + return true, x86CallerNumericWrite(form.result, int64(form.spec.destinationBits/8), state) +} diff --git a/amd64_mov_extension_identity_test.go b/amd64_mov_extension_identity_test.go new file mode 100644 index 000000000..04aedfe87 --- /dev/null +++ b/amd64_mov_extension_identity_test.go @@ -0,0 +1,42 @@ +package plan9asm + +import ( + "reflect" + "strings" + "testing" +) + +func TestX86ExtensionHelperAcceptsMatchingCaseAndEmptyIdentity(t *testing.T) { + for _, operation := range x86ExtensionTestOperations { + for _, identity := range []string{"", operation, strings.ToLower(operation), "Mov" + strings.ToLower(operation[3:])} { + for _, dispatch := range []Op{Op(operation), Op(strings.ToLower(operation))} { + instruction := Instr{Op: Op(identity), Raw: operation + " AX,BX", Args: []Operand{ + {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}, + }} + before := append([]Operand(nil), instruction.Args...) + form, recognized, err := x86ScalarExtensionProperties(dispatch, instruction, "amd64") + if !recognized || err != nil || form.result != BX || !reflect.DeepEqual(before, instruction.Args) || instruction.Op != Op(identity) { + t.Fatalf("%s identity %q: form=%+v recognized=%t err=%v", operation, identity, form, recognized, err) + } + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + if err := ctx.lowerScalarExtensionMove(dispatch, instruction); err != nil { + t.Fatalf("matching dispatch %s / %q: %v", operation, identity, err) + } + } + } + } +} + +func TestX86ExtensionHelperRejectsUnknownOrConflictingDispatchBeforeIR(t *testing.T) { + for _, operation := range []Op{"MOVLQSX", "UNKNOWN_EXTENSION", "MOVQ"} { + instruction := Instr{Op: "MOVWQSX", Raw: "MOVWQSX AX,BX", Args: []Operand{ + {Kind: OpReg, Reg: AX}, {Kind: OpReg, Reg: BX}, + }} + before := append([]Operand(nil), instruction.Args...) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + length := output.Len() + if err := ctx.lowerScalarExtensionMove(operation, instruction); err == nil || output.Len() != length || !reflect.DeepEqual(before, instruction.Args) { + t.Fatalf("dispatch %s: err=%v emitted=%d mutated=%t", operation, err, output.Len()-length, !reflect.DeepEqual(before, instruction.Args)) + } + } +} diff --git a/amd64_mov_extension_runtime_test.go b/amd64_mov_extension_runtime_test.go new file mode 100644 index 000000000..d4ced629f --- /dev/null +++ b/amd64_mov_extension_runtime_test.go @@ -0,0 +1,291 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86ExtensionNumericalCase struct { + op string + source, result int + signed bool + mode, name string +} + +func x86ExtensionNumericalCases() []x86ExtensionNumericalCase { + var cases []x86ExtensionNumericalCase + for _, operation := range []x86ExtensionNumericalCase{ + {op: "MOVBWSX", source: 8, result: 16, signed: true}, + {op: "MOVBWZX", source: 8, result: 16}, + {op: "MOVBLSX", source: 8, result: 32, signed: true}, + {op: "MOVBLZX", source: 8, result: 32}, + {op: "MOVBQSX", source: 8, result: 64, signed: true}, + {op: "MOVBQZX", source: 8, result: 64}, + {op: "MOVWLSX", source: 16, result: 32, signed: true}, + {op: "MOVWLZX", source: 16, result: 32}, + {op: "MOVWQSX", source: 16, result: 64, signed: true}, + {op: "MOVWQZX", source: 16, result: 64}, + {op: "MOVLQSX", source: 32, result: 64, signed: true}, + {op: "MOVLQZX", source: 32, result: 64}, + {op: "MOVSWW", source: 16, result: 16, signed: true}, + {op: "MOVZWW", source: 16, result: 16}, + } { + for _, mode := range []string{"Register", "Memory", "LowAlias", "InPlace", "PhysicalBytes"} { + if mode == "LowAlias" && operation.source == 32 || + mode == "LowAlias" && operation.source == 16 && operation.result != 16 { + continue + } + operation.mode = mode + operation.name = "extension" + operation.op + mode + cases = append(cases, operation) + } + } + // REX.W changes BH to DI's low byte. The source DI is explicitly numeric + // here, not the imported input pointer or an unknown entry register. + for _, operation := range []x86ExtensionNumericalCase{ + {op: "MOVBQZX", source: 8, result: 64}, + {op: "MOVBQSX", source: 8, result: 64, signed: true}, + {op: "MOVSWW", source: 16, result: 16, signed: true}, + {op: "MOVZWW", source: 16, result: 16}, + } { + operation.mode, operation.name = "PhysicalAlias", "extension"+operation.op+"PhysicalAlias" + cases = append(cases, operation) + } + return cases +} + +// Independent fixed encodings for the BX-to-AX forms. In particular LQSX +// requires REX.W; the ambiguous no-REX 63 form is a separate Context test. +func x86ExtensionNumericalRawBytes(operation string) string { + encodings := map[string][]byte{ + "MOVBWSX": {0x66, 0x0f, 0xbe, 0xc3}, + "MOVBWZX": {0x66, 0x0f, 0xb6, 0xc3}, + "MOVBLSX": {0x0f, 0xbe, 0xc3}, + "MOVBLZX": {0x0f, 0xb6, 0xc3}, + "MOVBQSX": {0x48, 0x0f, 0xbe, 0xc3}, + "MOVBQZX": {0x48, 0x0f, 0xb6, 0xc3}, + "MOVWLSX": {0x0f, 0xbf, 0xc3}, + "MOVWLZX": {0x0f, 0xb7, 0xc3}, + "MOVWQSX": {0x48, 0x0f, 0xbf, 0xc3}, + "MOVWQZX": {0x48, 0x0f, 0xb7, 0xc3}, + "MOVLQSX": {0x48, 0x63, 0xc3}, + "MOVLQZX": {0x8b, 0xc3}, + "MOVSWW": {0x66, 0x0f, 0xbf, 0xc3}, + "MOVZWW": {0x66, 0x0f, 0xb7, 0xc3}, + } + var source strings.Builder + for _, value := range encodings[operation] { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func x86ExtensionNumericalSource() (string, string) { + var declarations, assembly strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + for _, test := range x86ExtensionNumericalCases() { + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint64") + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", test.name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+test.name+"(", 1) + declarations.WriteString(declaration) + operand := "BX" + body := "MOVQ (DI),BX\nMOVQ $0x5a6b7c8d9eaf1021,AX\n" + switch test.mode { + case "Memory": + operand = "(DI)" + case "LowAlias": + operand = "BL" + case "InPlace": + body += "MOVQ BX,AX\n" + operand = "AX" + case "PhysicalAlias": + body += "MOVQ BX,DI\n" + operand = "BH" + } + body += "CMPQ BX,$0\n" + if test.mode == "PhysicalBytes" { + body += x86ExtensionNumericalRawBytes(test.op) + } else { + body += fmt.Sprintf("%s %s,AX\n", test.op, operand) + } + body += "JE done\nADDQ $7,AX\ndone:\nMOVQ AX,(DX)" + machine := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+test.name) + machine = strings.Replace(machine, "MOVL $7,AX\nMOVL AX,(DX)", body, 1) + assembly.WriteString(machine) + } + return declarations.String(), assembly.String() +} + +func x86ExtensionActualGoNumerical(t *testing.T, declaration, assembly string, runner []string) { + t.Helper() + var tests strings.Builder + tests.WriteString(`package caller +import "testing" +func reference(input uint64, source, result uint, signed, inPlace bool) uint64 { + mask := uint64(1)< +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +` + declarations.String() + ` +static uint64_t reference(uint64_t input, unsigned source, unsigned result, bool sign, bool in_place) { + uint64_t mask = (UINT64_C(1)<", llvm.TargetMachine{}, options); err != nil { + t.Fatal(err) + } + return module.String() +} + +func TestCrossLinuxRuntimeMatrixX86FloatMulDivIntegerRoundingExtremes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + ir := x86MulDivRoundHelperFixtureIR() + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2-%t", optimized), func(t *testing.T) { + source := ir + if optimized { + source = x86MulDivRoundOptimize(t, source) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "integer-mul-div-round", triple, source, x86MulDivRoundSSEOracle, prefix) + }) + } +} + +func TestX86FloatMulDivIntegerRoundingOriginalSSEObjects(t *testing.T) { + for _, row := range []struct { + opcode string + prefix byte + code byte + }{ + {"MULSS", 0xf3, 0x59}, {"MULSD", 0xf2, 0x59}, + {"DIVSS", 0xf3, 0x5e}, {"DIVSD", 0xf2, 0x5e}, + } { + listing := x86InsertExtractGoListing(t, "amd64", "TEXT f(SB),4,$0-0\n"+row.opcode+" X1,X0\nRET\n") + var actual []byte + for _, line := range strings.Split(string(listing), "\n") { + actual = append(actual, x86InsertExtractListedBytes(line)...) + } + want := []byte{row.prefix, 0x0f, row.code, 0xc1, 0xc3} + if !bytes.Equal(actual, want) { + t.Fatalf("actual Go %s object %x, want independent native SSE oracle %x", row.opcode, actual, want) + } + } +} + +func TestX86FloatMulDivIntegerRoundingOnlyIntegerArithmeticAndBoundedDivision(t *testing.T) { + for _, optimized := range []bool{false, true} { + ir := x86MulDivRoundHelperFixtureIR() + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + for _, forbidden := range []string{ + `\b(fadd|fsub|fmul|fdiv|frem|fcmp|fptosi|fptoui|sitofp|uitofp)\b`, + `\b(udiv|sdiv|urem|srem)\b`, `@llvm\.(fma|experimental\.constrained)`, + } { + if regexp.MustCompile(forbidden).MatchString(ir) { + t.Fatalf("O2=%t: fixed-round SAE helper contains forbidden arithmetic %s", optimized, forbidden) + } + } + if !optimized { + for _, limit := range []string{"icmp eq i32 %next_iteration, 26", "icmp eq i32 %next_iteration, 55"} { + if !strings.Contains(ir, limit) { + t.Fatalf("missing exact restoring-division bound %s", limit) + } + } + } + } +} + +func TestX86FloatMulDivIntegerRoundingFiveLLVM22Targets(t *testing.T) { + llc, nm := findLLVM22Tool("llc"), findLLVM22Tool("llvm-nm") + if llc == "" || nm == "" { + t.Fatal("LLVM22 llc/llvm-nm required") + } + for _, optimized := range []bool{false, true} { + ir := x86MulDivRoundHelperFixtureIR() + backendLevel := "-O0" + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + backendLevel = "-O2" + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + "i386-unknown-linux-gnu", "i686-pc-windows-msvc", + } { + t.Run(fmt.Sprintf("O2-%t/%s", optimized, triple), func(t *testing.T) { + dir := t.TempDir() + source, object := filepath.Join(dir, "integer-round.ll"), filepath.Join(dir, "integer-round.o") + assembly := filepath.Join(dir, "integer-round.s") + if err := os.WriteFile(source, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + out, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, backendLevel, "-mtriple="+triple, "-filetype=obj", source, "-o", object)) + cancel() + if err != nil { + t.Fatalf("LLVM22 object: %v\n%s", err, out) + } + ctx, cancel = runtimeOracleStageContext(t) + out, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, nm, "--undefined-only", object)) + cancel() + if err != nil { + t.Fatalf("LLVM22 nm: %v\n%s", err, out) + } + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + // COFF's CRT floating-point marker has an extra C-name + // prefix in mode32. It is not an arithmetic libcall. + marker := "_fltused" + if strings.HasPrefix(triple, "i686-") { + marker = "__fltused" + } + if !strings.Contains(triple, "windows") || len(fields) != 2 || fields[0] != "U" || fields[1] != marker { + t.Fatalf("integer helper has unresolved arithmetic dependency: %s", line) + } + } + ctx, cancel = runtimeOracleStageContext(t) + out, err = runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, llc, backendLevel, "-mtriple="+triple, "-filetype=asm", source, "-o", assembly)) + cancel() + if err != nil { + t.Fatalf("LLVM22 machine assembly: %v\n%s", err, out) + } + machine, err := os.ReadFile(assembly) + if err != nil { + t.Fatal(err) + } + // Includes every helper and its actual call sites. An integer + // IR body alone does not close a float-return x87 C ABI. + floating := regexp.MustCompile(`(?m)^[\t ]+(f[a-z]+|v?(add|sub|mul|div|sqrt|comi|ucomi)(ss|sd|ps|pd))(\s|$)`) + if instruction := floating.Find(machine); instruction != nil { + t.Fatalf("SAE bit carrier emits floating instruction %q", instruction) + } + }) + } + } +} + +const x86MulDivRoundSSEOracle = `#include +#include +#include +extern uint32_t mul_div32(uint32_t,uint32_t,unsigned,unsigned,unsigned); +extern uint64_t mul_div64(uint64_t,uint64_t,unsigned,unsigned,unsigned); +static unsigned mismatches; +static uint64_t state=UINT64_C(0x372bd0ae71d94285); +static uint64_t random_bits(void){state^=state<<13;state^=state>>7;state^=state<<17;return state;} + +/* Independent physical SSE. Language arithmetic, production classification, + integer helpers and AVX512 are not used to compute expected result bits. */ +__attribute__((noinline)) static uint64_t native_sse(unsigned width,uint64_t a,uint64_t b,unsigned divide,unsigned csr){ + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)::"memory"); + __asm__ volatile("ldmxcsr %0"::"m"(csr):"memory"); + uint64_t result=0; + if(width==32){ + uint32_t aa=a,bb=b,out;float left,right;memcpy(&left,&aa,4);memcpy(&right,&bb,4); + if(divide)__asm__ volatile("divss %1,%0":"+x"(left):"x"(right)); + else __asm__ volatile("mulss %1,%0":"+x"(left):"x"(right)); + memcpy(&out,&left,4);result=out; + }else{ + double left,right;memcpy(&left,&a,8);memcpy(&right,&b,8); + if(divide)__asm__ volatile("divsd %1,%0":"+x"(left):"x"(right)); + else __asm__ volatile("mulsd %1,%0":"+x"(left):"x"(right)); + memcpy(&result,&left,8); + } + __asm__ volatile("ldmxcsr %0"::"m"(saved):"memory");return result; +} + +static int check(unsigned width,uint64_t a,uint64_t b,unsigned divide,unsigned mode,unsigned ambient,unsigned control){ + unsigned actualCSR=control|(ambient<<13),referenceCSR=control|(mode<<13),saved,after; + __asm__ volatile("stmxcsr %0":"=m"(saved)::"memory"); + __asm__ volatile("ldmxcsr %0"::"m"(actualCSR):"memory"); + uint64_t want=native_sse(width,a,b,divide,referenceCSR); + uint64_t got=width==32?mul_div32(a,b,divide,mode,actualCSR):mul_div64(a,b,divide,mode,actualCSR); + __asm__ volatile("stmxcsr %0":"=m"(after)::"memory"); + __asm__ volatile("ldmxcsr %0"::"m"(saved):"memory"); + if(got!=want||after!=actualCSR){ + if(mismatches++<8)fprintf(stderr,"integer MUL/DIV width/divide/mode/ambient/control=%u/%u/%u/%u/%x a=%llx b=%llx got=%llx want=%llx CSR=%x want=%x\n",width,divide,mode,ambient,control,(unsigned long long)a,(unsigned long long)b,(unsigned long long)got,(unsigned long long)want,after,actualCSR); + return 1; + } + return 0; +} + +static int matrix(unsigned width,uint64_t a,uint64_t b){ + const unsigned controls[]={0x1f80,0x1fc0,0x9f80,0x9fc0,0x1f85,0x9fe0}; + int failed=0; + for(unsigned divide=0;divide<2;divide++)for(unsigned mode=0;mode<4;mode++) + for(unsigned ambient=0;ambient<4;ambient++)for(unsigned c=0;c<6;c++) + failed|=check(width,a,b,divide,mode,ambient,controls[c]); + return failed; +} + +int main(void){ + int failed=0; + /* The first focused RED independently observes RU, not ambient RN. */ + failed|=check(32,0x3f800001,0x3f800001,0,2,0,0x1f80); + failed|=check(64,UINT64_C(0x3ff0000000000000),UINT64_C(0x4008000000000000),1,2,0,0x1f80); + /* Tiny before rounding, even though RU would produce minimum normal. */ + failed|=check(32,0x00800000,0x3f800001,1,2,0,0x9f80); + failed|=check(64,UINT64_C(0x0010000000000000),UINT64_C(0x3ff0000000000001),1,2,0,0x9f80); + const uint64_t edges32[]={0,0x80000000,1,0x80000001,0x7fffff,0x807fffff,0x800000,0x80800000, + 0x800001,0x7f7fffff,0xff7fffff,0x3f000000,0xbf000000,0x3f800000,0xbf800000,0x3f800001, + 0xbf800001,0x3f800002,0x40400000,0xc0400000,0x7f800000,0xff800000,0x7fc12345,0xffc45678, + 0x7f812345,0xff845678,0x00800000,0x40000000}; + const uint64_t edges64[]={0,UINT64_C(0x8000000000000000),1,UINT64_C(0x8000000000000001), + UINT64_C(0xfffffffffffff),UINT64_C(0x800fffffffffffff),UINT64_C(0x10000000000000),UINT64_C(0x8010000000000000), + UINT64_C(0x10000000000001),UINT64_C(0x7fefffffffffffff),UINT64_C(0xffefffffffffffff), + UINT64_C(0x3fe0000000000000),UINT64_C(0xbfe0000000000000),UINT64_C(0x3ff0000000000000),UINT64_C(0xbff0000000000000), + UINT64_C(0x3ff0000000000001),UINT64_C(0xbff0000000000001),UINT64_C(0x3ff0000000000002), + UINT64_C(0x4008000000000000),UINT64_C(0xc008000000000000),UINT64_C(0x7ff0000000000000),UINT64_C(0xfff0000000000000), + UINT64_C(0x7ff8123456789abc),UINT64_C(0xfff8fedcba987654),UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff0fedcba987654), + UINT64_C(0x10000000000000),UINT64_C(0x4000000000000000)}; + for(unsigned i=0;i= len(c.blocks) || instruction < 0 || instruction+1 >= len(c.blocks[block].instrs) { + return context("the body is missing or separated by a source/CFG boundary") + } + body := c.blocks[block].instrs[instruction+1] + op := Op(strings.ToUpper(string(body.Op))) + if form, recognized, err := x86LockScalarRMWProperties(op, body, c.goarch); recognized { + if err != nil { + return err + } + return x86LockScalarNativeMemoryBound(form, body, c.goarch) + } + if form, recognized, err := x86CompareExchangeProperties(op, body, c.goarch); recognized { + if err != nil { + return err + } + if form.destination.Kind != OpMem && form.destination.Kind != OpSym || body.x86UnboundAddressOverride { + return context("the native CMPXCHG memory operand is not bound") + } + return nil + } + spec, known := amd64ExchangeSpecs[op] + if !known { + // A different RMW opcode may not inherit another family's witness. + return context("the whole opcode effect is not independently closed") + } + form, err := parseAMD64ExchangeForm(c.goarch, op, spec, body) + if err != nil { + return context("the complete exchange operand grammar is not bound") + } + if form.other.Kind != OpMem && form.other.Kind != OpSym || body.x86UnboundAddressOverride { + return context("the native memory operand is not bound") + } + if c.goarch == "386" && spec.bits == 8 { + // Go synthesizes unavailable byte-register views through another GP + // register. LOCK must not bind the first instruction of that expansion + // as though it were the final byte memory RMW. + switch form.register.Reg { + case AX, BX, CX, DX, AL, AH, BL, BH, CL, CH, DL, DH: + default: + return context("the source byte-register view needs a Go encoder expansion") + } + } + // These existing memory lowerers emit a complete native XADD with LOCK or + // an implicitly locked XCHG, with read/write memory constraints, no guessed + // alignment and their unchanged byte/flags effects. Address, initialization, + // frame and caller permissions still come from the ordinary consumers. + return nil +} + +func (c *amd64Ctx) verifyNamedLockPrefixes() error { + for block, body := range c.blocks { + for instruction, prefix := range body.instrs { + if !strings.EqualFold(string(prefix.Op), "LOCK") { + continue + } + if err := c.namedLockPrefixBinding(block, instruction, prefix); err != nil { + return err + } + } + } + return c.verifyNamedLockCarry() +} diff --git a/amd64_named_lock_prefix_test.go b/amd64_named_lock_prefix_test.go new file mode 100644 index 000000000..9c4402e8d --- /dev/null +++ b/amd64_named_lock_prefix_test.go @@ -0,0 +1,142 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +type x86NamedLockOracleForm struct { + text string + atomic string +} + +// This independent inventory follows Intel's nineteen LOCK RMW opcode stems. +// A nonempty atomic field describes an existing lowerer's emitted operation, +// not a memory, caller or source initialization permission. +func x86NamedLockOracleForms(arch string) []x86NamedLockOracleForm { + widths := "BWL" + if arch == "amd64" { + widths += "Q" + } + var forms []x86NamedLockOracleForm + for _, stem := range []string{"ADD", "ADC", "AND", "OR", "SBB", "SUB", "XOR", "CMPXCHG", "XADD", "XCHG"} { + for _, width := range widths { + register := "CX" + if width == 'B' { + register = "CL" + } + atomic := "" + switch stem { + case "CMPXCHG": + atomic = "asm sideeffect \"lock; cmpxchg" + strings.ToLower(string(width)) + case "XADD": + atomic = "lock; xadd" + case "XCHG": + atomic = "asm sideeffect \"xchg" + default: + atomic = "lock; " + strings.ToLower(stem+string(width)) + } + forms = append(forms, x86NamedLockOracleForm{fmt.Sprintf("%s%c %s,(SI)", stem, width, register), atomic}) + } + } + for _, stem := range []string{"INC", "DEC", "NEG", "NOT"} { + for _, width := range widths { + forms = append(forms, x86NamedLockOracleForm{ + text: fmt.Sprintf("%s%c (SI)", stem, width), atomic: "lock; " + strings.ToLower(stem+string(width)), + }) + } + } + for _, stem := range []string{"BTC", "BTR", "BTS"} { + for _, width := range strings.TrimPrefix(widths, "B") { + forms = append(forms, x86NamedLockOracleForm{ + text: fmt.Sprintf("%s%c CX,(SI)", stem, width), atomic: "lock; " + strings.ToLower(stem+string(width)), + }) + } + } + forms = append(forms, x86NamedLockOracleForm{text: "CMPXCHG8B (SI)", atomic: "asm sideeffect \"lock; cmpxchg8b"}) + if arch == "amd64" { + forms = append(forms, x86NamedLockOracleForm{text: "CMPXCHG16B (SI)", atomic: "asm sideeffect \"lock; cmpxchg16b"}) + } + return forms +} + +func x86NamedLockOracleSource(arch, body string) string { + load, args := "MOVL", 4 + if arch == "amd64" { + load, args = "MOVQ", 8 + } + return fmt.Sprintf(`TEXT ·locked(SB),4,$0-%d + %s p+0(FP),SI + MOVL $2,AX + MOVL $3,BX + MOVL $4,CX + MOVL $5,DX + %s + RET +`, args, load, body) +} + +func TestX86NamedLockCompleteRMWRequiresWholeAtomicBinding(t *testing.T) { + for _, target := range x86OrderingHintTargets { + forms := x86NamedLockOracleForms(target.arch) + want := 49 + if target.arch == "amd64" { + want = 67 + } + if len(forms) != want { + t.Fatal("complete LOCK stem/width denominator changed", target.arch, len(forms)) + } + for _, form := range forms { + t.Run(target.triple+"/"+form.text, func(t *testing.T) { + source := x86NamedLockOracleSource(target.arch, "CMPL AX,AX\n\tLOCK\n\t"+form.text) + actual := x86OrderingHintGoBytes(t, target.arch, source) + if !strings.Contains(string(actual), string([]byte{0xf0})) { + t.Fatalf("actual Go object lost named LOCK prefix: %x", actual) + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\nfunc locked(p *[16]byte)\n", target.arch, target.triple) + if form.atomic == "" { + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("actual Go LOCK RMW silently became a non-atomic operation: error=%v atomicrmw=%v cmpxchg=%v nativeLock=%v", err, + strings.Contains(ir, "atomicrmw "), strings.Contains(ir, " = cmpxchg "), strings.Contains(ir, "lock;")) + } + return + } + if err != nil || !strings.Contains(ir, form.atomic) { + t.Fatalf("existing whole atomic operation changed: %v, required emission=%q", err, form.atomic) + } + }) + } + }) + } + } +} + +func TestX86NamedLockDetachedAndSplitPrefixDoesNotBorrowAtomicBody(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, body := range []string{ + "LOCK", + "LOCK\n\tLOCK\n\tXADDL CX,(SI)", + "LOCK\nbody:\n\tXADDL CX,(SI)", + "LOCK\n\tCMPL CX,(SI)", + "LOCK\n\tNOP\n\tXADDL CX,(SI)", + } { + t.Run(arch+"/"+body, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, "text", x86NamedLockOracleSource(arch, body), + "package ordering\nfunc locked(p *[16]byte)\n", arch, "") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatal("unbound or split prefix borrowed a following operation", err) + } + }) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + known, terminated, err := ctx.lowerAtomic("LOCK", Instr{Op: "LOCK", Raw: "LOCK"}) + if !known || terminated || !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatal("standalone helper has no complete ordered instruction to bind", known, terminated, err) + } +} diff --git a/amd64_named_stack_expression_test.go b/amd64_named_stack_expression_test.go new file mode 100644 index 000000000..67101969c --- /dev/null +++ b/amd64_named_stack_expression_test.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestAMD64NamedStackConstantExpressions(t *testing.T) { + const source = `TEXT namedStackExpressions(SB),$64-0 + MOVL AX, tmpdig-(0*4)(SP) + MOVL BX, tmpdig-(1*4)(SP) + ADDL tmpdig-(0*4)(SP), AX + ADDL tmpdig-(1*4)(SP), BX + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if got := file.Funcs[0].Instrs[1].Args[1]; got.Kind != OpMem || got.Mem.Off != 0 { + t.Fatalf("first named stack slot parsed as %+v, want offset 0 memory", got) + } + if got := file.Funcs[0].Instrs[2].Args[1]; got.Kind != OpMem || got.Mem.Off != -4 { + t.Fatalf("second named stack slot parsed as %+v, want offset -4 memory", got) + } + ll, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedStackExpressions": {Name: "namedStackExpressions", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "add i32") { + t.Fatalf("named stack loads omitted from IR:\n%s", ll) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "named-stack-expressions.ll", "named-stack-expressions.o", ll) +} + +func TestNamedStackConstantOffsetRejectsUnresolvedExpressions(t *testing.T) { + for _, tc := range []struct { + prefix string + want int64 + ok bool + }{ + {prefix: "tmpdig-(7*4)", want: -28, ok: true}, + {prefix: "tmpdig+(2<<3)", want: 16, ok: true}, + {prefix: "tmpdig-(unknown*4)"}, + {prefix: "tmpdig-(4/0)"}, + {prefix: "tmp-dig-(1*4)"}, + } { + got, ok := parseNamedStackConstantOffset(tc.prefix) + if ok != tc.ok || ok && got != tc.want { + t.Errorf("parseNamedStackConstantOffset(%q) = %d, %v; want %d, %v", tc.prefix, got, ok, tc.want, tc.ok) + } + } +} diff --git a/amd64_needed.go b/amd64_needed.go index 9a52b4df8..c97a638ab 100644 --- a/amd64_needed.go +++ b/amd64_needed.go @@ -2,18 +2,30 @@ package plan9asm import "strings" -// funcNeedsAMD64CFG decides whether we need the CFG-based amd64 translator. -// The linear prototype cannot handle labels/branches and most vector ops. +// funcNeedsAMD64CFG decides whether the direct LLVM-module prototype can +// represent an amd64 function. Textual translation always uses the complete +// architecture-aware x86 CFG lowerer. func funcNeedsAMD64CFG(fn Func) bool { for _, ins := range fn.Instrs { + if _, known, _ := x86GPSIMDTransferProperties(ins, "amd64"); known { + return true // The scalar prototype has no X-bank or encoding state. + } if ins.Op == OpLABEL { return true } op := strings.ToUpper(string(ins.Op)) + for _, operand := range ins.Args { + if _, _, ok := x86MachineRegisterOperand(operand); ok { + return true + } + if x86SegmentRegisterOperand(operand) { + return true + } + } switch Op(op) { - case "JMP", "JL", "JLT", "JLE", "JG", "JGT", "JGE", + case "JMP", "JCXZW", "JCXZL", "JCXZQ", "JL", "JLT", "JLE", "JG", "JGT", "JGE", "JB", "JLO", "JBE", "JA", "JHI", "JAE", "JHS", - "JZ", "JE", "JEQ", "JNZ", "JNE", "JNC", "JC", "JCC", "JLS", "JNA", "JS", "JNS": + "JZ", "JE", "JEQ", "JNZ", "JNE", "JNC", "JC", "JCS", "JCC", "JLS", "JNA", "JS", "JNS": return true } // A handful of amd64 stdlib asm functions are straight-line, but if we @@ -21,10 +33,10 @@ func funcNeedsAMD64CFG(fn Func) bool { if strings.HasPrefix(op, "MOVO") || strings.HasPrefix(op, "PCLMUL") || strings.HasPrefix(op, "CRC32") || strings.HasPrefix(op, "PXOR") { return true } - // Keep the linear path only for the tiny subset it currently lowers. + // Keep the direct-module path only for the tiny subset it currently lowers. switch Op(op) { case OpTEXT, OpBYTE, OpMOVQ, OpMOVL, OpADDQ, OpSUBQ, OpXORQ, OpCPUID, OpXGETBV: - // For MOVQ/MOVL, linear lowering supports immediate/reg/FP value flow. + // For MOVQ/MOVL, direct lowering supports immediate/reg/FP value flow. // Addressing forms (mem/sym) require CFG lowering. if (op == "MOVQ" || op == "MOVL") && len(ins.Args) == 2 { for _, a := range ins.Args { diff --git a/amd64_new_opfamilies_test.go b/amd64_new_opfamilies_test.go index 096e191de..5b0480a20 100644 --- a/amd64_new_opfamilies_test.go +++ b/amd64_new_opfamilies_test.go @@ -1,6 +1,3 @@ -//go:build !llgo -// +build !llgo - package plan9asm import ( diff --git a/amd64_operand_forms_test.go b/amd64_operand_forms_test.go index dde7de5cc..95d71a172 100644 --- a/amd64_operand_forms_test.go +++ b/amd64_operand_forms_test.go @@ -1,6 +1,3 @@ -//go:build !llgo -// +build !llgo - package plan9asm import "testing" @@ -99,22 +96,19 @@ func TestAMD64VectorLoweringBranchCoverage(t *testing.T) { } } - check("gf2-z-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Z0"), reg("X1"), reg("Z2")}}, false, false) + check("gf2-z-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Z0"), reg("X1"), reg("Z2")}}, true, true) check("gf2-z-source-error", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Z0"), ident("bad"), reg("Z2")}}, true, true) - check("gf2-y-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("X1"), reg("Y2")}}, false, false) + check("gf2-y-source-class", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("X1"), reg("Y2")}}, true, true) check("gf2-y-source-error", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), ident("bad"), reg("Y2")}}, true, true) - check("gf2-unsupported-dst", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("Y1"), reg("X2")}}, false, false) + check("gf2-unsupported-dst", "VGF2P8AFFINEQB", Instr{Args: []Operand{imm(0), reg("Y0"), reg("Y1"), reg("X2")}}, true, true) - check("broadcast-i128-args", "VBROADCASTI128", Instr{Args: []Operand{reg("X0")}}, true, true) - check("broadcast-i128-dst", "VBROADCASTI128", Instr{Args: []Operand{reg("X0"), reg("X1")}}, false, false) - check("broadcast-i128-source", "VBROADCASTI128", Instr{Args: []Operand{ident("bad"), reg("Y1")}}, true, true) check("broadcast-scalar-args", "VBROADCASTSD", Instr{Args: []Operand{imm(1)}}, true, true) check("broadcast-scalar-source", "VBROADCASTSD", Instr{Args: []Operand{ident("bad"), reg("Y1")}}, true, true) - check("broadcast-scalar-immediate", "VBROADCASTSD", Instr{Args: []Operand{imm(1), reg("Y1")}}, false, false) - check("broadcast-scalar-gpr", "VBROADCASTSD", Instr{Args: []Operand{reg("AX"), reg("Y1")}}, false, false) + check("broadcast-scalar-immediate", "VBROADCASTSD", Instr{Args: []Operand{imm(1), reg("Y1")}}, true, true) + check("broadcast-scalar-gpr", "VBROADCASTSD", Instr{Args: []Operand{reg("AX"), reg("Y1")}}, true, true) check("broadcast-scalar-x-source", "VBROADCASTSD", Instr{Args: []Operand{reg("X0"), reg("Y1")}}, false, true) - check("broadcast-scalar-x-dst", "VBROADCASTSD", Instr{Args: []Operand{reg("X0"), reg("X1")}}, false, false) - check("broadcast-scalar-dst", "VBROADCASTSD", Instr{Args: []Operand{mem("AX"), reg("AX")}}, false, false) + check("broadcast-scalar-x-dst", "VBROADCASTSD", Instr{Args: []Operand{reg("X0"), reg("X1")}}, true, true) + check("broadcast-scalar-dst", "VBROADCASTSD", Instr{Args: []Operand{mem("AX"), reg("AX")}}, true, true) check("broadcast-q-x", "VPBROADCASTQ", Instr{Args: []Operand{reg("X0"), reg("X1")}}, false, true) check("broadcast-q-gpr", "VPBROADCASTQ", Instr{Args: []Operand{reg("AX"), reg("X1")}}, false, true) check("broadcast-q-immediate", "VPBROADCASTQ", Instr{Args: []Operand{imm(1), reg("X1")}}, false, false) @@ -129,6 +123,18 @@ func TestAMD64VectorLoweringBranchCoverage(t *testing.T) { check("punpckllq-src", "PUNPCKLLQ", Instr{Args: []Operand{reg("Y0"), reg("X1")}}, false, false) check("punpckhlq-source-error", "PUNPCKHLQ", Instr{Args: []Operand{ident("bad"), reg("X1")}}, true, true) check("punpckllq-memory", "PUNPCKLLQ", Instr{Args: []Operand{mem("AX"), reg("X1")}}, false, true) + for _, op := range []Op{"PMINUB", "PMINSB", "PMINUW", "PMINSW", "PMINUD", "PMINSD", "PMAXUB", "PMAXSB", "PMAXUW", "PMAXSW", "PMAXUD", "PMAXSD"} { + check(string(op)+"-args", op, Instr{Args: []Operand{reg("X0")}}, true, true) + check(string(op)+"-dst", op, Instr{Args: []Operand{reg("X0"), reg("Y1")}}, false, false) + check(string(op)+"-source", op, Instr{Args: []Operand{ident("bad"), reg("X1")}}, true, true) + check(string(op)+"-memory", op, Instr{Args: []Operand{mem("AX"), reg("X1")}}, false, true) + } + for _, op := range []Op{"PSLLQ", "PSRLQ"} { + check(string(op)+"-args", op, Instr{Args: []Operand{imm(1)}}, true, true) + check(string(op)+"-dst", op, Instr{Args: []Operand{imm(1), reg("Y1")}}, false, false) + check(string(op)+"-count", op, Instr{Args: []Operand{reg("AX"), reg("X1")}}, true, true) + check(string(op)+"-negative", op, Instr{Args: []Operand{imm(-1), reg("X1")}}, true, true) + } check("xorpd-args", "VXORPD", Instr{Args: []Operand{reg("X0")}}, true, true) check("xorpd-z", "VXORPD", Instr{Args: []Operand{reg("Z0"), reg("Z1"), reg("Z2")}}, false, true) @@ -140,11 +146,11 @@ func TestAMD64VectorLoweringBranchCoverage(t *testing.T) { check("xorpd-x", "VXORPD", Instr{Args: []Operand{reg("X0"), reg("X1"), reg("X2")}}, false, true) check("xorpd-x-source1", "VXORPD", Instr{Args: []Operand{ident("bad"), reg("X1"), reg("X2")}}, true, true) check("xorpd-x-source2", "VXORPD", Instr{Args: []Operand{reg("X0"), ident("bad"), reg("X2")}}, true, true) - check("xorpd-unsupported-dst", "VXORPD", Instr{Args: []Operand{reg("X0"), reg("X1"), reg("AX")}}, false, false) + check("xorpd-unsupported-dst", "VXORPD", Instr{Args: []Operand{reg("X0"), reg("X1"), reg("AX")}}, true, true) - check("ternlog-y-source1-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("X0"), reg("Y1"), reg("Y2")}}, false, false) - check("ternlog-y-source2-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("Y0"), reg("X1"), reg("Y2")}}, false, false) - check("ternlog-y-imm", "VPTERNLOGD", Instr{Args: []Operand{imm(0x95), reg("Y0"), reg("Y1"), reg("Y2")}}, true, true) + check("ternlog-y-source1-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("X0"), reg("Y1"), reg("Y2")}}, true, true) + check("ternlog-y-source2-class", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("Y0"), reg("X1"), reg("Y2")}}, true, true) + check("ternlog-y-imm", "VPTERNLOGD", Instr{Args: []Operand{imm(0x95), reg("Y0"), reg("Y1"), reg("Y2")}}, false, true) check("ternlog-y-source1", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), ident("bad"), reg("Y1"), reg("Y2")}}, true, true) check("ternlog-y-source2", "VPTERNLOGD", Instr{Args: []Operand{imm(0x96), reg("Y0"), ident("bad"), reg("Y2")}}, true, true) diff --git a/amd64_ordering_hint_family_test.go b/amd64_ordering_hint_family_test.go new file mode 100644 index 000000000..fbec94b34 --- /dev/null +++ b/amd64_ordering_hint_family_test.go @@ -0,0 +1,104 @@ +package plan9asm + +import ( + "bytes" + "reflect" + "strings" + "testing" +) + +// Independent complete Go ynone rows, not an implementation-spec projection. +var x86OrderingHintOracleRows = []struct { + op Op + registration string + machine []byte +}{ + {"LFENCE", "{ALFENCE, ynone, Pm, opBytes{0xae, 0xe8}}", []byte{0x0f, 0xae, 0xe8}}, + {"MFENCE", "{AMFENCE, ynone, Pm, opBytes{0xae, 0xf0}}", []byte{0x0f, 0xae, 0xf0}}, + {"SFENCE", "{ASFENCE, ynone, Pm, opBytes{0xae, 0xf8}}", []byte{0x0f, 0xae, 0xf8}}, + {"PAUSE", "{APAUSE, ynone, Px, opBytes{0xf3, 0x90}}", []byte{0xf3, 0x90}}, +} + +func TestX86OrderingHintsHaveExactEmptySourceEffects(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86OrderingHintOracleRows { + t.Run(arch+"/"+string(row.op), func(t *testing.T) { + ins := Instr{Op: row.op} + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 0 || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatalf("zero-operand instruction lacks exact no-output/no-flags source effect: %+v/%t/%v", effect, known, err) + } + move, suffix := Op("MOVQ"), "Q" + if arch == "386" { + move, suffix = "MOVL", "L" + } + if !x86SourceStackInstructionKnown(ins, move, suffix, arch) { + t.Fatal("valid typed instruction has an unregistered source-stack effect") + } + result := x86ResultState{flags: x86ByteFlagsAll} + before := result + if !x86ResultTransfer(ins, &result, nil, 0, arch) || !reflect.DeepEqual(before, result) { + t.Fatal("exact no-output instruction changed result initialization/flags") + } + caller := x86SourceStackState{numericFlags: true, known: map[Reg]uint8{AX: 0xff}} + flags, valid := x86CallerScalarStep(&ins, &caller, nil) + if !valid || flags || !caller.numericFlags || caller.known[AX] != 0xff { + t.Fatal("exact no-output instruction changed caller facts") + } + }) + } + } +} + +func TestX86OrderingHintsActualInstructionsNotNoops(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86OrderingHintOracleRows { + t.Run(arch+"/"+string(row.op), func(t *testing.T) { + source := "TEXT fence(SB),4,$0-0\n" + string(row.op) + "\nRET\n" + actual := x86OrderingHintGoBytes(t, arch, source) + want := append(append([]byte(nil), row.machine...), 0xc3) + if !bytes.Equal(actual, want) { + t.Fatalf("actual Go encoding %x, want %x", actual, want) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + start := output.Len() + known, terminated, err := ctx.lowerArith(row.op, file.Funcs[0].Instrs[1]) + if !known || terminated || err != nil { + t.Fatal("accepted complete zero-operand form", err) + } + if !strings.Contains(output.String()[start:], "asm sideeffect \""+strings.ToLower(string(row.op))+"\"") { + t.Fatal("actual ordering/hint instruction became a software no-op") + } + }) + } + } +} + +func TestX86OrderingHintsIllegalOperandsRejectBeforeEmission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range x86OrderingHintOracleRows { + for _, argument := range []string{"AX", "(AX)", "$1", "AX,DX"} { + t.Run(arch+"/"+string(row.op)+"/"+argument, func(t *testing.T) { + source := "TEXT fence(SB),4,$0-0\n" + string(row.op) + " " + argument + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := output.String(), ctx.tmp + known, terminated, err := ctx.lowerArith(row.op, file.Funcs[0].Instrs[1]) + if !known || terminated || err == nil || before != output.String() || temporary != ctx.tmp { + t.Fatalf("illegal operand did not fail before emission: %t/%t/%v", known, terminated, err) + } + }) + } + } + } +} diff --git a/amd64_ordering_hint_forms.go b/amd64_ordering_hint_forms.go new file mode 100644 index 000000000..1284bdb0e --- /dev/null +++ b/amd64_ordering_hint_forms.go @@ -0,0 +1,44 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86OrderingHintSpec struct { + native string +} + +// The complete Go ynone axis has no explicit or implicit data operands. +// Hardware ordering/hint execution is not a source-memory permission. +var x86OrderingHintSpecs = map[Op]x86OrderingHintSpec{ + "LFENCE": {native: "lfence"}, + "MFENCE": {native: "mfence"}, + "SFENCE": {native: "sfence"}, + "PAUSE": {native: "pause"}, +} + +func x86OrderingHintProperties(op Op, ins Instr, arch string) (x86OrderingHintSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86OrderingHintSpecs[Op(name)] + if !known { + return x86OrderingHintSpec{}, false, nil + } + if dotted || arch != "386" && arch != "amd64" || len(ins.Args) != 0 || + ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return spec, true, fmt.Errorf("%s %s requires its exact zero-operand Go ordering/hint form: %q", arch, op, ins.Raw) + } + return spec, true, nil +} + +func (c *amd64Ctx) lowerX86OrderingHint(op Op, ins Instr) (bool, bool, error) { + spec, known, err := x86OrderingHintProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + // Conservative compiler memory scheduling is separate from architectural + // semantics: these instructions have no source data or arithmetic outputs. + // A litmus sample is not a proof of the hardware memory model. + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q()\n", spec.native, "~{memory}") + return true, false, nil +} diff --git a/amd64_ordering_hint_linux_amd64_test.go b/amd64_ordering_hint_linux_amd64_test.go new file mode 100644 index 000000000..0b92421ce --- /dev/null +++ b/amd64_ordering_hint_linux_amd64_test.go @@ -0,0 +1,80 @@ +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Required Linux/amd64 CI runs the actual static ELF32 objects directly. The +// host/ELF checks reject emulation; no local Rosetta run establishes this axis. +func TestCrossLinuxRuntimeMatrixX86OrderingHintsNative386(t *testing.T) { + declarations, assembly, driver, reference := x86OrderingHintRuntimeFixture(32) + x86OrderingHintNative386(t, declarations, assembly, driver, reference) +} + +func TestCrossLinuxRuntimeMatrixX86RawNOPNative386(t *testing.T) { + declarations, assembly, driver, reference := x86RawNOPRuntimeFixture(32) + x86OrderingHintNative386(t, declarations, assembly, driver, reference) +} + +func TestCrossLinuxRuntimeMatrixX86RawNOPOriginalAMD64(t *testing.T) { + x87IA32RequireNativeHost(t) + declarations, assembly, driver, _ := x86RawNOPRuntimeFixture(64) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "amd64", declarations, assembly, driver) + if output := x86FloatCompareCaptureCommand(t, exec.Command(original)); string(output) != "original ordering observations 256\n" { + t.Fatalf("actual original Go AMD64 NOP state/no-memory oracle: %q", output) + } + x87IA32RequireNativeHost(t) +} + +func x86OrderingHintNative386(t *testing.T, declarations, assembly, driver, reference string) { + t.Helper() + x87IA32RequireNativeHost(t) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if err != nil { + t.Fatal("required native IA32 static C compiler:", err) + } + dir := t.TempDir() + original := x86OrderingHintBuildOriginal(t, dir, "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != "original ordering observations 256\n" { + t.Fatalf("unchanged original Go IA32 ordering/status/canary oracle: %q", output) + } + ir, err := x86OrderingHintTranslateAPI(t, "Go", assembly, declarations, "386", "i386-unknown-linux-gnu") + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := filepath.Join(dir, "ordering"+level+".ll") + object := filepath.Join(dir, "ordering"+level+".o") + main, executable := filepath.Join(dir, "main.c"), filepath.Join(dir, "ordering"+level) + for name, contents := range map[string]string{input: ir, main: reference} { + if err := os.WriteFile(name, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, + "-mtriple=i386-unknown-linux-gnu", "-filetype=obj", input, "-o", object)) + x86FloatCompareCaptureCommand(t, exec.Command(compiler, "-static", "-no-pie", + object, main, "-O2", "-o", executable)) + sealed, err := x87IA32StaticExecutable(executable) + if err != nil { + t.Fatal(err) + } + if output := x87IA32RunDirect(t, sealed, nil, os.Environ()); string(output) != "ordering observations 256\n" { + t.Fatalf("LLVM IA32 GP/six-status-flags/MXCSR/canary oracle: %q", output) + } + }) + } + x87IA32RequireNativeHost(t) +} diff --git a/amd64_ordering_hint_rule_consumers_test.go b/amd64_ordering_hint_rule_consumers_test.go new file mode 100644 index 000000000..f632655bb --- /dev/null +++ b/amd64_ordering_hint_rule_consumers_test.go @@ -0,0 +1,251 @@ +package plan9asm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +var x86OrderingHintTargets = []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, +} + +func x86OrderingHintGoBytes(t *testing.T, arch, source string) []byte { + t.Helper() + dir := t.TempDir() + input, object := filepath.Join(dir, "original.s"), filepath.Join(dir, "original.o") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, time.Minute) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-S", "-p", "ordering", "-o", object, input) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH="+arch, "GOFLAGS=") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, arch); failure != nil { + t.Fatalf("actual original Go object: %v\n%s", failure, output) + } + var code []byte + for _, line := range strings.Split(string(output), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, err := strconv.ParseUint(fields[0][2:], 16, 32) + if err != nil || int(offset) != len(code) { + t.Fatalf("noncontiguous original object listing: %s", line) + } + for index, field := range fields[1:] { + if index == 16 || len(field) != 2 { + break + } + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + break + } + code = append(code, byte(value)) + } + } + if len(code) == 0 { + t.Fatal("actual original object listing lacks complete bytes") + } + return code +} + +func TestX86OrderingHintsCompleteGoRowsAndDispatch(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(table), "var ynone = []ytab{\n\t{Zlit, 1, argList{}},\n}") || len(x86OrderingHintSpecs) != 4 { + t.Fatal("complete Go zero-operand family changed") + } + for _, row := range x86OrderingHintOracleRows { + if !strings.Contains(string(table), row.registration) { + t.Fatal("complete Go ordered row changed", row.registration) + } + for _, arch := range []string{"386", "amd64"} { + for _, identity := range []Op{"", row.op, Op(strings.ToLower(string(row.op))), "MOVQ", Op(string(row.op) + ".BAD")} { + t.Run(arch+"/"+string(row.op)+"/identity="+string(identity), func(t *testing.T) { + ins := Instr{Op: identity} + ctx, out := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before, temporary := out.String(), ctx.tmp + registers := fmt.Sprint(ctx.usedRegs) + known, terminated, err := ctx.lowerX86OrderingHint(row.op, ins) + invalid := identity != "" && !strings.EqualFold(string(identity), string(row.op)) + if !known || terminated || (err != nil) != invalid || invalid && (before != out.String() || temporary != ctx.tmp) { + t.Fatal("identity rejection/emission boundary", identity, err) + } + if fmt.Sprint(ctx.usedRegs) != registers { + t.Fatal("zero-operand source introduced implicit data registers", ctx.usedRegs) + } + }) + } + for _, suffix := range []string{".Z", ".SAE", ".BCST", ".BAD"} { + ins := Instr{Op: Op(string(row.op) + suffix)} + if _, known, err := x86OrderingHintProperties(ins.Op, ins, arch); !known || err == nil { + t.Fatal("invalid suffix was not closed", ins, err) + } + } + } + for _, arch := range []string{"", "arm", "arm64", "wasm"} { + if _, known, err := x86OrderingHintProperties(row.op, Instr{Op: row.op}, arch); !known || err == nil { + t.Fatal("unrelated architecture admitted an x86 form", arch, err) + } + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + before := output.String() + if known, terminated, err := ctx.lowerX86OrderingHint("UNKNOWN", Instr{}); known || terminated || err != nil || before != output.String() { + t.Fatal("unknown dispatch emitted a default instruction") + } +} + +func x86OrderingHintTranslateAPI(t *testing.T, api, source, declarations, arch, triple string) (string, error) { + t.Helper() + pkg := mustGoPackage(t, "ordering", declarations) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "Go" { + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: arch, TargetTriple: triple, ResolveSym: resolve}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + return "", err + } + return translated.Module.String(), nil + } + file, err := Parse(ArchAMD64, source) + if err != nil { + return "", err + } + sigs, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + return "", err + } + options := Options{Goarch: arch, TargetTriple: triple, ResolveSym: resolve, Sigs: sigs} + if api == "text" { + return Translate(file, options) + } + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + return "", err + } + return module.String(), nil +} + +func TestX86OrderingHintsThreeAPIsFiveTargetNativeObjects(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 llc and llvm-objdump required") + } + for _, target := range x86OrderingHintTargets { + for _, raw := range []bool{false, true} { + var source, declarations strings.Builder + declarations.WriteString("package ordering\n") + for _, row := range x86OrderingHintOracleRows { + name := strings.ToLower(string(row.op)) + fmt.Fprintf(&declarations, "func %s()\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-0\n", name) + if raw { + for _, value := range row.machine { + fmt.Fprintf(&source, "BYTE $%d\n", value) + } + } else { + fmt.Fprintln(&source, row.op) + } + source.WriteString("RET\n") + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(fmt.Sprintf("%s/raw=%t/%s", target.triple, raw, api), func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "ordering.ll"), filepath.Join(dir, "ordering.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", "-o", object, input)) + disassembly := x86FloatCompareCaptureCommand(t, exec.Command(disassembler, "--disassemble", "--no-show-raw-insn", object)) + for _, row := range x86OrderingHintOracleRows { + count := 0 + for _, line := range strings.Split(string(disassembly), "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && strings.HasSuffix(fields[0], ":") && fields[1] == strings.ToLower(string(row.op)) { + count++ + } + } + if count != 1 { + t.Fatalf("%s object lost or duplicated native %s: count=%d\n%s", level, row.op, count, disassembly) + } + } + } + }) + } + } + } +} + +func TestX86RawPrefixThreeAPIClassification(t *testing.T) { + for _, target := range x86OrderingHintTargets { + for _, row := range []struct { + name string + code []byte + context bool + }{ + {"invalid even with later ignored override", []byte{0xf0, 0x66, 0x0f, 0x31}, false}, + {"valid LOCK atomic effect unbound", []byte{0xf0, 0x01, 0x08}, true}, + {"legal ignored prefix", []byte{0x66, 0x66, 0x0f, 0x31}, false}, + } { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+row.name+"/"+api, func(t *testing.T) { + source := "TEXT ·prefix(SB),4,$0-0\n" + x86RawDisplacementDirectives(row.code) + "RET\n" + ir, err := x86OrderingHintTranslateAPI(t, api, source, "package ordering\nfunc prefix()\n", target.arch, target.triple) + if row.name == "legal ignored prefix" { + if err != nil || !strings.Contains(ir, "asm sideeffect \"rdtsc\"") { + t.Fatal("legal ignored prefix lost its actual instruction", err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), target.triple, "ignored.ll", "ignored.o", ir) + } else if err == nil || ir != "" || errors.Is(err, ErrProbeNeedsContext) != row.context { + t.Fatalf("invalid versus valid-unbound prefix classification: IR=%q/error=%v", ir, err) + } + }) + } + } + } + physical, err := x86asm.Decode([]byte{0xf0, 0x0f, 0x31}, 64) + if err != nil { + t.Fatal(err) + } + physical.Prefix[0] |= x86asm.PrefixIgnored + if err := decodedX86PrefixError(physical); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("synthetic Invalid|Ignored metadata tripwire washed invalidity", err) + } +} diff --git a/amd64_ordering_hint_runtime_test.go b/amd64_ordering_hint_runtime_test.go new file mode 100644 index 000000000..3e437284f --- /dev/null +++ b/amd64_ordering_hint_runtime_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "debug/pe" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86OrderingHintsOriginalWindowsExecutable(t *testing.T) { + declarations, assembly, driver, _ := x86OrderingHintRuntimeFixture(64) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "windows", "amd64", declarations, assembly, driver) + if filepath.Ext(original) != ".exe" { + t.Fatal("Windows original oracle requires its exact executable suffix", original) + } + object, err := pe.Open(original) + if err != nil { + t.Fatal("actual Windows oracle executable", err) + } + defer object.Close() + if object.Machine != pe.IMAGE_FILE_MACHINE_AMD64 || object.Characteristics&pe.IMAGE_FILE_EXECUTABLE_IMAGE == 0 { + t.Fatal("original Windows oracle is not an AMD64 executable") + } +} + +// These observations establish exact GP, six status bits and MXCSR retention, +// not an empirical proof of the architecture's concurrent memory model. +func TestCrossLinuxRuntimeMatrixX86OrderingHintsOriginalAndLLVM(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly, driver, reference := x86OrderingHintRuntimeFixture(64) + dir := t.TempDir() + original := x86OrderingHintBuildOriginal(t, dir, runtime.GOOS, "amd64", declarations, assembly, driver) + if output := execution.run(t, original); string(output) != "original ordering observations 256\n" { + t.Fatalf("actual original Go ordering/status/canary oracle: %q", output) + } + t.Log("256 original Go observations: four instructions, two complementary six-bit seeds, 32 repetitions") + ir := x86NarrowOracleIR(t, assembly, declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "ordering", ir, reference, level) + if string(output) != "ordering observations 256\n" { + t.Fatalf("independent GP/full-status/MXCSR/canary oracle: %q", output) + } + t.Log("256 independent LLVM observations: five GPs, six status flags, MXCSR and canary") + }) + } +} + +func x86OrderingHintBuildOriginal(t *testing.T, dir, goos, arch, declarations, assembly, driver string) string { + t.Helper() + for name, contents := range map[string]string{ + "go.mod": "module ordering\n\ngo 1.20\n", + "decl.go": declarations, "ordering_" + arch + ".s": assembly, + "main.go": driver, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-go") + if goos == "windows" { + executable += ".exe" + } + command := exec.Command(filepath.Join(runtime.GOROOT(), "bin/go"), "build", "-p=2", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+arch, + "CGO_ENABLED=0", "GOTOOLCHAIN=local", "GOENV=off", "GOWORK=off", "GOFLAGS=", "GOEXPERIMENT=") + x86FloatCompareCaptureCommand(t, command) + return executable +} + +func TestX86OrderingHintsOriginal386StaticELFCompileOnly(t *testing.T) { + declarations, assembly, driver, _ := x86OrderingHintRuntimeFixture(32) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go IA32 ELF bytes=%d SHA256=%x; compile only, required native execution is separate", + sealed.size, sealed.hash) +} + +// The independent reference constants never come from the production spec. +// Each sample resets every output byte and both complementary status seeds. +func x86OrderingHintRuntimeFixture(bits int) (declarations, assembly, driver, reference string) { + word, suffix, pop, push := "uint64", "Q", "POPFQ", "PUSHFQ" + values := []string{"0x123456789abcdef0", "0x2468ace013579bdf", "0x3141592653589793", "0x7e6d5c4b3a291807", "0xa1b2c3d4e5f60718"} + canary := "0x13579bdf2468ace0" + if bits == 32 { + word, suffix, pop, push = "uint32", "L", "POPFL", "PUSHFL" + values = []string{"0x9abcdef0", "0x13579bdf", "0x53589793", "0x3a291807", "0xe5f60718"} + canary = "0x2468ace0" + } + var decl, asm, names strings.Builder + decl.WriteString("package main\n") + step := bits / 8 + for _, op := range []string{"LFENCE", "MFENCE", "SFENCE", "PAUSE"} { + for index, seed := range []uint64{0x8c7, 0x12} { + name := fmt.Sprintf("%s_%d", strings.ToLower(op), index) + fmt.Fprintf(&decl, "func %s(out *[8]%s)\n", name, word) + fmt.Fprintf(&names, "%s,\n", name) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$0-%d\nMOV%s out+0(FP),DI\n", name, step, suffix) + for r, reg := range []string{"AX", "BX", "CX", "DX", "SI"} { + fmt.Fprintf(&asm, "MOV%s $%s,%s\n", suffix, values[r], reg) + } + fmt.Fprintf(&asm, "PUSH%s $%d\n%s\n%s\n", suffix, seed, pop, op) + for r, reg := range []string{"AX", "BX", "CX", "DX", "SI"} { + fmt.Fprintf(&asm, "MOV%s %s,%d(DI)\n", suffix, reg, r*step) + } + fmt.Fprintf(&asm, "%s\nPOP%s SI\nMOV%s SI,%d(DI)\nSTMXCSR %d(DI)\nRET\n", push, suffix, suffix, 5*step, 6*step) + } + } + decl.WriteString("func csr() uint32\n") + asm.WriteString("TEXT ·csr(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n") + driver = fmt.Sprintf(`package main + +import ( + "fmt" + "runtime" +) + +func main() { + runtime.LockOSThread() + samples := []func(*[8]%s){ + %s + } + want := [5]%s{%s} + seeds := [2]%s{0x8c7, 0x12} + observations := 0 + for index, sample := range samples { + for iteration := 0; iteration < 32; iteration++ { + out := [8]%s{0, 0, 0, 0, 0, 0, 0, %s} + before := csr() + sample(&out) + after := csr() + for register := 0; register < 5; register++ { + if out[register] != want[register] { + panic(fmt.Sprintf("original GP kernel=%%d register=%%d out=%%x", index, register, out)) + } + } + if out[5]&0x8d5 != seeds[index%%2]&0x8d5 || + out[6] != %s(before) || after != before || out[7] != %s { + panic(fmt.Sprintf("original status/CSR/canary kernel=%%d out=%%x before=%%x after=%%x", index, out, before, after)) + } + observations++ + } + } + fmt.Printf("original ordering observations %%d\n", observations) +} +`, word, names.String(), word, strings.Join(values, ", "), word, word, canary, word, canary) + var prototypes strings.Builder + for _, op := range []string{"lfence", "mfence", "sfence", "pause"} { + for index := 0; index < 2; index++ { + fmt.Fprintf(&prototypes, "extern void %s_%d(%s_t *);\n", op, index, word) + } + } + reference = fmt.Sprintf(` +#include +#include + +%s +int main(void) { + void (*samples[])(%s_t *) = { + %s + }; + const %s_t want[5] = {%s}; + const %s_t seeds[2] = {0x8c7, 0x12}; + unsigned observations = 0; + for (unsigned index = 0; index < 8; index++) { + for (unsigned iteration = 0; iteration < 32; iteration++) { + %s_t out[8] = {0, 0, 0, 0, 0, 0, 0, %s}; + unsigned before, after; + __asm__ volatile("stmxcsr %%0" : "=m"(before)); + samples[index](out); + __asm__ volatile("stmxcsr %%0" : "=m"(after)); + for (unsigned reg = 0; reg < 5; reg++) { + if (out[reg] != want[reg]) { + fprintf(stderr, "GP kernel=%%u register=%%u got=%%llx\n", + index, reg, (unsigned long long)out[reg]); + return 1; + } + } + if ((out[5] & 0x8d5) != (seeds[index %% 2] & 0x8d5) || + out[6] != before || after != before || out[7] != %s) { + fprintf(stderr, "status/CSR/canary kernel=%%u flags=%%llx CSR=%%llx/%%x/%%x\n", + index, (unsigned long long)out[5], (unsigned long long)out[6], before, after); + return 2; + } + observations++; + } + } + printf("ordering observations %%u\n", observations); + return 0; +} +`, prototypes.String(), word, names.String(), word, strings.Join(values, ", "), word, word, canary, canary) + return decl.String(), asm.String(), driver, reference +} diff --git a/amd64_original_package_observation_red_test.go b/amd64_original_package_observation_red_test.go new file mode 100644 index 000000000..ba88108d8 --- /dev/null +++ b/amd64_original_package_observation_red_test.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "github.com/xgo-dev/plan9asm/internal/goasmcmd" +) + +// Ordinary bodies and SDK imports may accompany a sealed object observation, +// but only bodyless assembly declarations enter its function/layout map. +// Observation remains separate from fixed-local/caller/native-memory permission. +type originalPackageObservationVariant struct { + name string + directory string + declaration string + data bool +} + +var originalPackageObservationVariants = []originalPackageObservationVariant{ + {name: "nested", directory: "internal/kernel"}, + {name: "bodyful", declaration: "func Reference(a uint64) uint64 { return a + 1 }\n"}, + {name: "import", declaration: "import _ \"math/bits\"\n"}, + {name: "data", data: true}, +} + +type originalPackageObservationSelected struct { + Dir, ImportPath, Export string + GoFiles, CompiledGoFiles, SFiles, Imports []string + ImportMap map[string]string + Module *struct { + Path, Version, Dir, GoMod, GoVersion string + Main bool + } +} + +func originalPackageObservationFixture(t *testing.T, variant originalPackageObservationVariant) (GoPackage, GoAMD64FrameCaptureRequest) { + t.Helper() + _, request := goAMD64FrameSourceFixture(t) + root := request.Directory + const module = "example.invalid/framefixture" + directory := root + packagePath := module + if variant.directory != "" { + directory = filepath.Join(root, filepath.FromSlash(variant.directory)) + packagePath += "/" + variant.directory + if err := os.MkdirAll(directory, 0700); err != nil { + t.Fatal(err) + } + } + declaration := "package framefixture\n" + variant.declaration + "func Value(a uint64) uint64\n" + files := map[string]string{ + "value.go": declaration, + "value_amd64.s": "#include \"textflag.h\"\nTEXT ·Value(SB),NOSPLIT,$160-16\n" + + "MOVQ a+0(FP),AX\nMOVQ AX,16(SP)\nMOVQ 16(SP),AX\nMOVQ AX,ret+8(FP)\nRET\n", + } + if variant.data { + files["data_amd64.s"] = "#include \"textflag.h\"\nDATA ·constant<>+0(SB)/8,$1\n" + + "GLOBL ·constant<>(SB),RODATA|NOPTR,$8\n" + } + for name, source := range files { + if err := os.WriteFile(filepath.Join(directory, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + request.Directory = directory + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + env := append([]string(nil), request.Environment...) + env = append(env, "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOFLAGS=", "GOWORK=off", "GOTOOLCHAIN=local", "GOENV=off") + data, err := goasmcmd.Run(ctx, directory, env, request.GoBinary, + "list", "-deps", "-compiled", "-export", "-mod=readonly", "-json", ".") + if err != nil { + t.Fatalf("first-party original Go export failed: %v; %s", err, data) + } + t.Logf("actual original Go selected/export graph (%s):\n%s", variant.name, data) + exports := make(map[string]string) + var selected *originalPackageObservationSelected + decoder := json.NewDecoder(bytes.NewReader(data)) + for { + var pkg originalPackageObservationSelected + if err := decoder.Decode(&pkg); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if pkg.Export != "" { + if _, duplicate := exports[pkg.ImportPath]; duplicate { + t.Fatalf("duplicate actual export: %s", pkg.ImportPath) + } + exports[pkg.ImportPath] = pkg.Export + } + if pkg.ImportPath == packagePath { + if selected != nil { + t.Fatal("duplicate actual original package selection") + } + copy := pkg + selected = © + } + } + if selected == nil || selected.Module == nil || !selected.Module.Main || + selected.Module.Path != module || selected.Module.Version != "" || + len(selected.CompiledGoFiles) == 0 || len(selected.SFiles) == 0 || selected.Export == "" { + t.Fatalf("incomplete actual original main-package export: %+v", selected) + } + actualDir, err := filepath.EvalSymlinks(selected.Dir) + if err != nil { + t.Fatal(err) + } + wantedDir, err := filepath.EvalSymlinks(directory) + if err != nil || actualDir != wantedDir { + t.Fatalf("actual package physical root differs: %s, %s, %v", actualDir, wantedDir, err) + } + if variant.data && len(selected.SFiles) != 2 { + t.Fatalf("original Go omitted the DATA-only sibling: %v", selected.SFiles) + } + if variant.name == "import" && exports["math/bits"] == "" { + t.Fatal("actual standard-library dependency export was not observed") + } + fset := token.NewFileSet() + var syntax []*ast.File + for _, name := range selected.CompiledGoFiles { + if !filepath.IsAbs(name) { + name = filepath.Join(selected.Dir, name) + } + file, err := parser.ParseFile(fset, name, nil, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + syntax = append(syntax, file) + } + actualImporter := importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) { + if actual, mapped := selected.ImportMap[path]; mapped { + path = actual + } + filename := exports[path] + if filename == "" { + t.Fatalf("missing actual selected export for %s", path) + } + return os.Open(filename) + }) + typed, err := (&types.Config{Importer: actualImporter, Sizes: types.SizesFor("gc", "amd64"), + GoVersion: "go" + selected.Module.GoVersion}).Check(packagePath, fset, syntax, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: packagePath, Types: typed, Syntax: syntax}, request +} + +func TestGoAMD64OriginalPackageObservationKeepsAuthorityBoundaries(t *testing.T) { + for _, variant := range originalPackageObservationVariants { + t.Run(variant.name, func(t *testing.T) { + pkg, request := originalPackageObservationFixture(t, variant) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + observed, err := CaptureGoAMD64FrameSource(ctx, pkg, request) + if observed != nil { + defer func() { + if err := observed.Close(); err != nil { + t.Error(err) + } + }() + } + if variant.name == "bodyful" || variant.name == "import" { + if err != nil || observed == nil { + t.Fatalf("ordinary original source lost its sealed observation: %v", err) + } + if len(observed.functions) != 1 || len(observed.declarations) != 1 || + observed.functions[pkg.Path+".Value"].locals != 168 { + t.Fatal("ordinary Go declaration manufactured an assembly frame observation") + } + if err := observed.Verify(ctx); err != nil { + t.Fatal(err) + } + // Even a live real-object observation supplies no aligned native + // stack storage to any translator without its separate Binder. + const assembly = "TEXT ·Value(SB),4,$160-16\nPXOR X0,X0\nMOVOU X0,16(SP)\nPADDQ 16(SP),X0\nMOVQ $1,ret+8(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.Replace(symbol, "·", pkg.Path+".", 1) } + sig := observed.functions[pkg.Path+".Value"].signature + opt := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, + Sigs: map[string]FuncSig{sig.Name: sig}} + if _, err := Translate(file, opt); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("ordinary observation granted native memory to text API: %v", err) + } + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("ordinary observation granted native memory to module API: %v", err) + } + translated, err := translateGoModuleInContext(context, pkg, []byte(assembly), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: opt.TargetTriple, ResolveSym: resolve}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("ordinary observation granted native memory to typed Go API: %v", err) + } + } else if !errors.Is(err, ErrProbeNeedsContext) || observed != nil { + t.Fatalf("unsupported original source role acquired an observation: %v", err) + } + }) + } +} diff --git a/amd64_original_sysv_test.go b/amd64_original_sysv_test.go new file mode 100644 index 000000000..401b70039 --- /dev/null +++ b/amd64_original_sysv_test.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// These test-only byte wrappers explicitly consume SysV registers, even on +// Windows. Both declarations and indirect calls must retain that convention; +// translated functions still use their target's ordinary C ABI. This is not a +// production Go ABI contract. See clang.llvm.org/docs/AttributeReference.html#sysv-abi. +const x86OriginalSysVAttribute = "__attribute__((sysv_abi))" + +func x86OriginalSysVDeclaration(result, name, arguments string) string { + return fmt.Sprintf("extern %s %s %s(%s);\n", result, x86OriginalSysVAttribute, name, arguments) +} + +func x86OriginalSysVPointer(result, name, arguments string) string { + return fmt.Sprintf("typedef %s (%s *%s)(%s);\n", result, x86OriginalSysVAttribute, name, arguments) +} + +func TestX86OriginalSysVDirectAndPointerTransport(t *testing.T) { + clang, llc := findLLVM22Tool("clang"), findLLVM22Tool("llc") + if clang == "" || llc == "" { + t.Fatal("LLVM 22 clang and llc required") + } + var source strings.Builder + source.WriteString("typedef unsigned char uint8_t; typedef unsigned int uint32_t; typedef unsigned long long uint64_t; typedef long long int64_t;\n") + for index, signature := range []struct { + result, parameters, arguments string + }{ + {"uint64_t", "uint32_t k,uint32_t h", "k,h"}, + {"uint64_t", "uint8_t *out,int64_t index,uint64_t value", "out,index,value"}, + {"void", "uint8_t *out,const uint8_t *input,uint64_t old,uint64_t flags", "out,input,old,flags"}, + {"void", "uint8_t *out,float input,uint64_t old,uint64_t flags", "out,input,old,flags"}, + {"void", "uint8_t *out,double input,uint64_t old,uint64_t flags", "out,input,old,flags"}, + {"void", "uint8_t *out,const uint8_t *a,const uint8_t *b,const uint8_t *old,uint64_t flags,uint32_t csr", "out,a,b,old,flags,csr"}, + } { + name, pointer := fmt.Sprintf("Original%d", index), fmt.Sprintf("OriginalProgram%d", index) + source.WriteString(x86OriginalSysVDeclaration(signature.result, name, signature.parameters)) + source.WriteString(x86OriginalSysVPointer(signature.result, pointer, signature.parameters)) + fmt.Fprintf(&source, "extern %s Translated%d(%s);\n", signature.result, index, signature.parameters) + fmt.Fprintf(&source, "void Check%d(%s pointer,%s) { %s(%s); pointer(%s); Translated%d(%s); }\n", + index, pointer, signature.parameters, name, signature.arguments, signature.arguments, index, signature.arguments) + } + for _, target := range []string{"x86_64-pc-windows-msvc", "x86_64-w64-windows-gnu", "x86_64-unknown-linux-gnu", "x86_64-apple-darwin"} { + t.Run(target, func(t *testing.T) { + directory := t.TempDir() + input, output := filepath.Join(directory, "transport.c"), filepath.Join(directory, "transport.ll") + if err := os.WriteFile(input, []byte(source.String()), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, clang, "-target", target, "-nostdinc", "-Werror", + "-O0", "-S", "-emit-llvm", input, "-o", output) + if data, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("typed direct/indirect transport: %v\n%s", err, data) + } + ir, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if strings.Contains(target, "windows") && strings.Count(string(ir), "call x86_64_sysvcc ") != 12 { + t.Fatal("Windows direct or pointer call erased the original SysV ABI") + } + translatedCalls := 0 + for _, line := range strings.Split(string(ir), "\n") { + if strings.Contains(line, "call ") && strings.Contains(line, "@Translated") { + translatedCalls++ + if strings.Contains(line, "x86_64_sysvcc") { + t.Fatal("original byte-wrapper ABI leaked into a translated call") + } + } + } + if translatedCalls != 6 { + t.Fatalf("translated call controls = %d, want 6", translatedCalls) + } + compileLLVMToObject(t, llc, target, "transport.ll", "transport.o", string(ir)) + }) + } +} diff --git a/amd64_packed_add_fault_runtime_test.go b/amd64_packed_add_fault_runtime_test.go new file mode 100644 index 000000000..7cce0415f --- /dev/null +++ b/amd64_packed_add_fault_runtime_test.go @@ -0,0 +1,124 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddNativeMemoryStatus(t *testing.T) { + x86PackedAddNativeMemoryStatus(t, x86WrappingAddOpcodes, []int{8, 16, 32, 64, 8, 16, 32, 64}) +} + +func TestCrossLinuxRuntimeMatrixX86PackedSaturatingAddNativeMemoryStatus(t *testing.T) { + x86PackedAddNativeMemoryStatus(t, []string{ + "PADDSB", "PADDSW", "PADDUSB", "PADDUSW", + "VPADDSB", "VPADDSW", "VPADDUSB", "VPADDUSW", + }, []int{8, 16, 8, 16, 8, 16, 8, 16}) +} + +func x86PackedAddNativeMemoryStatus(t *testing.T, opcodes []string, laneBits []int) { + t.Helper() + if len(opcodes) != len(laneBits) { + t.Fatal("opcode/lane inventory differs") + } + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, references, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for opcodeIndex, opcode := range opcodes { + legacy := !strings.HasPrefix(opcode, "V") + bits := laneBits[opcodeIndex] + modes := []int{-1} + if !legacy { + modes = []int{-1, 0, 1} + } + for _, mode := range modes { + name := fmt.Sprintf("packedaddfault%d", index) + index++ + row := opcode + " (BX),X0" + if !legacy { + row = opcode + " (BX),Z0,Z1" + if mode >= 0 { + row = opcode + ".Z (BX),Z0,K1,Z1" + } + } + mask := mode + if mask < 0 { + mask = 0 + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ input+0(FP),BX\nVPXORD Z0,Z0,Z0\nMOVL $%d,CX\nKMOVQ CX,K1\n%s\nRET\n", name, mask, row) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*);\n", name) + reference := fmt.Sprintf("reference%d", index) + if legacy { + mnemonic := strings.ToLower(opcode) + if mnemonic == "paddl" { + mnemonic = "paddd" + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t*p){typedef int32_t V __attribute__((vector_size(16)));V v={0,0,0,0};__asm__ volatile(\"%s (%%1),%%0\":\"+x\"(v):\"r\"(p):\"memory\");}\n", reference, mnemonic) + } else { + width := 64 + if mode == 0 { + width = 0 + } else if mode == 1 { + width = bits / 8 + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t*p){for(int i=0;i<%d;++i)(void)((const volatile uint8_t*)p)[i];}\n", reference, width) + } + fmt.Fprintf(&checks, "for(int n=0;n<16;++n)if(compare(%s,%s,memory+n))return 1;\n", reference, name) + for _, offset := range []int{0, 1, bits / 8, 16, 64} { + fmt.Fprintf(&checks, "if(compare(%s,%s,boundary-%d))return 1;\n", reference, name, offset) + } + fmt.Fprintf(&checks, "if(compare(%s,%s,0))return 1;\n", reference, name) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + references.String() + ` +typedef struct {int signal;int code;} Status; +static int observe(void(*fn)(const uint8_t*),const uint8_t*p,Status*out){ + pid_t child=fork();if(child<0)return 1; + if(child==0){struct rlimit limit={0,0};if(setrlimit(RLIMIT_CORE,&limit))_exit(3);fn(p);_exit(0);} + int status;if(waitpid(child,&status,0)!=child)return 1; + if(WIFSIGNALED(status)){*out=(Status){WTERMSIG(status),0};return 0;} + if(WIFEXITED(status)){*out=(Status){0,WEXITSTATUS(status)};return 0;}return 1; +} +static int compare(void(*original)(const uint8_t*),void(*translated)(const uint8_t*),const uint8_t*p){ + Status a,b;if(observe(original,p,&a)||observe(translated,p,&b))return 1; + if(a.signal!=b.signal||a.code!=b.code){fprintf(stderr,"pointer=%p mod16=%lu original=(%d,%d) translated=(%d,%d)\n",p,(unsigned long)((uintptr_t)p%16),a.signal,a.code,b.signal,b.code);return 1;}return 0; +} +int main(void){ + long page=sysconf(_SC_PAGESIZE);if(page<=0||page%64)return 1; + uint8_t*memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED)return 1;memset(memory,0x87,(size_t)page); + if(mprotect(memory,(size_t)page,PROT_READ)||mprotect(memory+page,(size_t)page,PROT_NONE))return 1; + uint8_t*boundary=memory+page; +` + checks.String() + "if(munmap(memory,(size_t)page*2))return 1;return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-add-fault", triple, text, main, prefix) + }) + } + t.Log("all four original SSE opcodes typed-status at every mod16/NULL/RO guard; E4/E4.nb zero/active masks observed against independent volatile reads, not unexecuted AVX512 hardware") +} diff --git a/amd64_packed_add_forms.go b/amd64_packed_add_forms.go new file mode 100644 index 000000000..3cae75198 --- /dev/null +++ b/amd64_packed_add_forms.go @@ -0,0 +1,69 @@ +package plan9asm + +import "strings" + +type x86PackedAddSpec struct { + x86PackedBinarySpec + laneBits int +} + +// The eight wrapping Go spellings, plus cmd/asm's explicit PADDD -> APADDL +// alias. Saturating adds have a distinct arithmetic contract and do not gain +// this family's private caller permission. +var x86PackedWrappingAddSpecs = map[Op]x86PackedAddSpec{ + "PADDB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 8}, + "PADDW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16}, + "PADDL": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 32}, + "PADDD": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 32}, + "PADDQ": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 64}, + "VPADDB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8}, + "VPADDW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16}, + "VPADDD": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32}, + "VPADDQ": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 8, zeroing: true}, 64}, +} + +type x86PackedAddForm struct { + x86PackedBinaryForm + spec x86PackedAddSpec +} + +type x86PackedIntegerAddForm struct { + x86PackedBinaryForm + spec amd64PackedAddSpec +} + +// Wrapping and saturating adds have the same Go operand grammar, but retain +// distinct arithmetic modes. Sharing a form does not extend caller authority. +func x86PackedIntegerAddProperties(ins Instr, goarch string) (x86PackedIntegerAddForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := amd64PackedAddSpecs[Op(base)] + if !known { + return x86PackedIntegerAddForm{}, false, nil + } + if spec.form != amd64PackedAddVEXEVEX { + form, _, err := x86LegacyIntegerArithmeticProperties(ins.Op, ins, goarch) + if err == nil { + err = x86LegacyIntegerArithmeticEmitterContext(form, ins) + } + return x86PackedIntegerAddForm{form.x86PackedBinaryForm, spec}, true, err + } + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true} + if spec.allowBroadcast { + grammar.broadcastBytes = spec.laneBits / 8 + } + form, err := x86PackedBinaryProperties(ins, goarch, grammar) + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + return x86PackedIntegerAddForm{form, spec}, true, err +} + +func x86PackedWrappingAddProperties(ins Instr, goarch string) (x86PackedAddForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86PackedWrappingAddSpecs[Op(base)] + if !known { + return x86PackedAddForm{}, false, nil + } + form, _, err := x86PackedIntegerAddProperties(ins, goarch) + return x86PackedAddForm{form.x86PackedBinaryForm, spec}, true, err +} diff --git a/amd64_packed_add_forms_test.go b/amd64_packed_add_forms_test.go new file mode 100644 index 000000000..808a6d8b6 --- /dev/null +++ b/amd64_packed_add_forms_test.go @@ -0,0 +1,315 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86WrappingAddOpcodes = []string{"PADDB", "PADDW", "PADDL", "PADDQ", "VPADDB", "VPADDW", "VPADDD", "VPADDQ"} + +func x86WrappingAddFormats(arch string) []string { + var rows []string + for _, opcode := range append(append([]string(nil), x86WrappingAddOpcodes...), "PADDD") { + spec := x86PackedWrappingAddSpecs[Op(opcode)] + if !strings.HasPrefix(opcode, "V") { + rows = append(rows, opcode+" X0,X7", opcode+" 16(AX)(BX*2),X7") + if arch == "amd64" { + rows = append(rows, opcode+" X15,X0", opcode+" data(SB),X0", opcode+" (FS),X0", opcode+" (GS),X0") + if opcode != "PADDQ" { + rows = append(rows, opcode+" M0,M7", opcode+" (AX),M7", opcode+" source+0(FP),M7") + } + } + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "16(AX)(BX*2)", "source+0(FP)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s7", opcode, source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s,%s31,K1,%s30", opcode, source, bank, bank), fmt.Sprintf("%s.Z %s,%s31,K7,%s30", opcode, source, bank, bank)) + } + } + if spec.broadcastBytes != 0 { + rows = append(rows, fmt.Sprintf("%s.BCST (AX),%s1,%s7", opcode, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s.BCST.Z source+0(FP),%s31,K7,%s30", opcode, bank, bank)) + } + } + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s31,%s30,%s29", opcode, bank, bank, bank)) + } + } + } + return rows +} + +func TestX86PackedWrappingAddActualGoCompleteAttributes(t *testing.T) { + if len(x86PackedWrappingAddSpecs) != 9 { + t.Fatal("eight wrapping operations and the actual Go PADDD alias required") + } + for index, opcode := range x86WrappingAddOpcodes { + spec := x86PackedWrappingAddSpecs[Op(opcode)] + bits := 8 << uint(index%4) + broadcast := 0 + if index >= 6 { + broadcast = bits / 8 + } + if spec.laneBits != bits || spec.broadcastBytes != broadcast || spec.zeroing != (index >= 4) { + t.Fatalf("%s has wrong independent arithmetic/tuple attributes: %+v", opcode, spec) + } + } + if x86PackedWrappingAddSpecs["PADDD"] != x86PackedWrappingAddSpecs["PADDL"] { + t.Fatal("actual Go alias must retain the exact canonical operation") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil || !strings.Contains(string(aliases), `instructions["PADDD"] = x86.APADDL`) { + t.Fatalf("actual Go PADDD alias registration: %v", err) + } + for index, opcode := range []string{"PADDB", "PADDW", "PADDL", "PADDQ"} { + byteOp := []int{0xfc, 0xfd, 0xfe, 0xd4}[index] + registration := fmt.Sprintf("{A%s, ymm, Py1, opBytes{0x%x, Pe, 0x%x}}", opcode, byteOp, byteOp) + if opcode == "PADDQ" { + registration = "{APADDQ, yxm, Pe, opBytes{0xd4}}" + } + if !strings.Contains(string(legacy), registration) { + t.Fatalf("actual Go legacy registration changed: %s", registration) + } + opcode = []string{"VPADDB", "VPADDW", "VPADDD", "VPADDQ"}[index] + start := strings.Index(string(avx), "{as: A"+opcode+",") + if start < 0 { + t.Fatal("actual Go missing " + opcode) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(row, "ytab: _yvandnpd") || strings.Contains(row, "Sae") || strings.Contains(row, "Rounding") { + t.Fatalf("actual Go operand attributes changed: %s", row) + } + for _, width := range []string{"vex128", "vex256"} { + if !strings.Contains(row, fmt.Sprintf("avxEscape | %s | vex66 | vex0F | vexW0, 0x%X", width, byteOp)) { + t.Fatal("missing actual VEX registration " + opcode) + } + } + for i, width := range []string{"evex128", "evex256", "evex512"} { + w, broadcast := 0, "" + if index >= 2 { + broadcast = fmt.Sprintf(" | evexBcstN%d", 4<<(index-2)) + } + if index == 3 { + w = 1 + } + want := fmt.Sprintf("avxEscape | %s | evex66 | evex0F | evexW%d, evexN%d%s | evexZeroingEnabled, 0x%X", width, w, 16<= 0 { + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := out.String() + _, _, err := ctx.lowerPackedIntegerAdd(ins.Op, ins) + if missing < 0 && err != nil || missing >= 0 && (err == nil || out.String() != before) { + t.Fatalf("%s/%s missing FP byte%d: %v", op, bank, missing, err) + } + } + } + } + } + for _, opcode := range []string{"PADDB", "PADDW", "PADDL", "PADDD"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+opcode+" source+0(FP),M1\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + for missing := -1; missing < 8; missing++ { + sig := x86WordMaddFPType(t) + if missing >= 0 { + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := out.String() + _, _, err := ctx.lowerPackedIntegerAdd(ins.Op, ins) + if missing < 0 && err != nil || missing >= 0 && (err == nil || out.String() != before) { + t.Fatalf("%s MMX missing FP byte%d: %v", opcode, missing, err) + } + } + } +} + +func TestX86PackedWrappingAdd386HighRegisterActualByteIdentities(t *testing.T) { + for _, opcode := range []string{"VPADDB", "VPADDW", "VPADDD", "VPADDQ"} { + for _, bank := range []string{"X", "Y"} { + for operand := 0; operand < 3; operand++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[operand] = bank + "16" + row := opcode + " " + strings.Join(args, ",") + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual Go mode32 object: %v\n%s", err, listing) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + for _, field := range fields[1:7] { + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + t.Fatal(err) + } + code = append(code, byte(value)) + } + break + } + ins, length, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if len(code) != 6 || !recognized || err != nil || length != 6 || ins.Op != Op(opcode) || len(ins.Args) != 3 { + t.Fatalf("actual EVEX byte identity %s/%x: %+v/%d/%t/%v", row, code, ins, length, recognized, err) + } + for index, argument := range args { + if ins.Args[index].Reg != Reg(argument) { + t.Fatalf("%s/%x operand%d: %s != %s", row, code, index, ins.Args[index].Reg, argument) + } + } + if _, _, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 32); !recognized || err == nil { + t.Fatalf("raw mode32 high bank obtained false permission: %s/%x", row, code) + } + t.Logf("%s actual Go bytes=%x; mode64 identities exact, mode32 rejected", row, code) + } + } + } +} diff --git a/amd64_packed_add_go_oracle_test.go b/amd64_packed_add_go_oracle_test.go new file mode 100644 index 000000000..d3167580c --- /dev/null +++ b/amd64_packed_add_go_oracle_test.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddOriginalGoBits(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import "testing" +func lane(p []byte,n int)uint64{var value uint64;for j:=0;j>24)}} + for i:=range patterns[0]{patterns[0][i]=0xff;patterns[1][i]=1;patterns[2][i]=0x80;patterns[3][i]=0} +`) + count := 0 + for index, opcode := range []string{"PADDB", "PADDW", "PADDL", "PADDQ"} { + bytes := 1 << uint(index) + for _, bank := range []string{"X", "M"} { + if bank == "M" && opcode == "PADDQ" { + continue // Actual Go PADDQ uses yxm, not Intel's MMX row. + } + forms := []string{"register"} + if bank == "M" { + forms = append(forms, "memory") + } + for _, form := range forms { + name := fmt.Sprintf("packedaddgo%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n", name) + move, width := "MOVOU", 16 + if bank == "M" { + move, width = "MOVQ", 8 + } + fmt.Fprintf(&assembly, "%s (BX),%s0\n", move, bank) + first := "(AX)" + if form == "register" { + first = bank + "1" + fmt.Fprintf(&assembly, "%s (AX),%s1\n", move, bank) + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\n%s %s,%s0\n%s %s0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nSTMXCSR 18(CX)\n", opcode, first, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&tests, ` + for _,a:=range patterns{for _,b:=range patterns{for mode:=0;mode<4;mode++{ + var out [24]byte;for i:=range out{out[i]=0xc3};csr:=uint32(0x1f80|mode<<13|mode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0]!=0xc3||out[23]!=0xc3||out[17]!=1||out[18]!=0{t.Fatalf("%[1]s flags/canary %%x",out)} + gotCSR:=uint32(out[19])|uint32(out[20])<<8|uint32(out[21])<<16|uint32(out[22])<<24 + if gotCSR!=csr{t.Fatalf("%[1]s CSR %%x != %%x",gotCSR,csr)} + for i:=0;i<%[2]d;i+=%[3]d{ + want:=lane(a[i:],%[3]d)+lane(b[i:],%[3]d) + for j:=0;j<%[3]d;j++{if out[i+j+1]!=byte(want>>uint(8*j)){t.Fatalf("%[1]s byte%%d got%%x want%%x",i+j,out,want)}} + } + for i:=%[2]d;i<16;i++{if out[i+1]!=0xc3{t.Fatal("touched upper MMX canary")}} + }}} +`, name, width, bytes) + } + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go modulo integer-add oracle: %v\n%s", err, output) + } + t.Logf("%d actual Go SSE/MMX functions × 256 independent full-lane wrap/MXCSR/flags/canary cases; no x87 tag-state claim", count) +} diff --git a/amd64_packed_add_runtime_test.go b/amd64_packed_add_runtime_test.go new file mode 100644 index 000000000..fc1a44df2 --- /dev/null +++ b/amd64_packed_add_runtime_test.go @@ -0,0 +1,227 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" +) + +const x86PackedAddIndependentOracle = ` +static const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; +static uint64_t lane(const uint8_t *p,int bytes) { + uint64_t v=0;for(int j=0;j>(i/bytes))&1); + for(int j=0;j>(j*8)) : mode==2 ? 0 : old[i+j]) : legacy ? b[i+j] : 0; + if(output[i+j+1]!=want) {fprintf(stderr,"%s round=%d mask=%llx byte=%d got=%x want=%x\n",name,round,(unsigned long long)mask,i+j,output[i+j+1],want);return 1;} + } + } + for(int i=0;i<96;++i) { + if((i=shift+64) && aStorage[i]!=0xa7) return 1; + if((i<3 || i>=67) && bStorage[i]!=0xb9) return 1; + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedWrappingAddCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for index, opcode := range x86WrappingAddOpcodes { + bits := 8 << uint(index%4) + legacy := !strings.HasPrefix(opcode, "V") + widths := []int{16} + if !legacy { + widths = []int{16, 32, 64} + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, kind := range []string{"register", "memory", "broadcast"} { + if kind == "broadcast" && (legacy || bits < 32) { + continue + } + for mode := 0; mode < 3; mode++ { + if legacy && mode != 0 { + continue + } + destination := mode + if legacy { + destination = 0 + } + name := fmt.Sprintf("packedadd%d", count) + count++ + first := bank + "1" + if kind != "register" { + first = "(BX)" + } + row := opcode + " " + first + ",X0" + if !legacy { + suffix := "" + if kind == "broadcast" { + suffix = ".BCST" + } + if mode == 2 { + suffix += ".Z" + } + row = opcode + suffix + " " + first + "," + bank + "0" + if mode != 0 { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", name, row, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkAdd(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d)) return 1;\n", name, name, width, bits/8, mode, x86FloatAddSubBoolInt(legacy), destination, x86FloatAddSubBoolInt(kind != "register"), x86FloatAddSubBoolInt(kind == "broadcast")) + } + } + } + } + for index, opcode := range []string{"PADDB", "PADDW", "PADDL"} { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("packedadd%d", count) + count++ + first := "M1" + if memory { + first = "(BX)" + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nVMOVUPS (CX),Z0\nVMOVUPS Z0,(AX)\nMOVQ (CX),M0\nMOVQ (BX),M1\nCMPQ AX,AX\nSTC\n%s %s,M0\nMOVQ M0,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nEMMS\nRET\n", name, opcode, first) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkAdd(%s,\"%s\",8,%d,0,1,0,%d,0))return 1;\n", name, name, 1<>(i/%[3]d))&1); + for(int j=0;j<%[3]d;++j) { + uint8_t want=active?(uint8_t)(sum>>(8*j)):%[5]d==2?0:before[i+j]; + if(output[i+j+1]!=want) {fprintf(stderr,"%[1]s FP byte=%%d got=%%x want=%%x\n",i+j,output[i+j+1],want);return 1;} + } + } + if(input[0]!=0xa7 || input[65]!=0xa7 || output[0]!=0xc3 || output[65]!=0xc3) return 1; + } +`, name, width, bits/8, x86FloatAddSubBoolInt(broadcast), mode) + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", definitions.String(), true) + file, err := Parse(ArchAMD64, definitions.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + ir += wrappers.String() + main := "#include \n#include \n#include \n" + declarations.String() + x86PackedAddIndependentOracle + "int main(void){\n" + checks.String() + "return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "wrapping-add-fp", triple, text, main, prefix) + }) + } + t.Logf("%d complete independent-byte FP programs × 64 logical-value cases at O0/O2; no native FP backing or alignment claim", count) +} diff --git a/amd64_packed_average_forms.go b/amd64_packed_average_forms.go new file mode 100644 index 000000000..a254aba26 --- /dev/null +++ b/amd64_packed_average_forms.go @@ -0,0 +1,79 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedAverageSpec struct { + x86PackedBinarySpec + laneBits int +} + +// Go's complete unsigned rounded-average family: ymm/Py1 for legacy and +// _yvandnpd for VEX/EVEX. E4.nb has mask suppression but no broadcast or SAE. +var x86PackedAverageSpecs = map[Op]x86PackedAverageSpec{ + "PAVGB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 8}, + "PAVGW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16}, + "VPAVGB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 8}, + "VPAVGW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16}, +} + +type x86PackedAverageForm struct { + x86PackedBinaryForm + spec x86PackedAverageSpec + source Operand +} + +// A shape/effect observation supplies no input initialization, caller, frame +// or native storage authority. All consumers retain their own proof gates. +func x86PackedAverageProperties(op Op, ins Instr, arch string) (x86PackedAverageForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedAverageSpecs[Op(base)] + if !known { + return x86PackedAverageForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedAverageForm{}, true, fmt.Errorf("packed average dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + local.Args = append([]Operand(nil), ins.Args...) + if spec.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return x86PackedAverageForm{}, true, err + } + } + grammarArch := arch + if arch == "386" && local.x86Encoded && spec.encoding == x86PackedLogicalVEXOrEVEX { + grammarArch = "amd64" // Only the complete physical decoder sets this. + } + binary, err := x86PackedBinaryProperties(local, grammarArch, spec.x86PackedBinarySpec) + form := x86PackedAverageForm{x86PackedBinaryForm: binary, spec: spec} + if err == nil { + form.source = local.Args[binary.first] + if arch == "386" && grammarArch != arch && binary.memory >= 0 && !x86PackedShiftMemoryValid(form.source, arch) { + err = fmt.Errorf("386 %s physical source has an unbound address", op) + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && binary.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: packed average has no bound address-size override", ErrProbeNeedsContext) + } + } + return form, true, err +} + +func x86TypedPackedAverageByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedAverageProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: form.first, operand: form.source, width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), read: true, write: true}, + }}, true, nil +} diff --git a/amd64_packed_average_fp_test.go b/amd64_packed_average_fp_test.go new file mode 100644 index 000000000..b31c45e7e --- /dev/null +++ b/amd64_packed_average_fp_test.go @@ -0,0 +1,174 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "go/types" + "os" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedAverageFPSpansThreeAPIsFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + t.Run(triple, func(t *testing.T) { + var source, declarations strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, op := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, mode := range []int{0, 1, 2} { + if arch == "386" && mode != 0 { + continue // Actual named Go rejection is tested independently. + } + name := fmt.Sprintf("F%d", count) + count++ + row := op + if mode == 2 { + row += ".Z" + } + row += " input+0(FP)," + bank + "1" + if mode != 0 { + row += ",K1" + } + row += "," + bank + "2" + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-64\n%s\nRET\n", name, row) + fmt.Fprintf(&declarations, "func %s(input [64]byte)\n", name) + } + } + } + if arch == "amd64" { + for _, op := range x86AverageNames[:2] { + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-64\n%s input+0(FP),M1\nRET\n", count, op) + fmt.Fprintf(&declarations, "func F%d(input [64]byte)\n", count) + count++ + } + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, ir, "average-fp-text") + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, module.String(), "average-fp-module") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{ + GOARCH: arch, Sizes: types.SizesFor("gc", arch), TargetTriple: triple, ResolveSym: options.ResolveSym}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, translated.Module.String(), "average-fp-go") + }) + } +} + +func TestX86PackedAverageFPHolesCannotBeMaskedAway(t *testing.T) { + for _, op := range x86AverageNames[2:] { + for index, bank := range []string{"X", "Y", "Z"} { + width := 16 << index + for _, start := range []int{0, 1, -1, 16} { + for _, mode := range []string{"", ".Z"} { + row := fmt.Sprintf("%s%s input%+d(FP),%s1,K1,%s2", op, mode, start, bank, bank) + source := "TEXT f(SB),4,$0-8\nKMOVQ AX,K1\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, frame := range []FrameLayout{ + {Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}}}, + {Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}, {Offset: int64(width - 1), Type: I8, Index: 1, Field: -1}}}, + } { + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void, Args: []LLVMType{I64, I64}, Frame: frame}, nil) + before := output.String() + ins := file.Funcs[0].Instrs[2] + if _, _, err := ctx.lowerPackedUnsignedAverage(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("%s incomplete logical byte span emitted IR: %v", row, err) + } + } + } + } + } + } +} + +func TestX86PackedAverageRequiredLinuxOriginalVEXRegistration(t *testing.T) { + const name = "amd64_packed_average_original_linux_test.go" + for _, goos := range []string{"linux", "darwin", "windows"} { + ctx := build.Default + ctx.GOOS, ctx.GOARCH = goos, "amd64" + selected, err := ctx.MatchFile(".", name) + if err != nil || selected != (goos == "linux") { + t.Fatal("required original AVX axis has wrong actual Go selection", goos, selected, err) + } + } + file, err := parser.ParseFile(token.NewFileSet(), name, nil, 0) + if err != nil { + t.Fatal(err) + } + if len(file.Decls) != 2 { + t.Fatal("required wrapper declaration changed") + } + fn, ok := file.Decls[1].(*ast.FuncDecl) + if !ok || fn.Name.Name != "TestCrossLinuxRuntimeMatrixX86PackedAverageOriginalVEXAliases" || fn.Body == nil || len(fn.Body.List) != 1 { + t.Fatal("required original AVX wrapper omitted or bypassed") + } + expression, ok := fn.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("required wrapper is not an actual unconditional call") + } + call, ok := expression.X.(*ast.CallExpr) + if !ok || len(call.Args) != 2 { + t.Fatal("required wrapper lost its original Go helper") + } + callee, ok := call.Fun.(*ast.Ident) + wide, wideOK := call.Args[1].(*ast.Ident) + if !ok || callee.Name != "x86AverageOriginalGo" || !wideOK || wide.Name != "true" { + t.Fatal("required original wide alias scope was narrowed") + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatal("required Linux CI selector does not select this family", err) + } +} diff --git a/amd64_packed_average_original_go_test.go b/amd64_packed_average_original_go_test.go new file mode 100644 index 000000000..799b1d34f --- /dev/null +++ b/amd64_packed_average_original_go_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedAverageOriginalGo(t *testing.T) { + x86AverageOriginalGo(t, false) +} + +func x86AverageOriginalGo(t *testing.T, wide bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + if wide { + execution.requireAVX(t) + x86AverageRequireAVX2(t, execution) + } + var assembly, declarations, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle + +import ( + "testing" + "unsafe" + "runtime" + "encoding/binary" +) + +func aligned(storage *[192]byte) (*[64]byte,int) { + offset:=64+int((-uintptr(unsafe.Pointer(&storage[64])))&63) + return (*[64]byte)(unsafe.Pointer(&storage[offset])),offset +} + +func TestOriginal(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + for sample:=0;sample<257;sample++ { + for status:=0;status<2;status++ { + var a,b,out [192]byte + ap,_:=aligned(&a) + bp,_:=aligned(&b) + op,offset:=aligned(&out) + for i:=range a { + a[i]=0xa7 + b[i]=0xb9 + out[i]=0xc3 + } + + for i:=0;i<64;i++ { + ap[i]=byte(sample*71+i*37) + bp[i]=byte(sample*19-i*13) + if sample<4 { + ap[i]=[4]byte{255,0,128,255}[sample] + bp[i]=[4]byte{255,1,128,0}[sample] + } + } + beforeA,beforeB:=a,b + flags:=uint64(2) + if status!=0 { + flags|=0x8d5 + } + csr:=uint32(0x1f80|((sample%4)<<13)|(sample%64)) +`) + count := 0 + for _, row := range x86AverageNumericCases() { + if row.width == 64 || row.masking != 0 || !wide && strings.HasPrefix(row.op, "V") { + continue // Native EVEX is a distinct, unexecuted axis here. + } + move, bank, observedWidth := "MOVUPS", "X", 16 + if wide { + move, bank, observedWidth = "VMOVUPS", "Y", 32 + } + name := fmt.Sprintf("Average%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$8-36 +STMXCSR saved-4(SP) +LDMXCSR csr+32(FP) +MOVQ out+0(FP),AX +MOVQ a+8(FP),BX +MOVQ b+16(FP),CX +`, name) + fmt.Fprintf(&assembly, "%s (CX),%s0\n%s (CX),%s2\n", move, bank, move, bank) + if !row.memory || row.destination == 1 { + fmt.Fprintf(&assembly, "%s (BX),%s1\n", move, bank) + } + if row.mmx { + fmt.Fprintf(&assembly, "MOVQ (CX),M0\nMOVQ (BX),M1\n%s %s0,(AX)\n", move, bank) + } + assembly.WriteString("PUSHQ flags+24(FP)\nPOPFQ\n" + row.instruction() + "\n") + if row.mmx { + assembly.WriteString("MOVQ M0,(AX)\nEMMS\n") + } else { + fmt.Fprintf(&assembly, "%s %s%d,(AX)\n", move, bank, row.destination) + } + assembly.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,32(AX)\nSTMXCSR 40(AX)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&tests, `{ + for i:=range out { + out[i]=0xc3 + } + %s(op,ap,bp,flags,csr) + if binary.LittleEndian.Uint64(op[32:40])&0x8d5!=flags&0x8d5 || binary.LittleEndian.Uint32(op[40:44])!=csr || a!=beforeA || b!=beforeB { + t.Fatal("original flags/CSR/input canary") + } + for i:=0;i<%d;i++ { + start:=i/%d*%d + x,y:=uint32(ap[start]),uint32(bp[start]) + if %d==2 { + x|=uint32(ap[start+1])<<8 + y|=uint32(bp[start+1])<<8 + } + value:=(x+y+1)/2 + want:=byte(value>>uint(8*(i%%%d))) + if i>=%d { + want=0 + if %t { + want=bp[i] + } + } + if op[i]!=want { + t.Fatalf("%s sample%%d byte%%d got%%x want%%x",sample,i,op[i],want) + } + } + for i:=range out { + if (i=offset+44||i>=offset+%d&&i +int main(void) { + __builtin_cpu_init(); + int avx2 = !!__builtin_cpu_supports("avx2"); + printf("required original VPAVG Y-width AVX2=%d\n", avx2); + return avx2 ? 0 : 1; +} +` + if err := os.WriteFile(source, []byte(preflight), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + t.Logf("actual original VPAVG capability: %s", strings.TrimSpace(string(execution.run(t, executable)))) +} diff --git a/amd64_packed_average_original_linux_test.go b/amd64_packed_average_original_linux_test.go new file mode 100644 index 000000000..069969777 --- /dev/null +++ b/amd64_packed_average_original_linux_test.go @@ -0,0 +1,11 @@ +//go:build linux + +package plan9asm + +import "testing" + +// Required by the existing Linux cross-matrix selector. Its actual AVX +// preflight and pinned guest transport must fail, not skip, if unavailable. +func TestCrossLinuxRuntimeMatrixX86PackedAverageOriginalVEXAliases(t *testing.T) { + x86AverageOriginalGo(t, true) +} diff --git a/amd64_packed_average_physical_test.go b/amd64_packed_average_physical_test.go new file mode 100644 index 000000000..882827031 --- /dev/null +++ b/amd64_packed_average_physical_test.go @@ -0,0 +1,319 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86AverageNames = []string{"PAVGB", "PAVGW", "VPAVGB", "VPAVGW"} + +// Independent of the production grammar. Named mode32 MMX is excluded by +// Go's Py1 selector, while low X/Y/Z operands have physical mode32 identities. +func x86AveragePhysicalRows(arch string) []string { + var rows []string + for _, name := range x86AverageNames[:2] { + for _, source := range []string{"X0", "X7", "(AX)", "16(AX)(CX*4)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+",X1") + } + if arch == "amd64" { + rows = append(rows, name+" X15,X0", name+" M0,M7", name+" (AX),M7") + } + } + for _, name := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", bank + "7", "(AX)", "16(AX)(CX*4)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s2", name, source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s,%s31,K1,%s30", name, source, bank, bank), + fmt.Sprintf("%s.Z %s,%s31,K7,%s30", name, source, bank, bank)) + } + } + } + } + return rows +} + +func TestX86PackedAverageThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + var source, declarations strings.Builder + declarations.WriteString("package oracle\n") + for index, row := range x86AveragePhysicalRows(arch) { + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$0-0\n%s\nRET\n", index, row) + fmt.Fprintf(&declarations, "func F%d()\n", index) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, ir, "average-text") + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, module.String(), "average-module") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, translated.Module.String(), "average-go") + x86ResultInitializationObject(t, llc, triple, x86MulDivRoundOptimize(t, ir), "average-O2") + }) + } +} + +func TestX86PackedAverageShapesEffectsDoNotGrantCallerPermissions(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, row := range append(x86AveragePhysicalRows(arch), "VPAVGB.Z value+0(FP),X1,K1,X2", "VPAVGW value+0(FP),Z1,Z2") { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + before := fmt.Sprintf("%#v", ins) + form, known, err := x86PackedAverageProperties(ins.Op, ins, arch) + if arch == "386" && strings.Contains(row, "K1") { + if err == nil { + t.Fatal("named mode32 mask became a physical encoding") + } + continue + } + if !known || err != nil || before != fmt.Sprintf("%#v", ins) || x86CallerVectorMoveKnown(ins) { + t.Fatalf("shape changed input or acquired caller permission: %s: %v", row, err) + } + for index := range ins.Args { + width, write, valid := x86InstructionMemoryShape(ins, index, arch) + if valid != (index == form.memory) || valid && (width != int64(form.memoryBytes) || write) { + t.Fatal("memory read shape differs from native width", row, width, write, valid) + } + } + effect, observed, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if form.legacy && (!observed || err != nil || len(effect.operands) != 2 || + effect.flagsDefined != 0 || effect.flagsKilled != 0 || + !effect.operands[0].read || effect.operands[0].write || + !effect.operands[1].read || !effect.operands[1].write || + effect.operands[0].width != int64(form.byteWidth) || effect.operands[1].width != int64(form.byteWidth)) { + t.Fatal("legacy full input/output byte effect changed", row, effect, err) + } + // All three dispatch identities are equivalent, but a contradictory + // source never emits IR. Keep the existing empty direct-helper API. + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "PADDW"} { + local := ins + local.Op = identity + copy := local + _, _, err := x86PackedAverageProperties(ins.Op, local, arch) + bad := identity == "PADDW" + if (err != nil) != bad || !reflect.DeepEqual(copy, local) { + t.Fatal("dispatch identity mismatch", ins.Op, identity, err) + } + } + } + } +} + +func TestX86PackedAverageActualGoCompleteTables(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APAVGB, ymm, Py1, opBytes{0xe0, Pe, 0xe0}}", + "{APAVGW, ymm, Py1, opBytes{0xe3, Pe, 0xe3}}", + } { + if strings.Count(string(legacy), row) != 1 { + t.Fatalf("actual Go legacy registration changed: %s", row) + } + } + start := strings.Index(string(legacy), "var ymm =") + if start < 0 { + t.Fatal("missing actual ymm table") + } + ymm := strings.SplitN(string(legacy)[start:], "\n}", 2)[0] + for _, row := range []string{"{Zm_r_xm, 1, argList{Ymm, Ymr}}", "{Zm_r_xm, 2, argList{Yxm, Yxr}}"} { + if strings.Count(ymm, "argList{") != 2 || strings.Count(ymm, row) != 1 { + t.Fatal("actual ymm class/selector table changed", ymm) + } + } + for index, name := range x86AverageNames[2:] { + opcode := []string{"E0", "E3"}[index] + expected := fmt.Sprintf(`{as: A%s, ytab: _yvandnpd, prefix: Pavx, op: opBytes{ + avxEscape | vex128 | vex66 | vex0F | vexW0, 0x%s, + avxEscape | vex256 | vex66 | vex0F | vexW0, 0x%s, + avxEscape | evex128 | evex66 | evex0F | evexW0, evexN16 | evexZeroingEnabled, 0x%s, + avxEscape | evex256 | evex66 | evex0F | evexW0, evexN32 | evexZeroingEnabled, 0x%s, + avxEscape | evex512 | evex66 | evex0F | evexW0, evexN64 | evexZeroingEnabled, 0x%s, + }},`, name, opcode, opcode, opcode, opcode, opcode) + if err := x86SignedPackedFloatValidateGoOpcodeRows(string(avx), name, expected); err != nil { + t.Fatal(err) + } + } + start = strings.Index(string(avx), "var _yvandnpd =") + if start < 0 { + t.Fatal("missing actual ytab") + } + table := strings.SplitN(string(avx)[start:], "\n}", 2)[0] + "\n}" + if err := x86AverageValidateGoYtab(table); err != nil { + t.Fatal(err) + } +} + +const x86AverageGoYtab = `var _yvandnpd = []ytab{ + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}}, + {zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yym, Yyr, Yyr}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YxmEvex, YxrEvex, YxrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YxmEvex, YxrEvex, Yknot0, YxrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{YymEvex, YyrEvex, YyrEvex}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{YymEvex, YyrEvex, Yknot0, YyrEvex}}, + {zcase: Zevex_rm_v_r, zoffset: 0, args: argList{Yzm, Yzr, Yzr}}, + {zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{Yzm, Yzr, Yknot0, Yzr}}, +}` + +func x86AverageValidateGoYtab(table string) error { + actual, err := x86SignedPackedFloatGoOpcodeTokens(table) + if err != nil { + return err + } + expected, err := x86SignedPackedFloatGoOpcodeTokens(x86AverageGoYtab) + if err != nil { + return err + } + if !reflect.DeepEqual(actual, expected) { + return fmt.Errorf("complete actual Go packed-average selector table changed:\n%s", table) + } + return nil +} + +func TestX86PackedAverageGoSelectorMutationsAreDetected(t *testing.T) { + if err := x86AverageValidateGoYtab(x86AverageGoYtab); err != nil { + t.Fatal(err) + } + for _, mutation := range []struct{ old, replacement string }{ + {"zoffset: 2", "zoffset: 3"}, + {"Zvex_rm_v_r", "Zevex_rm_v_r"}, + {"Yknot0", "Yk"}, + {"YxmEvex", "YymEvex"}, + {"{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}},", ""}, + } { + changed := strings.Replace(x86AverageGoYtab, mutation.old, mutation.replacement, 1) + if err := x86AverageValidateGoYtab(changed); err == nil { + t.Fatalf("selector mutation retained source authority: %#v", mutation) + } + } +} + +func TestX86PackedAverageActualGoModeAndPhysicalBoundary(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + for _, row := range x86AveragePhysicalRows(arch) { + source.WriteString(row + "\n") + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, arch, source.String(), "", true) + } + for _, name := range x86AverageNames[:2] { + for _, source := range []string{"M0", "(AX)", "AX"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+name+" "+source+",M1\nRET\n", "", false) + } + } + for _, name := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y"} { + for position := 0; position < 3; position++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[position] = bank + "20" + assembly := "TEXT f(SB),4,$0-0\n" + name + " " + strings.Join(args, ",") + "\nRET\n" + goAssemblerOracleOutput(t, "386", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := out.String() + known, _, err := ctx.lowerPackedUnsignedAverage(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !known || !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Errorf("unproved physical high bank emitted IR: %s: %v", assembly, err) + } + } + } + } +} + +func TestX86PackedAverageNativeStorageAndIdentityBoundaries(t *testing.T) { + for _, name := range x86AverageNames[:2] { + for _, row := range []string{name + " (SP),X0", name + " (BP),X0", name + " (AX)(BP*2),X0", name + " value+0(FP),X0"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], x86WordMaddFPType(t), nil) + before := out.String() + _, _, err = ctx.lowerPackedUnsignedAverage(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]) + if !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Errorf("virtual storage gained native m128 alignment: %s: %v", row, err) + } + } + } + for _, name := range x86AverageNames { + row := name + " X0,X1" + if strings.HasPrefix(name, "V") { + row += ",X2" + } + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + ins.Op = "PADDW" + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + if _, _, err := ctx.lowerPackedUnsignedAverage(Op(name), ins); err == nil || out.String() != before { + t.Errorf("contradictory opcode emitted PAVG IR: %s: %v", name, err) + } + } +} diff --git a/amd64_packed_average_raw_test.go b/amd64_packed_average_raw_test.go new file mode 100644 index 000000000..0d6d86ebd --- /dev/null +++ b/amd64_packed_average_raw_test.go @@ -0,0 +1,123 @@ +package plan9asm + +import ( + "encoding/json" + "errors" + "fmt" + "os/exec" + "strings" + "testing" +) + +func TestX86PackedAverageActualRawPhysicalModes(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + llc := findLLVM22Tool("llc") + if mc == "" || llc == "" { + t.Fatal("LLVM22 llvm-mc and llc required") + } + for _, op := range x86AverageNames[2:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, memory := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + first, name := bank+"1", op + if memory { + first = "16(AX)" + } + if mode == 2 { + name += ".Z" + } + args := first + "," + bank + "0" + if mode != 0 { + args += ",K1" + } + row := name + " " + args + "," + bank + "2" + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) < 2 || code[len(code)-1] != 0xc3 { + t.Fatal("original object lacks exact RET", row, code) + } + code = code[:len(code)-1] + for _, arch := range []string{"amd64", "386"} { + bits, triple := 64, "x86_64-unknown-linux-gnu" + if arch == "386" { + bits, triple = 32, "i386-unknown-linux-gnu" + var bytes, source strings.Builder + for _, value := range code { + fmt.Fprintf(&bytes, "0x%02x ", value) + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + goAssemblerOracleOutput(t, arch, "TEXT raw(SB),4,$0-0\n"+source.String()+"RET\n", "", true) + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "-triple="+triple, "-disassemble", "-mattr=+avx2,+avx512f,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(bytes.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || !strings.Contains(string(output), strings.ToLower(op)) || strings.Contains(string(output), "warning") { + t.Fatalf("actual mode32 physical instruction invalid: %x: %v\n%s", code, err, output) + } + } + ins, length, known, err := decodedX86PackedAverageInstruction(code, bits) + if !known || err != nil || length != len(code) || !ins.x86Encoded { + t.Fatalf("complete physical bytes not bound: %x: %v", code, err) + } + if _, known, err := x86PackedAverageProperties(ins.Op, ins, arch); !known || err != nil { + t.Fatal("physical source rejected", row, arch, err) + } + encoded, err := json.Marshal(ins) + if err != nil { + t.Fatal(err) + } + var copied Instr + if err := json.Unmarshal(encoded, &copied); err != nil || copied.x86Encoded { + t.Fatal("private marker survived portable JSON", err) + } + if arch == "386" && mode != 0 { + if _, _, err := x86PackedAverageProperties(copied.Op, copied, arch); err == nil { + t.Fatal("named/JSON four operands forged physical mode32 authority") + } + } + var source strings.Builder + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+source.String()+"RET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}}) + if err != nil { + t.Fatal("complete raw source failed translation", row, arch, err) + } + x86ResultInitializationObject(t, llc, triple, ir, "average-raw") + } + } + } + } + } +} + +func TestX86PackedAverageRawReservedAndPrefixBoundaries(t *testing.T) { + valid := []byte{0x62, 0xf1, 0x7d, 0x89, 0xe0, 0xd1} + for _, mutation := range []struct { + index int + value byte + }{{1, 0xf2}, {2, 0x7c}, {3, 0x99}, {3, 0xe9}, {3, 0x88}} { + code := append([]byte(nil), valid...) + code[mutation.index] = mutation.value + _, _, known, err := decodedX86PackedAverageInstruction(code, 64) + if known && err == nil { + t.Fatalf("reserved encoding acquired semantics: %x", code) + } + } + ins, _, known, err := decodedX86PackedAverageInstruction(valid, 32) + if !known || err != nil { + t.Fatal("valid physical mask rejected", err) + } + ins.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + for _, bits := range []int{16, 32, 64} { + local := ins + local.x86AddressBits = bits + if _, _, err := x86PackedAverageProperties(local.Op, local, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("changed address prefix retained a source shape", bits, err) + } + } +} diff --git a/amd64_packed_average_runtime_test.go b/amd64_packed_average_runtime_test.go new file mode 100644 index 000000000..0e31ea1cc --- /dev/null +++ b/amd64_packed_average_runtime_test.go @@ -0,0 +1,298 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86AverageNumericCase struct { + op string + width, bits int + masking, destination int + memory, mmx bool +} + +func x86AverageNumericCases() []x86AverageNumericCase { + var cases []x86AverageNumericCase + for index, op := range x86AverageNames { + widths := []int{8, 16} + if strings.HasPrefix(op, "V") { + widths = []int{16, 32, 64} + } + for _, width := range widths { + for _, memory := range []bool{false, true} { + modes, destinations := 1, 1 + if strings.HasPrefix(op, "V") { + modes, destinations = 3, 3 + } + for mode := 0; mode < modes; mode++ { + for destination := 0; destination < destinations; destination++ { + cases = append(cases, x86AverageNumericCase{op, width, 8 << uint(index%2), mode, destination, memory, width == 8}) + } + } + } + } + } + return cases +} + +func (row x86AverageNumericCase) instruction() string { + bank := map[int]string{8: "M", 16: "X", 32: "Y", 64: "Z"}[row.width] + first := bank + "1" + if row.memory { + first = "(BX)" + } + if !strings.HasPrefix(row.op, "V") { + return row.op + " " + first + "," + bank + "0" + } + op := row.op + if row.masking == 2 { + op += ".Z" + } + args := first + "," + bank + "0" + if row.masking != 0 { + args += ",K1" + } + return fmt.Sprintf("%s %s,%s%d", op, args, bank, row.destination) +} + +func x86AverageNumericProgram(t *testing.T, triple string, cases []x86AverageNumericCase) (string, string) { + t.Helper() + var source, declarations, table strings.Builder + sigs := make(map[string]FuncSig) + for index, row := range cases { + name := fmt.Sprintf("average%d", index) + fmt.Fprintf(&source, `TEXT %s(SB),4,$0-40 +MOVQ out+0(FP),AX +MOVQ a+8(FP),BX +MOVQ b+16(FP),CX +MOVQ mask+24(FP),DX +VMOVUPS (CX),Z0 +VMOVUPS (CX),Z2 +KMOVQ DX,K1 +`, name) + if !row.memory || row.destination == 1 { + source.WriteString("VMOVUPS (BX),Z1\n") + } + if row.mmx { + source.WriteString("MOVQ (CX),M0\nMOVQ (BX),M1\nVMOVUPS Z0,(AX)\n") + } + source.WriteString("PUSHQ flags+32(FP)\nPOPFQ\n" + row.instruction() + "\n") + if row.mmx { + source.WriteString("MOVQ M0,(AX)\nEMMS\n") + } else { + fmt.Fprintf(&source, "VMOVUPS Z%d,(AX)\n", row.destination) + } + source.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void*,const void*,const void*,uint64_t,uint64_t);\n", name) + fmt.Fprintf(&table, "{%s,%d,%d,%d,%d,%d},\n", name, row.width, row.bits/8, + row.masking, row.destination, x86FloatAddSubBoolInt(!strings.HasPrefix(row.op, "V"))) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{TargetTriple: triple, Goarch: "amd64", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + driver := declarations.String() + "static const struct {Kernel run;int width,bytes,masking,destination,legacy;} cases[]={\n" + table.String() + "};\n" + return ir, driver +} + +// Byte arithmetic is independent of the lowering specification. W lanes use +// explicit little-endian assembly and a 17-bit sum, including 65535+65535. +const x86AverageIndependentC = `#include +#include +#include +#include +typedef void (*Kernel)(void *, const void *, const void *, uint64_t, uint64_t); +static unsigned lane(const uint8_t *p, int bytes) { + unsigned value = p[0]; + if (bytes == 2) { + value |= (unsigned)p[1] << 8; + } + return value; +} +static const uint64_t masks[] = {0, 1, UINT64_C(0xaaaaaaaa55555555), UINT64_MAX}; +` + +const x86AverageCheckC = `int main(void) { + for (unsigned row = 0; row < sizeof(cases) / sizeof(cases[0]); row++) { + for (int sample = 0; sample < 257; sample++) { + for (int status = 0; status < 2; status++) { + for (int m = 0; m < 4; m++) { + _Alignas(64) uint8_t a[192], b[192], out[192]; + memset(a, 0xa7, sizeof(a)); + memset(b, 0xb9, sizeof(b)); + memset(out, 0xc3, sizeof(out)); + + for (int i = 0; i < 64; i++) { + a[64 + i] = (uint8_t)(sample * 71 + i * 37); + b[64 + i] = (uint8_t)(sample * 19 - i * 13); + if (sample < 4) { + a[64 + i] = (uint8_t[]){255, 0, 128, 255}[sample]; + b[64 + i] = (uint8_t[]){255, 1, 128, 0}[sample]; + } + } + uint8_t beforeA[192], beforeB[192]; + memcpy(beforeA, a, sizeof(a)); + memcpy(beforeB, b, sizeof(b)); + + uint32_t saved = _mm_getcsr(), + profile = (uint32_t)(0x1f80 | ((sample % 4) << 13) | (sample % 64)); + _mm_setcsr(profile); + uint64_t seed = status ? UINT64_C(0x8d5) : 0; + cases[row].run(out + 64, a + 64, b + 64, masks[m], seed); + uint32_t after = _mm_getcsr(); + _mm_setcsr(saved); + uint64_t flags; + memcpy(&flags, out + 128, 8); + if (after != profile || (flags & UINT64_C(0x8d5)) != seed || + memcmp(a, beforeA, sizeof(a)) || memcmp(b, beforeB, sizeof(b))) { + fprintf(stderr, "row%u sample%d CSR/flags/input\n", row, sample); + return 1; + } + + for (int i = 0; i < 64; i++) { + int start = i / cases[row].bytes * cases[row].bytes; + unsigned value = (lane(a + 64 + start, cases[row].bytes) + + lane(b + 64 + start, cases[row].bytes) + 1) / + 2; + int active = + !cases[row].masking || ((masks[m] >> (i / cases[row].bytes)) & 1); + uint8_t old = cases[row].destination == 1 ? a[64 + i] : b[64 + i]; + uint8_t want = + i < cases[row].width + ? (active ? (uint8_t)(value >> (8 * (i % cases[row].bytes))) + : cases[row].masking == 2 ? 0 + : old) + : cases[row].legacy ? b[64 + i] + : 0; + if (out[64 + i] != want) { + fprintf(stderr, "row%u sample%d mask%llx byte%d got%x want%x\n", row, + sample, (unsigned long long)masks[m], i, out[64 + i], want); + return 2; + } + } + for (int i = 0; i < 192; i++) { + if ((i < 64 || i >= 136) && out[i] != 0xc3) { + return 3; + } + } + } + } + } + } + puts("complete average bytes/flags/MXCSR/input/output canaries"); + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedAverageIndependentBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + cases := x86AverageNumericCases() + ir, table := x86AverageNumericProgram(t, execution.triple, cases) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "average-independent", ir, + x86AverageIndependentC+table+x86AverageCheckC, level) + if !strings.Contains(string(output), "complete average bytes") { + t.Fatal("missing full numerical receipt", string(output)) + } + }) + } + t.Logf("%d programs × 257 patterns × 2 flags × 4 masks, full 64-byte views, O0/O2; translated wide shadow views, not original EVEX runtime", len(cases)) +} + +func TestCrossLinuxRuntimeMatrixX86PackedAverageInactiveMemory(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var cases []x86AverageNumericCase + for _, op := range x86AverageNames[2:] { + bits := 8 + if strings.HasSuffix(op, "W") { + bits = 16 + } + for _, width := range []int{16, 32, 64} { + for _, mode := range []int{1, 2} { + cases = append(cases, x86AverageNumericCase{op, width, bits, mode, 0, true, false}) + } + } + } + ir, table := x86AverageNumericProgram(t, execution.triple, cases) + // Unlike a post-load select, an E4.nb masked read cannot touch the guard + // page. With mask 1 only the first byte/word is accessible; mask 0 has no + // accessible source bytes. Full virtual FP spans are tested separately. + const main = `#include +#include +#include +int main(void) { + struct rlimit core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &core)) { + return 9; + } + long size = sysconf(_SC_PAGESIZE); + if (size <= 0) { + return 10; + } + uint8_t *page = + mmap(0, (size_t)size * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + if (page == MAP_FAILED || mprotect(page + size, (size_t)size, PROT_NONE)) { + return 11; + } + + _Alignas(64) uint8_t b[64], out[80]; + memset(b, 0x91, 64); + for (unsigned row = 0; row < sizeof(cases) / sizeof(cases[0]); row++) { + for (int active = 0; active < 2; active++) { + memset(out, 0xc3, 80); + uint8_t *a = page + size - (active ? cases[row].bytes : 0); + if (active) { + memset(a, 0xff, (size_t)cases[row].bytes); + } + cases[row].run(out + 1, a, b, (uint64_t)active, 0); + for (int i = 0; i < 64; i++) { + unsigned value = (cases[row].bytes == 1 ? 255 : 65535) + + (cases[row].bytes == 1 ? 0x91 : 0x9191) + 1; + value /= 2; + uint8_t want = i < cases[row].width + ? (active && i < cases[row].bytes + ? (uint8_t)(value >> (8 * (i % cases[row].bytes))) + : cases[row].masking == 2 ? 0 + : b[i]) + : 0; + if (out[i + 1] != want) { + return 12; + } + } + for (int i = 0; i < 80; i++) { + if ((i == 0 || i >= 73) && out[i] != 0xc3) { + return 13; + } + } + } + } + if (munmap(page, (size_t)size * 2)) { + return 14; + } + puts("inactive guard-page reads suppressed"); + return 0; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "average-inactive", ir, x86AverageIndependentC+table+main, level) + if !strings.Contains(string(output), "inactive guard-page reads suppressed") { + t.Fatal("missing masked memory receipt", string(output)) + } + }) + } +} diff --git a/amd64_packed_conversion_effects.go b/amd64_packed_conversion_effects.go new file mode 100644 index 000000000..a7fcd9b6e --- /dev/null +++ b/amd64_packed_conversion_effects.go @@ -0,0 +1,103 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func x86TypedPackedConversionByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + if _, known := x86PackedConversionSpecification(Op(base)); !known { + return x86TypedByteEffect{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(op), string(ins.Op)) { + return x86TypedByteEffect{}, true, fmt.Errorf("packed conversion effect/source mismatch") + } + ins.Op = op + f, known, err := x86PackedConversionProperties(ins, arch) + if err != nil { + return x86TypedByteEffect{}, known, err + } + width := f.inputBytes + if f.memory >= 0 { + width = f.memoryBytes + } + effect := x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: int64(width), read: true}, + {index: f.destination, operand: ins.Args[f.destination], width: int64(f.shape.destinationBytes), write: true}, + }} + if f.mask >= 0 { + effect.operands = append(effect.operands, x86TypedByteOperand{index: f.mask, operand: ins.Args[f.mask], width: 8, read: true}) + if !f.properties.zeroing { + effect.operands = append(effect.operands, x86TypedByteOperand{index: f.destination, operand: ins.Args[f.destination], + width: int64(f.shape.lanes * f.spec.destinationBits / 8), read: true}) + } + } + return effect, true, nil // Integer FLAGS do not change; MXCSR remains native. +} + +func x86ResultPackedConversionStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + f, known, err := x86PackedConversionProperties(ins, arch) + if !known { + return false, false + } + if err != nil || f.mmx { + return false, true + } + if f.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[f.mask].Reg) + if !valid || !state.masks[index] { + return false, true + } + } + if f.memory >= 0 { + if !x86ResultRead(ins.Args[0], int64(f.memoryBytes), state, owned, arch) { + return false, true + } + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[0].Reg, f.shape.sourceBytes) + input := x86CallerVectorByteMask(int64(f.inputBytes)) + if state.vectorBytes(index)&input != input { + return false, true + } + } + index, _ := amd64VectorRegisterIndex(ins.Args[f.destination].Reg, f.shape.destinationBytes) + active := x86CallerVectorByteMask(int64(f.shape.lanes * f.spec.destinationBits / 8)) + if f.mask >= 0 && !f.properties.zeroing && state.vectorBytes(index)&active != active { + return false, true + } + state.writeVector(index, f.shape.destinationBytes, x86CallerVectorByteMask(int64(f.shape.destinationBytes)), f.spec.legacy) + return true, true +} + +func x86CallerPackedConversionStep(ins *Instr, f x86PackedConversionForm, state *x86SourceStackState, proof *x86GoCallerRegionProof) bool { + if f.mmx || f.mask >= 0 && !state.masks[ins.Args[f.mask].Reg] { + return false + } + if f.memory >= 0 { + source := ins.Args[0] + if source.Kind == OpFP || source.Kind == OpMem && + (source.Mem.Base == SP || source.Mem.Base == BP || source.Mem.Index == BP) { + return false + } + if !x86CallerOrdinaryMemoryStep(ins, f.memory, state, proof, int64(f.memoryBytes), false) { + return false + } + } else { + input := x86CallerVectorByteMask(int64(f.inputBytes)) + if state.vectors[x86CallerVectorRoot(ins.Args[0].Reg)]&input != input { + return false + } + } + destination := x86CallerVectorRoot(ins.Args[f.destination].Reg) + active := x86CallerVectorByteMask(int64(f.shape.lanes * f.spec.destinationBits / 8)) + if f.mask >= 0 && !f.properties.zeroing && state.vectors[destination]&active != active { + return false + } + if f.spec.legacy { + state.vectors[destination] |= x86CallerVectorByteMask(int64(f.shape.destinationBytes)) + } else { + state.vectors[destination] = ^uint64(0) + } + return true +} diff --git a/amd64_packed_conversion_effects_test.go b/amd64_packed_conversion_effects_test.go new file mode 100644 index 000000000..51bac27b8 --- /dev/null +++ b/amd64_packed_conversion_effects_test.go @@ -0,0 +1,234 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86PackedConversionIndependentEveryByteEffects(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, spec := range x86PackedConversionOracleSpecs() { + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, masking := range []string{"", "merge", "zero"} { + if spec.legacy && masking != "" { + continue + } + op, mask := spec.name, "" + if masking != "" { + mask = "K1," + } + if masking == "zero" { + op += ".Z" + } + ins := x86ScalarLaneInstruction(t, op+" "+bank[shape.source]+"0,"+mask+bank[shape.destination]+"1") + f, known, err := x86PackedConversionProperties(ins, arch) + if err != nil || !known || f.inputBytes != shape.lanes*spec.input/8 || f.shape.destinationBytes != shape.destination { + t.Fatal(ins.Raw, f, err) + } + for missing := -1; missing < 64; missing++ { + for _, missingDestination := range []bool{false, true} { + state := x86ResultState{flags: x86ByteFlagsAll} + state.masks[1] = true + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + index := 0 + if missingDestination { + index = 1 + } + if missing >= 0 { + state.writeVector(index, 64, ^(uint64(1) << uint(missing)), false) + } + want := missing < 0 || !missingDestination && missing >= shape.lanes*spec.input/8 || + missingDestination && (masking != "merge" || missing >= shape.lanes*spec.output/8) + accepted, recognized := x86ResultPackedConversionStep(ins, &state, nil, arch) + if !recognized || accepted != want { + t.Fatal("required byte mismatch", ins.Raw, arch, missing, missingDestination, accepted, want) + } + if accepted { + expected := ^uint64(0) + if spec.legacy && missingDestination && missing >= 16 { + expected &^= uint64(1) << uint(missing) + } + if state.vectorBytes(1) != expected || state.flags != x86ByteFlagsAll { + t.Fatal("upper alias or integer FLAGS changed", ins.Raw, state.vectorBytes(1), expected) + } + } + } + } + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if err != nil || !known || effect.flagsRead != 0 || effect.flagsKilled != 0 || effect.flagsDefined != 0 { + t.Fatal("typed family effect/FLAGS disagreement", ins.Raw, effect, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0), "Z1": ^uint64(0)}, masks: map[Reg]bool{"K1": true}} + if !x86CallerPackedConversionStep(&ins, f, &state, nil) { + t.Fatal("initialized register grammar missing caller effect", ins.Raw) + } + } + } + } + } +} + +func TestX86PackedConversionResultAndStorageBoundaries(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + shape := spec.shapes()[0] + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, initialized := range []bool{false, true} { + setup := "" + if initialized { + setup = "XORPS X0,X0\n" + if !spec.legacy { + setup = "VPXORD Z0,Z0,Z0\n" + } + } + assembly := fmt.Sprintf("TEXT ·f(SB),4,$0-4\n%s%s %s0,%s1\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n", setup, spec.name, bank[shape.source], bank[shape.destination]) + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, assembly, "func f() uint32", "amd64", "x86_64-unknown-linux-gnu", api) + if initialized && err != nil || !initialized && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("uninitialized conversion result boundary", spec.name, initialized, api, err) + } + } + } + for _, operand := range []string{"data+0(FP)", "local-16(SP)", "(SP)", "(BP)"} { + assembly := "TEXT ·f(SB),4,$64-64\n" + spec.name + " " + operand + "," + bank[shape.destination] + "1\nRET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": { + Name: "f", Ret: Void, Args: []LLVMType{I64, I64, I64, I64, I64, I64, I64, I64}, + }}}) + if err == nil { + t.Fatal("logical slots acquired packed native backing", spec.name, operand) + } + } + } +} + +func TestX86PackedConversionPhysicalBroadcastTuple(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + if spec.legacy { + continue + } + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[shape.destination] + row := spec.name + ".BCST (AX)," + bank + "1" + ir, err := x86ScalarFPBitViewAPI(t, "TEXT ·f(SB),4,$0-0\n"+row+"\nRET\n", "func f()", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(row, err) + } + if !strings.Contains(ir, fmt.Sprintf("{1to%d}", shape.lanes)) || !strings.Contains(ir, fmt.Sprintf("elementtype([%d x i8])", spec.input/8)) { + t.Fatal("physical broadcast confused carrier with active lanes", row, ir) + } + } + } +} + +func TestX86PackedConversionMalformedTypedGrammar(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + shape := spec.shapes()[0] + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + source, destination := bank[shape.source]+"0", bank[shape.destination]+"1" + invalid := []string{ + spec.name + " " + source, + spec.name + " $1," + destination, + spec.name + " " + source + ",(AX)", + spec.name + " " + source + ",K0," + destination, + spec.name + " " + source + ",K8," + destination, + spec.name + ".Z " + source + "," + destination, + spec.name + ".BCST " + source + "," + destination, + spec.name + ".RN_SAE (AX)," + destination, + spec.name + ".SAE (AX)," + destination, + spec.name + ".RN_SAE.SAE " + source + "," + destination, + spec.name + " " + source + ",Y32", + } + if spec.legacy { + invalid = append(invalid, spec.name+".BCST (AX),"+destination, + spec.name+" "+source+",K1,"+destination) + } + for _, row := range invalid { + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86PackedConversionProperties(ins, "amd64"); !known || err == nil { + t.Fatal("malformed complete-family form accepted", row, err) + } + } + for _, position := range []int{0, 1} { + ins := x86ScalarLaneInstruction(t, spec.name+" "+source+","+destination) + ins.Args[position].Reg = Reg(bank[amd64VectorByteWidth(ins.Args[position].Reg)] + "8") + if _, known, err := x86PackedConversionProperties(ins, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("mode32 invented physical vector register", ins, err) + } + } + } + context := &amd64Ctx{goarch: "amd64"} + if known, _, err := context.lowerPackedNumericConversion("UNKNOWN", Instr{Op: "OTHER"}); known || err != nil { + t.Fatal("conversion lowerer claimed another instruction family", known, err) + } + valid := x86ScalarLaneInstruction(t, "CVTPS2PL X0,X1") + if known, _, err := context.lowerPackedNumericConversion("OTHER", valid); known || err != nil { + t.Fatal("unknown dispatch claimed a packed source spelling", known, err) + } + if known, _, err := context.lowerPackedNumericConversion("CVTPS2PL", Instr{Op: "OTHER"}); !known || err == nil { + t.Fatal("known dispatch lost its mismatched source identity", known, err) + } +} + +func TestX86PackedConversionCompleteCallerConsumer(t *testing.T) { + var originals strings.Builder + count := 0 + for _, spec := range x86PackedConversionOracleSpecs() { + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + row := spec.name + " " + bank[shape.source] + "0," + bank[shape.destination] + "1" + body := "VMOVUPS (DI),Z0\n" + row + "\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + originals.WriteString(strings.Replace(assembly, "TEXT ·kernel", fmt.Sprintf("TEXT ·kernel%d", count), 1)) + count++ + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal("complete initialized conversion caller", row, err) + } + for _, operand := range []string{"(BX)", "(DX)", "(SP)", "(BP)", "a+0(FP)"} { + body := spec.name + " " + operand + "," + bank[shape.destination] + "1\nVMOVSS X1,(DX)" + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body, 1) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("conversion acquired unproved address/envelope/backing", row, operand, err) + } + } + } + } + goAssemblerOracleOutput(t, "amd64", originals.String(), "", true) + t.Logf("%d complete caller shapes, five independent address/backing negatives each", count) +} + +func TestX86PackedConversionWholeFileSyntaxBeforeContext(t *testing.T) { + for _, context := range []struct{ name, arch, triple, instruction string }{ + {"target", "amd64", "", "CVTPS2PL X0,X1"}, + {"foreign target", "amd64", "aarch64-unknown-linux-gnu", "CVTPS2PL X0,X1"}, + {"physical MMX", "amd64", "x86_64-unknown-linux-gnu", "CVTPS2PL X0,M0"}, + {"physical mode32", "386", "i386-unknown-linux-gnu", "VCVTPS2DQ X8,X0"}, + } { + for _, api := range []string{"text", "module", "Go"} { + source := "TEXT ·f(SB),4,$0-0\n" + context.instruction + "\nRET\n" + _, err := x86ScalarFPBitViewAPI(t, source, "func f()", context.arch, context.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("original physical/target boundary lost", context.name, api, err) + } + for _, malformed := range []string{"VCVTPS2DQ.Z X0,X1", "CVTPS2PL X0,(AX)"} { + for _, suffix := range []struct{ assembly, declaration string }{ + {malformed + "\n", "func f()"}, + {"TEXT ·g(SB),4,$0-0\nRET\n" + malformed + "\n", "func f();func g()"}, + } { + _, err := x86ScalarFPBitViewAPI(t, source+suffix.assembly, suffix.declaration, context.arch, context.triple, api) + if err == nil || errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "packed numeric conversion grammar") { + t.Fatal("earlier Context hid dead malformed source", context.name, api, malformed, err) + } + } + } + } + } +} diff --git a/amd64_packed_conversion_fault_runtime_test.go b/amd64_packed_conversion_fault_runtime_test.go new file mode 100644 index 000000000..2679dbd81 --- /dev/null +++ b/amd64_packed_conversion_fault_runtime_test.go @@ -0,0 +1,247 @@ +//go:build unix + +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +type x86PackedConversionFaultRow struct { + name string + width int + laneBytes int + masked bool + floating bool + precision bool + aligned bool + conversion string +} + +func x86PackedConversionFaultRows(wide bool) []x86PackedConversionFaultRow { + var rows []x86PackedConversionFaultRow + for _, spec := range x86PackedConversionOracleSpecs() { + if !wide && !spec.legacy { + continue + } + for _, shape := range spec.shapes() { + destination := map[int]string{16: "X1", 32: "Y1", 64: "Z1"}[shape.destination] + floating := strings.Contains(strings.Split(spec.name, "2")[0], "PS") || strings.Contains(strings.Split(spec.name, "2")[0], "PD") + precision := floating || spec.input > map[int]int{32: 24, 64: 53}[spec.output] + rows = append(rows, x86PackedConversionFaultRow{name: spec.name, width: shape.lanes * spec.input / 8, laneBytes: spec.input / 8, + floating: floating, precision: precision, aligned: spec.legacy, conversion: spec.name + " (DI)," + destination}) + if !spec.legacy { + rows = append(rows, + x86PackedConversionFaultRow{name: spec.name + ".Z", width: shape.lanes * spec.input / 8, laneBytes: spec.input / 8, masked: true, + floating: floating, precision: precision, conversion: spec.name + ".Z (DI),K1," + destination}, + x86PackedConversionFaultRow{name: spec.name + ".BCST.Z", width: spec.input / 8, laneBytes: spec.input / 8, masked: true, + floating: floating, precision: precision, conversion: spec.name + ".BCST.Z (DI),K1," + destination}) + } + } + } + return rows +} + +// One actual Go TEXT holds every independent straight-line kernel. Split at +// decoded RET boundaries, never a byte search (0xc3 can occur in displacement). +// This creates no production native/source authorization. +func x86PackedConversionFaultOriginalBytes(t *testing.T, rows []x86PackedConversionFaultRow) [][]byte { + t.Helper() + var source strings.Builder + source.WriteString("TEXT original(SB),4,$0-0\n") + for _, row := range rows { + source.WriteString("LDMXCSR (SI)\n") + if row.masked { + source.WriteString("KMOVW DX,K1\n") + } + source.WriteString(row.conversion + "\nRET\n") + } + code := x86PackedShiftActualGoBytes(t, "amd64", source.String()) + var result [][]byte + offset := 0 + for _, row := range rows { + start := offset + header := []byte{0x0f, 0xae, 0x16} // Actual Go LDMXCSR (SI). + if row.masked { + header = append(header, 0xc5, 0xf8, 0x92, 0xca) // KMOVW DX,K1. + } + if len(code)-offset < len(header) || !bytes.Equal(code[offset:offset+len(header)], header) { + t.Fatal("actual original header changed", row.name, offset) + } + offset += len(header) + _, length, known, err := decodedX86PackedNumericConversionInstruction(code[offset:], 64) + if !known { + native, decodeErr := x86asm.Decode(code[offset:], 64) + err, length = decodeErr, native.Len + } + if err != nil || length <= 0 || offset+length >= len(code) || code[offset+length] != 0xc3 { + t.Fatal("whole original conversion/RET boundary", row.name, offset, err) + } + offset += length + 1 + result = append(result, append([]byte(nil), code[start:offset]...)) + } + if offset != len(code) { + t.Fatal("unconsumed original Go instruction bytes") + } + return result +} + +func x86PackedConversionDeadFaults(t *testing.T, wide, nativeTraps bool) { + t.Helper() + axis := x86FloatCompareSSE + if wide || nativeTraps { + axis = x86FloatCompareEVEX + } + execution := x86FloatCompareRuntimeTools(t, axis) + rows := x86PackedConversionFaultRows(wide) + const limit = 96 + for start := 0; start < len(rows); start += limit { + end := start + limit + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + batch := rows[start:end] + original := x86PackedConversionFaultOriginalBytes(t, batch) + var source, decl, native, prototypes, pairs strings.Builder + for index, row := range batch { + name, reference := fmt.Sprintf("fault%d", index), fmt.Sprintf("original%d", index) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nMOVQ p+0(FP),DI\nMOVQ csr+8(FP),SI\nLDMXCSR (SI)\n", name) + if row.masked { + source.WriteString("MOVL mask+16(FP),DX\nKMOVW DX,K1\n") + } + source.WriteString(row.conversion + "\nRET\n") + fmt.Fprintf(&decl, "func %s(p *uint64,csr *uint32,mask uint32)\n", name) + var octets []string + for _, value := range original[index] { + octets = append(octets, fmt.Sprintf("0x%02x", value)) + } + symbol := reference + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(octets, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint64_t*,uint32_t*,uint32_t),%s(uint64_t*,uint32_t*,uint32_t);\n", name, reference) + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d,%d,%d,%d},\n", reference, name, row.width, row.laneBytes, + x86FloatAddSubBoolInt(row.masked), x86FloatAddSubBoolInt(row.floating), + x86FloatAddSubBoolInt(row.precision), x86FloatAddSubBoolInt(row.aligned)) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf("#define NATIVE_TRAPS %d\n", x86FloatAddSubBoolInt(nativeTraps)) + `#include +#include +#include +#include +#include +#include +#include +#include +` + native.String() + prototypes.String() + ` +typedef void (*program)(uint64_t*,uint32_t*,uint32_t); +static const struct {program original,translated;int width,lane,masked,floating,precision,aligned;} pairs[]={ +` + pairs.String() + `}; +static int outcome(program fn,uint8_t *p,uint32_t csr,uint32_t mask) { + pid_t child=fork();if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0};if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn((uint64_t*)p,&csr,mask);_exit(0); + } + int status;if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status)&&WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status)) {int sig=WTERMSIG(status);if(sig==SIGSEGV||sig==SIGBUS)return 1;if(sig==SIGFPE)return 2;} + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *p=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(p==MAP_FAILED||mprotect(p+page,page,PROT_NONE))return 2; + memset(p,0,page); + for(unsigned i=0;i len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + original := x86PackedConversionFaultOriginalBytes(t, rows[start:end]) + if len(original) != end-start { + t.Fatal("original native fault fixture lost a conversion") + } + consumed += len(original) + }) + } + if consumed != len(rows) { + t.Fatal("native fault batches overlap or omit rows") + } + t.Logf("%d original-Go complete memory/masked/broadcast fault kernels; assembly only, not native execution", consumed) +} diff --git a/amd64_packed_conversion_forms.go b/amd64_packed_conversion_forms.go new file mode 100644 index 000000000..015a44929 --- /dev/null +++ b/amd64_packed_conversion_forms.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" +) + +// Element types, active lanes and physical registers are separate axes. In +// particular X <- X can read 8 or 16 bytes and can write only 8 numeric bytes +// followed by eight architectural zero bytes. +type x86PackedConversionSpec struct { + amd64PackedNumericConversionSpec + legacy, vex bool +} + +func x86PackedConversionSpecification(op Op) (x86PackedConversionSpec, bool) { + if spec, ok := amd64PackedNumericConversionSpecs[op]; ok { + return x86PackedConversionSpec{amd64PackedNumericConversionSpec: spec}, true + } + base := string(op) + legacy := !strings.HasPrefix(base, "V") + spec := amd64PackedNumericConversionSpec{ + sourceKind: amd64PackedNumericFloat, destinationKind: amd64PackedNumericSigned, + sourceBits: 32, destinationBits: 32, truncate: strings.HasPrefix(base, "CVTT") || strings.HasPrefix(base, "VCVTT"), + } + switch base { + case "CVTPS2PL", "CVTTPS2PL": + spec.shapes = amd64PackedNumericSame32[:1] + case "VCVTPS2DQ", "VCVTTPS2DQ": + spec.shapes, spec.explicit = amd64PackedNumericSame32, "round" + case "CVTPD2PL", "CVTTPD2PL", "VCVTPD2DQX", "VCVTTPD2DQX": + spec.sourceBits = 64 + spec.shapes = []amd64PackedNumericShape{{sourceBytes: 16, destinationBytes: 16, lanes: 2}} + case "VCVTPD2DQY", "VCVTTPD2DQY": + spec.sourceBits = 64 + spec.shapes = []amd64PackedNumericShape{{sourceBytes: 32, destinationBytes: 16, lanes: 4}} + case "VCVTPD2DQ", "VCVTTPD2DQ": + spec.sourceBits, spec.explicit = 64, "round" + spec.shapes = []amd64PackedNumericShape{{sourceBytes: 64, destinationBytes: 32, lanes: 8}} + default: + return x86PackedConversionSpec{}, false + } + if spec.explicit != "" && spec.truncate { + spec.explicit = "sae" + } + return x86PackedConversionSpec{amd64PackedNumericConversionSpec: spec, legacy: legacy, vex: !legacy}, true +} + +type x86PackedConversionForm struct { + spec x86PackedConversionSpec + shape amd64PackedNumericShape + properties amd64BinaryFloatingSuffix + inputBytes, memoryBytes int + destination, memory, mask int + mmx, evex bool +} + +// This grammar supplies widths, not a native frame, caller, pointee or MMX/x87 +// certificate. Memory consumers must independently retain those boundaries. +func x86PackedConversionProperties(ins Instr, arch string) (x86PackedConversionForm, bool, error) { + base, suffix, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86PackedConversionSpecification(Op(base)) + f := x86PackedConversionForm{spec: spec, destination: len(ins.Args) - 1, memory: -1, mask: -1} + if !known { + return f, false, nil + } + fail := func() (x86PackedConversionForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside packed numeric conversion grammar: %q", arch, ins.Op, ins.Raw) + } + var valid bool + f.properties, valid = parseAMD64BinaryFloatingSuffix(suffix) + if !valid || arch != "amd64" && arch != "386" || len(ins.Args) != 2 && len(ins.Args) != 3 || + spec.legacy && (suffix != "" || len(ins.Args) != 2) || + f.properties.rounding != "" && spec.explicit != "round" || f.properties.sae && spec.explicit != "sae" { + return fail() + } + if len(ins.Args) == 3 { + if !amd64NonzeroKOperand(ins.Args[1]) { + return fail() + } + f.mask = 1 + } else if f.properties.zeroing { + return fail() + } + destination := ins.Args[f.destination] + _, f.mmx = amd64ParseMReg(destination.Reg) + if destination.Kind != OpReg || f.mmx && !spec.legacy { + return fail() + } + width := amd64VectorByteWidth(destination.Reg) + if f.mmx { + f.shape = amd64PackedNumericShape{sourceBytes: 16, destinationBytes: 8, lanes: 2} + } else { + for _, shape := range spec.shapes { + if shape.destinationBytes == width { + f.shape = shape + break + } + } + if f.shape.lanes == 0 || !amd64EVEXVectorRegister(destination, width) || + spec.legacy && !amd64VEXVectorRegister(destination, 16) { + return fail() + } + } + f.inputBytes = f.shape.lanes * spec.sourceBits / 8 + source := ins.Args[0] + if source.Kind == OpReg { + if f.properties.broadcast || !amd64EVEXVectorRegister(source, f.shape.sourceBytes) || + spec.legacy && !amd64VEXVectorRegister(source, 16) { + return fail() + } + } else if isAMD64MemoryOperand(source) && x86SignedPackedFloatMemoryOperand(source, arch) { + f.memory, f.memoryBytes = 0, f.inputBytes + if f.properties.broadcast { + f.memoryBytes = spec.sourceBits / 8 + } + } else { + return fail() + } + if (f.properties.rounding != "" || f.properties.sae) && + (source.Kind != OpReg || f.shape != spec.shapes[len(spec.shapes)-1]) { + return fail() + } + f.evex = !spec.legacy && (!spec.vex || ins.x86PackedNumericEVEX || f.mask >= 0 || + f.properties.broadcast || f.properties.rounding != "" || f.properties.sae || + f.shape.sourceBytes == 64 || f.shape.destinationBytes == 64) + for index, operand := range []Operand{source, destination} { + if operand.Kind != OpReg || index == 1 && f.mmx { + continue + } + index, _ := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + if arch == "386" && index >= 8 { + return f, true, fmt.Errorf("%w: %s register %s has no physical mode32 binding", ErrProbeNeedsContext, ins.Op, operand.Reg) + } + f.evex = f.evex || index >= 16 + } + return f, true, nil +} + +func validateX86PackedConversionFile(file *File, opt Options) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + var pending error + deferContext := func(err error) { + if pending == nil { + pending = err + } + } + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + form, known, err := x86PackedConversionProperties(ins, opt.Goarch) + if !known { + continue + } + if err != nil { + // Physical mode32 binding is not a grammar error. Retain its + // refusal, but still validate every later/dead source format. + if errors.Is(err, ErrProbeNeedsContext) { + deferContext(fmt.Errorf("%s: %w", fn.Sym, err)) + continue + } + return fmt.Errorf("%s: %w", fn.Sym, err) + } + if form.mmx { + deferContext(fmt.Errorf("%w: %s has no complete physical MMX/x87 caller contract", ErrProbeNeedsContext, ins.Op)) + } + if !x86ScalarFloatIntegerNativeTarget(opt.Goarch, opt.TargetTriple) { + deferContext(fmt.Errorf("%w: %s needs its matching native x86 MXCSR target", ErrProbeNeedsContext, ins.Op)) + } + } + } + return pending +} diff --git a/amd64_packed_conversion_matrix_test.go b/amd64_packed_conversion_matrix_test.go new file mode 100644 index 000000000..a5b1a4f64 --- /dev/null +++ b/amd64_packed_conversion_matrix_test.go @@ -0,0 +1,325 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +// Independent Go/Intel encoding inventory. No expected class, tuple, opcode +// or semantic width is obtained from the production conversion tables. +type x86PackedConversionOracle struct { + name, shape, control, pp, opcode string + input, output, w int + legacy, vex bool +} + +func x86PackedConversionOracleSpecs() []x86PackedConversionOracle { + return []x86PackedConversionOracle{ + {"CVTPS2PL", "single", "", "Pe", "0x5b", 32, 32, 0, true, false}, + {"CVTTPS2PL", "single", "", "Pf3", "0x5b", 32, 32, 0, true, false}, + {"CVTPD2PL", "narrow16", "", "Pf2", "0xe6", 64, 32, 0, true, false}, + {"CVTTPD2PL", "narrow16", "", "Pe", "0xe6", 64, 32, 0, true, false}, + {"VCVTPS2DQ", "same", "round", "66", "0x5B", 32, 32, 0, false, true}, + {"VCVTTPS2DQ", "same", "sae", "F3", "0x5B", 32, 32, 0, false, true}, + {"VCVTPD2DQX", "narrow16", "", "F2", "0xE6", 64, 32, 1, false, true}, + {"VCVTPD2DQY", "narrow32", "", "F2", "0xE6", 64, 32, 1, false, true}, + {"VCVTPD2DQ", "narrow64", "round", "F2", "0xE6", 64, 32, 1, false, false}, + {"VCVTTPD2DQX", "narrow16", "", "66", "0xE6", 64, 32, 1, false, true}, + {"VCVTTPD2DQY", "narrow32", "", "66", "0xE6", 64, 32, 1, false, true}, + {"VCVTTPD2DQ", "narrow64", "sae", "66", "0xE6", 64, 32, 1, false, false}, + {"VCVTPD2QQ", "same", "round", "66", "0x7B", 64, 64, 1, false, false}, + {"VCVTPD2UQQ", "same", "round", "66", "0x79", 64, 64, 1, false, false}, + {"VCVTTPD2QQ", "same", "sae", "66", "0x7A", 64, 64, 1, false, false}, + {"VCVTTPD2UQQ", "same", "sae", "66", "0x78", 64, 64, 1, false, false}, + {"VCVTPS2UDQ", "same", "round", "", "0x79", 32, 32, 0, false, false}, + {"VCVTTPS2UDQ", "same", "sae", "", "0x78", 32, 32, 0, false, false}, + {"VCVTUDQ2PS", "same", "round", "F2", "0x7A", 32, 32, 0, false, false}, + {"VCVTQQ2PD", "same", "round", "F3", "0xE6", 64, 64, 1, false, false}, + {"VCVTUQQ2PD", "same", "round", "F3", "0x7A", 64, 64, 1, false, false}, + {"VCVTPS2QQ", "wide", "round", "66", "0x7B", 32, 64, 0, false, false}, + {"VCVTPS2UQQ", "wide", "round", "66", "0x79", 32, 64, 0, false, false}, + {"VCVTTPS2QQ", "wide", "sae", "66", "0x7A", 32, 64, 0, false, false}, + {"VCVTTPS2UQQ", "wide", "sae", "66", "0x78", 32, 64, 0, false, false}, + {"VCVTUDQ2PD", "wide", "", "F3", "0x7A", 32, 64, 0, false, false}, + {"VCVTPD2UDQX", "narrow16", "", "", "0x79", 64, 32, 1, false, false}, + {"VCVTPD2UDQY", "narrow32", "", "", "0x79", 64, 32, 1, false, false}, + {"VCVTPD2UDQ", "narrow64", "round", "", "0x79", 64, 32, 1, false, false}, + {"VCVTTPD2UDQX", "narrow16", "", "", "0x78", 64, 32, 1, false, false}, + {"VCVTTPD2UDQY", "narrow32", "", "", "0x78", 64, 32, 1, false, false}, + {"VCVTTPD2UDQ", "narrow64", "sae", "", "0x78", 64, 32, 1, false, false}, + {"VCVTQQ2PSX", "narrow16", "", "", "0x5B", 64, 32, 1, false, false}, + {"VCVTQQ2PSY", "narrow32", "", "", "0x5B", 64, 32, 1, false, false}, + {"VCVTQQ2PS", "narrow64", "round", "", "0x5B", 64, 32, 1, false, false}, + {"VCVTUQQ2PSX", "narrow16", "", "F2", "0x7A", 64, 32, 1, false, false}, + {"VCVTUQQ2PSY", "narrow32", "", "F2", "0x7A", 64, 32, 1, false, false}, + {"VCVTUQQ2PS", "narrow64", "round", "F2", "0x7A", 64, 32, 1, false, false}, + } +} + +type x86PackedOracleShape struct{ source, destination, lanes int } + +func (spec x86PackedConversionOracle) shapes() []x86PackedOracleShape { + var widths []int + switch spec.shape { + case "single", "narrow16": + widths = []int{16} + case "narrow32": + widths = []int{32} + case "narrow64": + widths = []int{64} + default: + widths = []int{16, 32, 64} + } + var result []x86PackedOracleShape + for _, width := range widths { + source, destination := width, width + lanes := width * 8 / spec.input + if spec.shape == "wide" { + source, lanes = width/2, width*8/spec.output + } else if strings.HasPrefix(spec.shape, "narrow") { + destination = width / 2 + } + if source < 16 { + source = 16 + } + if destination < 16 { + destination = 16 + } + result = append(result, x86PackedOracleShape{source, destination, lanes}) + } + return result +} + +func TestX86PackedConversionIndependentCompleteGoRows(t *testing.T) { + legacy, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(legacy), "var yxcvm1 = []ytab{\n\t{Zm_r_xm, 2, argList{Yxm, Yxr}},\n\t{Zm_r_xm, 2, argList{Yxm, Ymr}},\n}") { + t.Fatal("complete legacy operand table changed") + } + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, spec := range x86PackedConversionOracleSpecs() { + if spec.legacy { + mmxPrefix, mmxOpcode := "Pm", "0x2d" + if spec.input == 64 { + mmxPrefix = "Pe" + } + if strings.Contains(spec.name, "CVTT") { + mmxOpcode = "0x2c" + } + want := fmt.Sprintf("{A%s, yxcvm1, Px, opBytes{%s, %s, %s, %s}}", spec.name, spec.pp, spec.opcode, mmxPrefix, mmxOpcode) + if !strings.Contains(string(legacy), want) { + t.Fatal("legacy opcode attributes changed", want) + } + count += 2 + continue + } + shapes := spec.shapes() + table := "_yvcvtpd2qq" + switch { + case spec.shape == "same" && spec.vex: + table = "_yvcvtdq2ps" + case spec.shape == "wide": + table = "_yvcvtps2qq" + if spec.name == "VCVTUDQ2PD" { + table = "_yvcvtudq2pd" + } + case spec.shape == "narrow64": + table = "_yvcvtpd2dq" + case spec.shape == "narrow16", spec.shape == "narrow32": + table = "_yvcvtpd2udq" + strings.ToLower(spec.name[len(spec.name)-1:]) + if spec.vex { + table = strings.Replace(table, "udq", "dq", 1) + } + } + type row struct { + classes []string + zcase string + offset int + } + var rows []row + var opbytes []string + prefix := map[int]string{16: "x", 32: "y", 64: "z"} + appendShape := func(shape x86PackedOracleShape, evex bool) { + encoding, word, extra := "vex", 0, "" + if evex { + encoding, word, extra = "evex", spec.w, "Evex" + } + src := "Y" + prefix[shape.source] + "m" + extra + dst := "Y" + prefix[shape.destination] + "r" + extra + src, dst = strings.ReplaceAll(src, "zmEvex", "zm"), strings.ReplaceAll(dst, "zrEvex", "zr") + bits := shape.source * 8 + if spec.shape == "wide" { + bits = shape.destination * 8 + } + attributes := fmt.Sprintf("avxEscape | %s%d", encoding, bits) + if spec.pp != "" { + attributes += " | " + encoding + spec.pp + } + attributes += fmt.Sprintf(" | %s0F | %sW%d", encoding, encoding, word) + opbytes = append(opbytes, attributes) + if evex { + attributes = fmt.Sprintf("evexN%d | evexBcstN%d", shape.lanes*spec.input/8, spec.input/8) + if bits == 512 && spec.control != "" { + attributes += map[string]string{"round": " | evexRoundingEnabled", "sae": " | evexSaeEnabled"}[spec.control] + } + opbytes = append(opbytes, attributes+" | evexZeroingEnabled") + rows = append(rows, row{[]string{src, dst}, "Zevex_rm_v_r", 0}, row{[]string{src, "Yknot0", dst}, "Zevex_rm_k_r", 3}) + } else { + rows = append(rows, row{[]string{src, dst}, "Zvex_rm_v_r", 2}) + } + opbytes = append(opbytes, spec.opcode) + } + if spec.vex { + for _, shape := range shapes { + if shape.source < 64 && shape.destination < 64 { + appendShape(shape, false) + } + } + } + if len(shapes) == 3 && spec.name != "VCVTUDQ2PD" { + shapes = []x86PackedOracleShape{shapes[2], shapes[0], shapes[1]} + } + for _, shape := range shapes { + appendShape(shape, true) + } + index, offset := 0, 0 + for _, actual := range rules { + if actual.Opcode != spec.name { + continue + } + if index >= len(rows) { + t.Fatal("unexpected extra Go encoder row", spec.name) + } + want := rows[index] + if actual.Ytab != table || actual.Prefix != "Pavx" || actual.RowIndex != index || actual.EncodingOffset != offset || + actual.ZCase != want.zcase || actual.ZOffset != want.offset || !reflect.DeepEqual(actual.Classes, want.classes) || !reflect.DeepEqual(actual.OpBytes, opbytes) { + t.Fatalf("%s row%d changed: %+v\nwant %s %+v %v", spec.name, index, actual, table, want, opbytes) + } + index, offset, count = index+1, offset+want.offset, count+1 + } + if index != len(rows) { + t.Fatal("missing encoder rows", spec.name, index, len(rows)) + } + } + t.Logf("38 independent names, %d complete Go encoder rows", count) +} + +func x86PackedConversionOracleRows(arch string) []string { + var rows []string + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, spec := range x86PackedConversionOracleSpecs() { + for _, shape := range spec.shapes() { + destination := bank[shape.destination] + "7" + for _, source := range []string{bank[shape.source] + "0", "(AX)", "3(BX)(CX*4)"} { + rows = append(rows, spec.name+" "+source+","+destination) + if !spec.legacy { + rows = append(rows, spec.name+" "+source+",K1,"+destination, spec.name+".Z "+source+",K7,"+destination) + } + } + if !spec.legacy { + rows = append(rows, spec.name+".BCST (AX),"+destination, spec.name+".BCST.Z 3(BX)(CX*4),K7,"+destination) + if arch == "amd64" { + rows = append(rows, spec.name+" "+bank[shape.source]+"31,"+bank[shape.destination]+"31") + } + } + if (shape.source == 64 || shape.destination == 64) && spec.control != "" { + suffixes := []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} + if spec.control == "sae" { + suffixes = []string{"SAE"} + } + for _, suffix := range suffixes { + rows = append(rows, spec.name+"."+suffix+" "+bank[shape.source]+"0,"+destination, + spec.name+"."+suffix+".Z "+bank[shape.source]+"0,K7,"+destination) + } + } + } + } + return rows +} + +func TestX86PackedConversionFiveTargetsThreeAPIObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + rows := x86PackedConversionOracleRows(target.arch) + seen := make(map[string]bool) + const batchSize = 96 + for start := 0; start < len(rows); start += batchSize { + end := start + batchSize + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + var source, declarations strings.Builder + for index, row := range rows[start:end] { + if seen[row] { + t.Fatal("duplicate matrix row", row) + } + seen[row] = true + fmt.Fprintf(&source, "TEXT ·f%d(SB),4,$0-0\n%s\nRET\n", index, row) + fmt.Fprintf(&declarations, "func f%d()\n", index) + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(start, api, err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, input) + } + compileLLVMToObject(t, llc, target.triple, "packed-conversion.ll", "packed-conversion.o", input) + } + } + }) + } + if len(seen) != len(rows) { + t.Fatal("incomplete batch union") + } + t.Logf("%d unique forms, <=96 functions/module, 3 APIs x 2 IR pipelines", len(rows)) + }) + } +} + +func TestX86PackedConversionMMXAndTargetRemainContext(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + shape := spec.shapes()[0] + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + row := spec.name + " " + bank[shape.source] + "0," + bank[shape.destination] + "1" + for _, target := range []string{"", "aarch64-unknown-linux-gnu", "i386-unknown-linux-gnu"} { + _, err := x86ScalarFPBitViewAPI(t, "TEXT ·f(SB),4,$0-0\nRET\n"+row+"\n", "func f()", "amd64", target, "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("dead instruction acquired a foreign/missing target", row, target, err) + } + } + if spec.legacy { + for _, destination := range []string{"M0", "M7"} { + row := spec.name + " X0," + destination + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + _, err := x86ScalarFPBitViewAPI(t, "TEXT ·f(SB),4,$0-0\n"+row+"\nRET\n", "func f()", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("virtual MMX destination granted physical x87 state", row, err) + } + } + } + } +} diff --git a/amd64_packed_conversion_native_address_test.go b/amd64_packed_conversion_native_address_test.go new file mode 100644 index 000000000..fbe389307 --- /dev/null +++ b/amd64_packed_conversion_native_address_test.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// Independent named memory inventory from Go's legacy and AVX encoder rows. +// This address-role witness does not borrow the production spec or assert that +// a Context form was translated; the separate complete family matrix retains +// all widths, masks, rounding and raw encodings. +func x86PackedNativeAddressRows() []struct{ name, destination string } { + var rows []struct{ name, destination string } + for _, name := range []string{ + "CVTPS2PL", "CVTTPS2PL", "CVTPD2PL", "CVTTPD2PL", + "VCVTPS2DQ", "VCVTTPS2DQ", "VCVTPD2DQX", "VCVTPD2DQY", "VCVTTPD2DQX", "VCVTTPD2DQY", + "VCVTPD2QQ", "VCVTPD2UQQ", "VCVTTPD2QQ", "VCVTTPD2UQQ", + "VCVTPS2UDQ", "VCVTTPS2UDQ", "VCVTUDQ2PS", "VCVTQQ2PD", "VCVTUQQ2PD", + "VCVTPS2QQ", "VCVTPS2UQQ", "VCVTTPS2QQ", "VCVTTPS2UQQ", "VCVTUDQ2PD", + "VCVTPD2UDQX", "VCVTPD2UDQY", "VCVTTPD2UDQX", "VCVTTPD2UDQY", + "VCVTQQ2PSX", "VCVTQQ2PSY", "VCVTUQQ2PSX", "VCVTUQQ2PSY", + } { + rows = append(rows, struct{ name, destination string }{name, "X1"}) + } + for _, name := range []string{"VCVTPD2DQ", "VCVTTPD2DQ", "VCVTPD2UDQ", "VCVTTPD2UDQ", "VCVTQQ2PS", "VCVTUQQ2PS"} { + rows = append(rows, struct{ name, destination string }{name, "Y1"}) + } + return rows +} + +func TestX86PackedConversionOriginalNativeAddressRequired(t *testing.T) { + rows := x86PackedNativeAddressRows() + if len(rows) != 38 { + t.Fatal("complete native conversion memory-name inventory changed", len(rows)) + } + for _, arch := range []string{"386", "amd64"} { + var original strings.Builder + for index, row := range rows { + for _, address := range []string{"(SP)", "32(SP)", "(BP)", "-17(BP)", "32(SP)(CX*2)", "32(AX)(BP*2)"} { + fmt.Fprintf(&original, "TEXT original%d_%d(SB),4,$128-0\n%s %s,%s\nRET\n", index, original.Len(), row.name, address, row.destination) + } + } + goAssemblerOracleOutput(t, arch, original.String(), "", true) + } + for _, target := range x86RawReturnTargets { + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + for _, row := range rows { + for _, address := range []string{"(SP)", "32(SP)", "(BP)", "-17(BP)", "32(SP)(CX*2)", "32(AX)(BP*2)"} { + source := fmt.Sprintf("TEXT ·view(SB),4,$128-0\n%s %s,%s\nRET\n", row.name, address, row.destination) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view()", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("unbound physical SP/BP acquired synthetic-stack native memory: %s %s (%s), error=%v\n%s", row.name, address, row.destination, err, ir) + } + } + } + }) + } + } +} + +func TestX86PackedConversionOrdinaryNativeAddressControls(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + move, frame := "MOVQ", 8 + if target.arch == "386" { + move, frame = "MOVL", 4 + } + var source, declarations strings.Builder + for index, row := range x86PackedNativeAddressRows() { + for variant, address := range []string{"(AX)", "16(AX)(CX*2)"} { + name := fmt.Sprintf("control%d_%d", index, variant) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s p+0(FP),AX\nMOVL $0,CX\n%s %s,%s\nRET\n", name, frame, move, row.name, address, row.destination) + fmt.Fprintf(&declarations, "func %s(p *[128]byte)\n", name) + } + } + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("ordinary GP source control failed", err) + } + if strings.Count(ir, "asm sideeffect") != 76 { + t.Fatal("dead conversions lost whole native effects") + } + compileLLVMToObject(t, llc, target.triple, "native-address.ll", "native-address.o", x86MulDivRoundOptimize(t, ir)) + }) + } + }) + } +} diff --git a/amd64_packed_conversion_raw_test.go b/amd64_packed_conversion_raw_test.go new file mode 100644 index 000000000..1aae6d7a4 --- /dev/null +++ b/amd64_packed_conversion_raw_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Assemble a bounded batch once and partition every byte by real instruction +// boundaries. Runtime raw cases do not use our mnemonic-to-opcode tables. +func x86PackedConversionOriginalRawRows(t *testing.T, arch string, rows []string) []string { + t.Helper() + code := x86PackedShiftActualGoBytes(t, arch, "TEXT raw(SB),4,$0-0\n"+strings.Join(rows, "\n")+"\n") + mode := 64 + if arch == "386" { + mode = 32 + } + var raw []string + position := 0 + for _, row := range rows { + _, length, known, err := decodedX86PackedNumericConversionInstruction(code[position:], mode) + if !known { + instruction, decodeErr := x86asm.Decode(code[position:], mode) + length, err = instruction.Len, decodeErr + } + if err != nil || length < 1 || length > len(code)-position { + t.Fatal("original conversion boundary", row, err) + } + var directives strings.Builder + for _, value := range code[position : position+length] { + fmt.Fprintf(&directives, "BYTE $%d\n", value) + } + raw = append(raw, strings.TrimSuffix(directives.String(), "\n")) + position += length + } + if position != len(code) { + t.Fatal("original conversion batch has unconsumed bytes") + } + return raw +} + +func TestX86PackedConversionRawCompleteOriginalGoForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + rows := x86PackedConversionOracleRows(arch) + var source strings.Builder + source.WriteString("TEXT raw(SB),4,$0-0\n") + for _, row := range rows { + source.WriteString(row + "\n") + } + source.WriteString("RET\n") + code := x86PackedShiftActualGoBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(arch, err) + } + if len(decoded.Instrs) != len(rows)+1 { + t.Fatal("whole original Go byte stream lost instructions", arch, len(decoded.Instrs), len(rows)+1) + } + for index, row := range rows { + want, got := x86ScalarLaneInstruction(t, row), decoded.Instrs[index] + if want.Op != got.Op || len(want.Args) != len(got.Args) { + t.Fatal("raw opcode/mask mismatch", arch, row, got) + } + for operand := range want.Args { + if want.Args[operand].String() != got.Args[operand].String() { + t.Fatal("raw physical operand mismatch", arch, row, got) + } + } + } + t.Logf("%s: %d actual-Go forms, complete raw byte/typed-operand agreement", arch, len(rows)) + } +} + +func TestX86PackedConversionRawPrefixAndTruncationBoundaries(t *testing.T) { + for _, spec := range x86PackedConversionOracleSpecs() { + if spec.legacy { + continue + } + for _, shape := range spec.shapes() { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + row := spec.name + ".Z 64(AX)(CX*4),K1," + bank[shape.destination] + "7" + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT raw(SB),4,$0-0\n"+row+"\n") + ins, length, known, err := decodedX86PackedNumericConversionInstruction(code, 64) + if err != nil || !known || length != len(code) || ins.Op != Op(spec.name+".Z") { + t.Fatal(row, fmt.Sprintf("%x", code), ins, length, err) + } + for end := 0; end < len(code); end++ { + if _, _, known, err := decodedX86PackedNumericConversionInstruction(code[:end], 64); known && err == nil { + t.Fatal("truncated instruction accepted", row, end) + } + } + for _, mutation := range []struct { + index int + xor byte + }{{1, 0x08}, {2, 0x04}, {2, 0x08}, {3, 0x08}, {3, 0x01}} { + bad := append([]byte(nil), code...) + bad[mutation.index] ^= mutation.xor + if _, _, known, err := decodedX86PackedNumericConversionInstruction(bad, 64); known && err == nil { + t.Fatal("invalid fixed/vvvv/zero field accepted", row, fmt.Sprintf("%x", bad)) + } + } + // Exact high source/destination registers are unavailable in mode32; + // a legal K1 mask by itself is not a high-register binding. + if _, _, known, err := decodedX86PackedNumericConversionInstruction(code, 32); !known || err != nil { + t.Fatal("legal mode32 masked conversion rejected", row, err) + } + } + } + for _, encoding := range []struct { + pp, opcode byte + name string + }{ + {1, 0x5b, "VCVTPS2DQ"}, {2, 0x5b, "VCVTTPS2DQ"}, + {3, 0xe6, "VCVTPD2DQX"}, {1, 0xe6, "VCVTTPD2DQX"}, + } { + for _, mode := range []int{32, 64} { + for _, w := range []byte{0, 0x80} { + code := []byte{0xc4, 0xe1, 0x78 | encoding.pp | w, encoding.opcode, 0xc1} + ins, n, known, err := decodedX86PackedNumericConversionInstruction(code, mode) + if err != nil || !known || n != len(code) || string(ins.Op) != encoding.name { + t.Fatal("Intel VEX.WIG must not become W0-only", mode, fmt.Sprintf("%x", code), ins, err) + } + } + } + } +} diff --git a/amd64_packed_conversion_runtime_linux_amd64_test.go b/amd64_packed_conversion_runtime_linux_amd64_test.go new file mode 100644 index 000000000..7aa7e7643 --- /dev/null +++ b/amd64_packed_conversion_runtime_linux_amd64_test.go @@ -0,0 +1,17 @@ +package plan9asm + +import "testing" + +// This required owner is separate from Rosetta SSE and object-only evidence. +// CPUID/OS-state and original-opcode preflight must execute successfully. +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512Status(t *testing.T) { + x86PackedConversionRuntimeMatrix(t, true) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeIndependentC(t *testing.T) { + amd64PackedNumericConversionNativeRuntimeSemantics(t) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaults(t *testing.T) { + x86PackedConversionDeadFaults(t, true, true) +} diff --git a/amd64_packed_conversion_runtime_selection_test.go b/amd64_packed_conversion_runtime_selection_test.go new file mode 100644 index 000000000..454b123b5 --- /dev/null +++ b/amd64_packed_conversion_runtime_selection_test.go @@ -0,0 +1,67 @@ +package plan9asm + +import ( + "go/ast" + "go/build" + "go/parser" + "go/token" + "os" + "strings" + "testing" +) + +func TestX86PackedConversionRequiredNativeOwners(t *testing.T) { + const file = "amd64_packed_conversion_runtime_linux_amd64_test.go" + for _, target := range []struct{ os, arch string }{ + {"linux", "amd64"}, {"linux", "386"}, {"linux", "arm"}, {"linux", "arm64"}, + {"js", "wasm"}, {"darwin", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}, + } { + context := build.Default + context.GOOS, context.GOARCH = target.os, target.arch + selected, err := context.MatchFile(".", file) + if err != nil || selected != (target.os == "linux" && target.arch == "amd64") { + t.Fatal("required native execution owner selection", target, selected, err) + } + } + syntax, err := parser.ParseFile(token.NewFileSet(), file, nil, 0) + if err != nil { + t.Fatal(err) + } + want := map[string][]string{ + "TestCrossLinuxRuntimeMatrixX86PackedConversionNativeAVX512Status": {"x86PackedConversionRuntimeMatrix", "t", "true"}, + "TestCrossLinuxRuntimeMatrixX86PackedConversionNativeIndependentC": {"amd64PackedNumericConversionNativeRuntimeSemantics", "t"}, + "TestCrossLinuxRuntimeMatrixX86PackedConversionNativeDeadFaults": {"x86PackedConversionDeadFaults", "t", "true", "true"}, + } + for _, declaration := range syntax.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok { + continue + } + expected, found := want[function.Name.Name] + if !found || function.Body == nil || len(function.Body.List) != 1 { + t.Fatal("mandatory native entry changed or became conditional", function.Name.Name) + } + statement, ok := function.Body.List[0].(*ast.ExprStmt) + if !ok { + t.Fatal("required native helper not executed directly") + } + call, ok := statement.X.(*ast.CallExpr) + if !ok || len(call.Args)+1 != len(expected) { + t.Fatal("native helper lost an execution axis") + } + for index, expression := range append([]ast.Expr{call.Fun}, call.Args...) { + identifier, ok := expression.(*ast.Ident) + if !ok || identifier.Name != expected[index] { + t.Fatal("native helper or required full-family argument changed") + } + } + delete(want, function.Name.Name) + } + if len(want) != 0 { + t.Fatal("missing required native owners", want) + } + workflow, err := os.ReadFile(".github/workflows/go-ci.yml") + if err != nil || !strings.Contains(string(workflow), "-ci-selector '^(TestCrossLinuxRuntimeMatrix|") { + t.Fatal("required native runtime roots not selected by CI", err) + } +} diff --git a/amd64_packed_conversion_runtime_test.go b/amd64_packed_conversion_runtime_test.go new file mode 100644 index 000000000..0f3dd86ae --- /dev/null +++ b/amd64_packed_conversion_runtime_test.go @@ -0,0 +1,264 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86PackedConversionRuntimeRow struct { + instruction string + wide, dead bool + inputBits int + mask bool + sae bool +} + +func x86PackedConversionRuntimeRows(wide bool) []x86PackedConversionRuntimeRow { + var rows []x86PackedConversionRuntimeRow + bank := map[int]string{16: "X", 32: "Y", 64: "Z"} + for _, spec := range x86PackedConversionOracleSpecs() { + if !wide && !spec.legacy { + continue + } + for _, shape := range spec.shapes() { + destination := bank[shape.destination] + "1" + for _, input := range []string{bank[shape.source] + "0", "(SI)"} { + rows = append(rows, x86PackedConversionRuntimeRow{instruction: spec.name + " " + input + "," + destination, wide: wide, inputBits: spec.input}) + } + rows = append(rows, x86PackedConversionRuntimeRow{instruction: spec.name + " " + bank[shape.source] + "1," + destination, wide: wide, inputBits: spec.input}, + x86PackedConversionRuntimeRow{instruction: spec.name + " " + bank[shape.source] + "0," + destination, wide: wide, dead: true, inputBits: spec.input}) + if !spec.legacy { + for _, spelling := range []string{ + spec.name + " " + bank[shape.source] + "0,K1," + destination, + spec.name + ".Z (SI),K1," + destination, + spec.name + ".BCST (SI),K1," + destination, + spec.name + ".BCST.Z (SI),K1," + destination, + } { + rows = append(rows, x86PackedConversionRuntimeRow{instruction: spelling, wide: wide, mask: true, inputBits: spec.input}) + } + } + if (shape.source == 64 || shape.destination == 64) && spec.control != "" { + controls := []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} + if spec.control == "sae" { + controls = []string{"SAE"} + } + for _, control := range controls { + rows = append(rows, x86PackedConversionRuntimeRow{ + instruction: spec.name + "." + control + ".Z " + bank[shape.source] + "0,K1," + destination, + wide: wide, mask: true, sae: true, inputBits: spec.input, + }) + } + } + } + } + return rows +} + +func x86PackedConversionRuntimeFixture(rows []x86PackedConversionRuntimeRow) (assembly, declarations, original, driver string) { + var asm, decl, goCalls, cDecl, cCalls strings.Builder + for index, row := range rows { + name := fmt.Sprintf("packed%d", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$8-24\nSTMXCSR saved-4(SP)\nMOVQ out+0(FP),DI\nMOVQ in+8(FP),SI\n", name) + if row.wide { + asm.WriteString("VMOVDQU64 (SI),Z0\nVMOVDQU64 (DI),Z1\nMOVL mask+20(FP),CX\nKMOVW CX,K1\n") + } else { + asm.WriteString("MOVOU (SI),X0\nMOVOU (DI),X1\n") + } + asm.WriteString("LDMXCSR csr+16(FP)\nMOVQ $-1,DX\nCMPQ DX,$-1\n") + asm.WriteString(row.instruction + "\n") + if !row.dead { + if row.wide { + asm.WriteString("VMOVDQU64 Z1,(DI)\n") + } else { + asm.WriteString("MOVOU X1,(DI)\n") + } + } + asm.WriteString("STMXCSR 72(DI)\nSETEQ 64(DI)\nSETCS 65(DI)\nSETOS 66(DI)\nSETMI 67(DI)\nSETPC 68(DI)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&decl, "func %s(out *[12]uint64,in *[8]uint64,csr,mask uint32)\n", name) + masks, sae := 1, 0 + if row.mask { + masks = 4 + } + if row.sae { + sae = 1 + } + fmt.Fprintf(&goCalls, "check(%d,%d,%d,%d,%s)\n", index, row.inputBits, masks, sae, name) + fmt.Fprintf(&cDecl, "extern void %s(uint64_t*,const uint64_t*,uint32_t,uint32_t);\n", name) + fmt.Fprintf(&cCalls, "check(%d,%d,%d,%d,%s);\n", index, row.inputBits, masks, sae, name) + } + var gValues, cValues strings.Builder + values := append(append([]uint64{}, x86ScalarFloatIntegerSSInputs...), x86ScalarFloatIntegerSDInputs...) + values = append(values, 0xffffff, 0x1000001, 0xffffffff, 0x1fffffffffffff, 0x20000000000001, 0x7fffffffffffffff, 0xffffffffffffffff) + for _, value := range values { + fmt.Fprintf(&gValues, "%#x,", value) + fmt.Fprintf(&cValues, "UINT64_C(%#x),", value) + } + original = `package main +import "fmt" +var values=[]uint64{` + gValues.String() + `} +var masks=[]uint32{0xffff,0,1,0xaaaa} +func check(index,bits,maskCount,sae int,fn func(*[12]uint64,*[8]uint64,uint32,uint32)) { + hash:=uint64(1469598103934665603); count:=0 + for n:=range values { for rc:=0;rc<4;rc++ { for controls:=0;controls<8;controls++ { for m:=0;m>shift)))*1099511628211} } + count++ + }}}} + fmt.Printf("%d %d %016x\n",index,count,hash) +} +` + decl.String() + "func main(){\n" + goCalls.String() + "}\n" + driver = `#include +#include +static const uint64_t values[]={` + cValues.String() + `}; +static const uint32_t masks[]={0xffff,0,1,0xaaaa}; +static void check(unsigned index,unsigned bits,unsigned maskCount,unsigned sae,void (*fn)(uint64_t*,const uint64_t*,uint32_t,uint32_t)) { + uint64_t hash=UINT64_C(1469598103934665603);unsigned count=0,N=sizeof(values)/sizeof(*values); + for(unsigned n=0;n>shift))*UINT64_C(1099511628211); + count++; + } + printf("%u %u %016llx\n",index,count,(unsigned long long)hash); +} +` + cDecl.String() + "int main(void){\n" + cCalls.String() + "return 0;}\n" + return asm.String(), decl.String(), original, driver +} + +func x86PackedConversionRuntimeMatrix(t *testing.T, wide bool) { + t.Helper() + axis := x86FloatCompareSSE + if wide { + axis = x86FloatCompareEVEX + } + execution := x86FloatCompareRuntimeTools(t, axis) + rows := x86PackedConversionRuntimeRows(wide) + const limit = 96 + consumed := 0 + for start := 0; start < len(rows); start += limit { + end := start + limit + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + assembly, declarations, main, driver := x86PackedConversionRuntimeFixture(rows[start:end]) + want := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, main) + if strings.Count(string(want), "\n") != end-start { + t.Fatal("incomplete original Go function/case receipts") + } + var originals []string + for _, row := range rows[start:end] { + originals = append(originals, row.instruction) + } + raw := x86PackedConversionOriginalRawRows(t, "amd64", originals) + rawRows := append([]x86PackedConversionRuntimeRow(nil), rows[start:end]...) + for index := range rawRows { + rawRows[index].instruction = raw[index] + } + rawAssembly, _, _, _ := x86PackedConversionRuntimeFixture(rawRows) + for _, spelling := range []struct{ name, source string }{{"named", assembly}, {"raw", rawAssembly}} { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, spelling.source, declarations, "amd64", execution.triple, api) + if err != nil { + t.Fatal(start, api, err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + for _, level := range []string{"-O0", "-O2"} { + got := execution.compileAndRunLLVM(t, "packed-conversion-matrix", input, driver, level) + if string(got) != string(want) { + t.Fatalf("batch %d %s/%s optimized=%v %s\nGo:\n%sLLVM:\n%s", start, spelling.name, api, optimized, level, want, got) + } + } + } + } + } + consumed += end - start + }) + } + if consumed != len(rows) { + t.Fatal("runtime batch partition lost rows") + } + t.Logf("%d independent conversion kernels, 63 bit-pattern seeds x 4 rounding x 4 DAZ/FTZ x 2 clean/sticky status; masked kernels also 4 masks; named/raw x 3 APIs x 4 optimization axes", len(rows)) +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionLegacyIEEEStatusFlags(t *testing.T) { + x86PackedConversionRuntimeMatrix(t, false) +} + +// Object receipts for the actual wide runtime fixture are not execution +// receipts. They close its source/API/lifecycle path before a native owner +// attempts the same kernels, without executing unsupported host instructions. +func TestX86PackedConversionNativeRuntimeFixtureObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + rows := x86PackedConversionRuntimeRows(true) + const limit = 96 + consumed := 0 + for start := 0; start < len(rows); start += limit { + end := start + limit + if end > len(rows) { + end = len(rows) + } + t.Run(fmt.Sprintf("batch-%04d", start), func(t *testing.T) { + batch := append([]x86PackedConversionRuntimeRow(nil), rows[start:end]...) + var instructions []string + for _, row := range batch { + instructions = append(instructions, row.instruction) + } + raw := x86PackedConversionOriginalRawRows(t, "amd64", instructions) + for _, spelling := range []string{"named", "raw"} { + if spelling == "raw" { + for index := range batch { + batch[index].instruction = raw[index] + } + } + source, declarations, _, _ := x86PackedConversionRuntimeFixture(batch) + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(spelling, api, err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "native-runtime.ll", "native-runtime.o", input) + } + } + } + consumed += end - start + }) + } + if consumed != len(rows) { + t.Fatal("native runtime fixture partition lost rows") + } + t.Logf("%d complete wide runtime kernels, named/raw x 3 APIs x 2 IR pipelines; object-only", consumed) +} diff --git a/amd64_packed_conversion_status_runtime_test.go b/amd64_packed_conversion_status_runtime_test.go new file mode 100644 index 000000000..eff5f8183 --- /dev/null +++ b/amd64_packed_conversion_status_runtime_test.go @@ -0,0 +1,108 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The four legacy Yxm/Yxr rows in Go's yxcvm1 table are an independently +// executable status witness for packed float-to-dword conversion. MMX rows +// and AVX/EVEX forms need their own physical-state execution owners. +func x86PackedConversionConstantStatusOracle() (assembly, declarations, original, driver string) { + var asm, decl, goCalls, cDecl, cCalls strings.Builder + index := 0 + for _, name := range []string{"CVTPS2PL", "CVTTPS2PL", "CVTPD2PL", "CVTTPD2PL"} { + values := []uint64{0x7fc12345, 0x7f800000, 0xff800000, 0x4f000000, 0x3fc00000, 1} + if strings.Contains(name, "PD") { + values = []uint64{0x7ff8123456789abc, 0x7ff0000000000000, 0xfff0000000000000, 0x41e0000000000000, 0x3ff8000000000000, 1} + } + for _, bits := range values { + if strings.Contains(name, "PS") { + bits |= bits << 32 + } + for _, live := range []bool{false, true} { + function := fmt.Sprintf("packedProbe%d", index) + fmt.Fprintf(&asm, "TEXT ·%s(SB),4,$8-12\n", function) + asm.WriteString("STMXCSR saved-4(SP)\nMOVQ out+0(FP),DI\n") + fmt.Fprintf(&asm, "MOVQ $%#x,AX\nMOVQ AX,X0\nPUNPCKLQDQ X0,X0\n", bits) + fmt.Fprintf(&asm, "LDMXCSR csr+8(FP)\n%s X0,X1\n", name) + if live { + asm.WriteString("MOVOU X1,(DI)\n") + } + asm.WriteString("STMXCSR 16(DI)\nLDMXCSR saved-4(SP)\nRET\n") + fmt.Fprintf(&decl, "func %s(out *[4]uint64, csr uint32)\n", function) + fmt.Fprintf(&goCalls, "check(%d, %s)\n", index, function) + fmt.Fprintf(&cDecl, "extern void %s(uint64_t*, uint32_t);\n", function) + fmt.Fprintf(&cCalls, "check(%d, %s);\n", index, function) + index++ + } + } + } + original = `package main +import "fmt" +func check(index int, fn func(*[4]uint64, uint32)) { + for mode := 0; mode < 4; mode++ { + out := [4]uint64{0x1111111111111111, 0x2222222222222222, 0x3333333333333333, 0x4444444444444444} + fn(&out, 0x1f80|uint32(mode)<<13) + fmt.Printf("%d %d %016x %016x %016x %016x\n", index, mode, out[0], out[1], out[2], out[3]) + } +} +` + decl.String() + "func main() {\n" + goCalls.String() + "}\n" + driver = `#include +#include +static void check(int index, void (*fn)(uint64_t*, uint32_t)) { + for (unsigned mode = 0; mode < 4; mode++) { + uint64_t out[] = {UINT64_C(0x1111111111111111), UINT64_C(0x2222222222222222), UINT64_C(0x3333333333333333), UINT64_C(0x4444444444444444)}; + fn(out, 0x1f80 | (mode << 13)); + printf("%d %u %016llx %016llx %016llx %016llx\n", index, mode, + (unsigned long long)out[0], (unsigned long long)out[1], + (unsigned long long)out[2], (unsigned long long)out[3]); + } +} +` + cDecl.String() + "int main(void) {\n" + cCalls.String() + "return 0;\n}\n" + return asm.String(), decl.String(), original, driver +} + +func TestCrossLinuxRuntimeMatrixX86PackedConversionOriginalConstantsAndStatus(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + assembly, declarations, main, driver := x86PackedConversionConstantStatusOracle() + original := x86ScalarFloatIntegerOriginalOutput(t, execution, assembly, main) + want := strings.Split(strings.TrimSpace(string(original)), "\n") + if len(want) != 192 { + t.Fatalf("original Go produced %d rows, want 4 instructions x 6 inputs x 2 live/dead x 4 MXCSR modes", len(want)) + } + ir, err := x86ScalarFPBitViewAPI(t, assembly, declarations, "amd64", execution.triple, "text") + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, ir) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run(fmt.Sprintf("IR_O2=%v/backend%s", optimized, backend), func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "packed-conversion-status", input, driver, backend) + got := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(got) != len(want) { + t.Fatalf("LLVM row count=%d original=%d", len(got), len(want)) + } + mismatches := 0 + for index := range want { + if got[index] != want[index] { + if mismatches == 0 { + t.Logf("first mismatch: original %s; LLVM %s", want[index], got[index]) + } + mismatches++ + } + } + if mismatches != 0 { + t.Errorf("original Go packed value/MXCSR/canary mismatches=%d/%d", mismatches, len(want)) + } + }) + } + } +} diff --git a/amd64_packed_double_dword_native_linux_amd64_test.go b/amd64_packed_double_dword_native_linux_amd64_test.go new file mode 100644 index 000000000..179d95d7e --- /dev/null +++ b/amd64_packed_double_dword_native_linux_amd64_test.go @@ -0,0 +1,9 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixAMD64PackedDoubleDwordNativeAVX512(t *testing.T) { + execution := x86PackedDoubleDwordNativeTools(t) + x86PackedDoubleDwordOriginalGo(t, execution) + amd64PackedDoubleDwordRuntime(t, execution, true) +} diff --git a/amd64_packed_double_dword_native_test.go b/amd64_packed_double_dword_native_test.go new file mode 100644 index 000000000..7ecc9a584 --- /dev/null +++ b/amd64_packed_double_dword_native_test.go @@ -0,0 +1,236 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func x86PackedDoubleDwordGoAssemblerSource(source string) string { + // Preserve the original NOSPLIT spelling and let this actual driver's + // original textflag.h define it, rather than silently rewriting TEXT. + return fmt.Sprintf("#include %q\n", filepath.Join(runtime.GOROOT(), "pkg", "include", "textflag.h")) + source +} + +const x86PackedDoubleDwordMaskPreflightC = ` +#undef main +__attribute__((target("avx512f,avx512bw"))) static int original_mask64(void) { + uint64_t input=0xa5,output; + __asm__ volatile("kmovq %1,%%k1; kmovq %%k1,%0" : "=r"(output) : "r"(input) : "k1"); + return output==input; +} +static int mask64_required_state(unsigned bw) { return bw!=0; } +int main(void) { + int failure=compare_preflight_main(); + if (failure) return failure; + unsigned a,b,c,d,bw=0; + if (__get_cpuid_count(7,0,&a,&b,&c,&d)) bw=(b>>30)&1; + printf("original double/dword mask transport AVX512BW=%u\n",bw); + fflush(stdout); + if (!mask64_required_state(bw)) return 83; + return original_mask64() ? 0 : 84; +} +` + +func x86PackedDoubleDwordNativeTools(t *testing.T) x86FloatCompareExecution { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + dir := t.TempDir() + source, binary := filepath.Join(dir, "mask-preflight.c"), filepath.Join(dir, "mask-preflight") + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + x86PackedDoubleDwordMaskPreflightC + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + t.Logf("actual source KMOVQ preflight: %s", execution.run(t, binary)) + return execution +} + +func TestX86PackedDoubleDwordNativeMaskPreflight(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, binary := filepath.Join(dir, "mask-capabilities.c"), filepath.Join(dir, "mask-capabilities") + program := "#define REQUIRED_AXIS 3\n#define main compare_preflight_main\n" + x86FloatCompareOpcodePreflightC + + strings.Replace(x86PackedDoubleDwordMaskPreflightC, "int main(void)", "int actual_mask_main(void)", 1) + ` +int main(void) { return !mask64_required_state(1) || mask64_required_state(0); } +` + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, binary) + execution.run(t, binary) +} + +func TestX86PackedDoubleDwordRuntimeRequiresNativeUnion(t *testing.T) { + full, local := x86PackedDoubleDwordOriginalAssembly(), x86PackedDoubleDwordSSEAssembly(t) + for _, opcode := range []string{"VCVTDQ2PD (CX), K1, Z1", "VCVTTPD2DQY (BX), X2", "VCVTPD2DQ.RD_SAE.Z Z0, K1, Y3", "CVTPD2PL X4, X0"} { + if strings.Count(full, opcode) != 1 { + t.Fatal("lost or repeated original conversion", opcode) + } + } + parsed, err := Parse(ArchAMD64, local) + if err != nil { + t.Fatal(err) + } + for _, instruction := range parsed.Funcs[0].Instrs { + if strings.HasPrefix(string(instruction.Op), "V") || strings.HasPrefix(string(instruction.Op), "KMOV") { + t.Fatal("portable SSE counterpart borrowed a wide opcode", instruction.Op) + } + } + if strings.Count(local, "CVTPD2PL X4, X0") != 1 { + t.Fatal("portable SSE counterpart borrowed a wide ISA or lost its original legacy conversion") + } + file, err := parser.ParseFile(token.NewFileSet(), "amd64_packed_double_dword_native_linux_amd64_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + var body *ast.BlockStmt + for _, declaration := range file.Decls { + if fn, ok := declaration.(*ast.FuncDecl); ok && fn.Name.Name == "TestCrossLinuxRuntimeMatrixAMD64PackedDoubleDwordNativeAVX512" { + body = fn.Body + } + } + if body == nil || len(body.List) != 3 { + t.Fatal("complete original Go and named/raw LLVM runtime must consume native transport") + } + for index, name := range []string{"x86PackedDoubleDwordNativeTools", "x86PackedDoubleDwordOriginalGo", "amd64PackedDoubleDwordRuntime"} { + var call *ast.CallExpr + if index == 0 { + if assignment, ok := body.List[index].(*ast.AssignStmt); ok && len(assignment.Rhs) == 1 { + call, _ = assignment.Rhs[0].(*ast.CallExpr) + } + } else if statement, ok := body.List[index].(*ast.ExprStmt); ok { + call, _ = statement.X.(*ast.CallExpr) + } + if call == nil { + t.Fatal("required conversion consumer cannot skip or return", name) + } + callee, ok := call.Fun.(*ast.Ident) + if !ok || callee.Name != name { + t.Fatal("required conversion consumer missing", name) + } + if index > 0 { + route, ok := call.Args[1].(*ast.Ident) + if !ok || route.Name != "execution" { + t.Fatal("conversion consumer lost its exact preflight route") + } + } + if index == 2 { + if len(call.Args) != 3 { + t.Fatal("complete wide conversion execution was omitted") + } + wide, ok := call.Args[2].(*ast.Ident) + if !ok || wide.Name != "true" { + t.Fatal("complete wide conversion assertions cannot be disabled") + } + } + } +} + +func x86PackedDoubleDwordOriginalGo(t *testing.T, execution x86FloatCompareExecution) { + t.Helper() + if execution.triple != "x86_64-unknown-linux-gnu" || len(execution.prefix) != 0 || execution.preserveArgv0 { + t.Fatal("original packed double/dword requires actual native Linux AMD64") + } + dir := x86PackedDoubleDwordOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-count=1", "-run", "^TestOriginalDoubleDword$", "-v", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + t.Logf("actual original Go named/raw double/dword: %s", x86FloatCompareCaptureCommand(t, command)) +} + +func TestX86PackedDoubleDwordCompleteOriginalGoCompiles(t *testing.T) { + dir := x86PackedDoubleDwordOriginalPackage(t) + command := exec.Command("go", "test", "-p=2", "-c", "-o", filepath.Join(dir, "original"), ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOWORK=off", "GOENV=off", "GOARCH=amd64", "GOOS=linux") + x86FloatCompareCaptureCommand(t, command) +} + +func x86PackedDoubleDwordOriginalPackage(t *testing.T) string { + t.Helper() + dir := t.TempDir() + named := strings.Replace(x86PackedDoubleDwordOriginalAssembly(), "packeddoubledword(SB)", "·named(SB)", 1) + raw := strings.Replace(x86PackedDoubleDwordRawAssembly(x86PackedDoubleDwordOriginalAssembly()), "packeddoubledword(SB)", "·raw(SB)", 1) + assembly := "#include \"textflag.h\"\n" + named + raw + "TEXT ·setCSR(SB),4,$0-4\nLDMXCSR csr+0(FP)\nRET\nTEXT ·getCSR(SB),4,$0-4\nSTMXCSR ret+0(FP)\nRET\n" + for name, contents := range map[string]string{ + "go.mod": "module originaldoubledword\n\ngo 1.20\n", "original_amd64.s": assembly, + "original_test.go": x86PackedDoubleDwordOriginalGoTest, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + return dir +} + +// This is output-only numerical evidence, not an MMX/x87 TAG/TOS identity +// claim. The original fixture's MM0 conversion remains unchanged. +const x86PackedDoubleDwordOriginalGoTest = `package original + +import ( + "encoding/binary" + "math" + "runtime" + "testing" +) + +func named(out *[120]byte, doubles *[8]float64, dwords *[8]int32, old *[8]float64, mask uint64) +func raw(out *[120]byte, doubles *[8]float64, dwords *[8]int32, old *[8]float64, mask uint64) +func setCSR(csr uint32) +func getCSR() uint32 + +func TestOriginalDoubleDword(t *testing.T) { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + defer setCSR(getCSR()) + + doubles := [8]float64{ + 1.75, -2.75, 2147483648.0, math.Float64frombits(0x7ff8000000000000), + -2147483648.0, -2147483649.0, 3.0, math.Float64frombits(0x8000000000000000), + } + dwords := [8]int32{0, 1, -1, 2147483647, -2147483648, 123456789, -987654321, 42} + old := [8]float64{100.5, 101.5, 102.5, 103.5, 104.5, 105.5, 106.5, 107.5} + truncated := [4]int32{1, -2, -2147483648, -2147483648} + down := [8]int32{1, 0, -2147483648, 0, 0, -2147483648, 0, 0} + + for index, fn := range []func(*[120]byte, *[8]float64, *[8]int32, *[8]float64, uint64){named, raw} { + var guarded [122]byte + for i := range guarded { + guarded[i] = 0xcc + } + setCSR(0x1f80) + fn((*[120]byte)(guarded[1:121]), &doubles, &dwords, &old, 0xa5) + output := guarded[1:121] + if guarded[0] != 0xcc || guarded[121] != 0xcc { + t.Fatal("original conversion output canary", index) + } + + for i := 0; i < 8; i++ { + want := old[i] + if 0xa5&(1<= 8 { + widths = []int{16, 32, 64} + } + for _, width := range widths { + variants := []int{0} + if opIndex >= 8 { + variants = []int{1, 2} + if width < 64 { + variants = append(variants, 0) + } + if bits >= 32 { + variants = append(variants, 3, 4) + } + } + for _, variant := range variants { + masked, broadcast := variant == 2 || variant == 4, variant >= 3 + name := fmt.Sprintf("faultcompare%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(input *byte,mask uint64)\n", name) + fmt.Fprintf(&prototypes, "extern void %s(const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ input+0(FP),BX\nMOVQ mask+8(FP),AX\nKMOVQ AX,K1\nVPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\n", name) + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + operands := "(BX),X1" + if opIndex >= 8 { + mask, destination := "", bank+"0" + if variant != 0 { + destination = "K2" + } + if masked { + mask = "K1," + } + operands = fmt.Sprintf("(BX),%s1,%s%s", bank, mask, destination) + } + instruction := op + if broadcast { + instruction += ".BCST" + } + fmt.Fprintf(&source, "%s %s\nRET\n", instruction, operands) + fmt.Fprintf(&references, "__attribute__((noinline)) static void reference%d(const uint8_t *p,uint64_t mask) {\n", count-1) + if opIndex < 8 { + mnemonic := strings.ToLower(strings.ReplaceAll(op, "L", "D")) + references.WriteString(" typedef int32_t V __attribute__((vector_size(16)));\n V value = {0,0,0,0};\n") + fmt.Fprintf(&references, " __asm__ volatile(\"%s (%%1),%%0\" : \"+x\"(value) : \"r\"(p) : \"memory\");\n", mnemonic) + } else { + bytes := bits / 8 + fmt.Fprintf(&references, " for (int lane = 0; lane < %d; ++lane) {\n", width/bytes) + if masked { + references.WriteString(" if (!((mask>>lane)&1)) continue;\n") + } + offset := "lane*" + fmt.Sprint(bytes) + if broadcast { + offset = "0" + } + fmt.Fprintf(&references, " for (int j = 0; j < %d; ++j) {\n (void)((const volatile uint8_t*)p)[%s+j];\n }\n", bytes, offset) + references.WriteString(" }\n") + } + references.WriteString("}\n") + lanes := width * 8 / bits + masks := []uint64{0} + if masked { + masks = []uint64{0, 1, uint64(1) << uint(lanes-1), ^uint64(0)} + } + for _, mask := range masks { + expectedReadable := "0" + if opIndex < 8 { + expectedReadable = "offset == 0 ? 0 : 2" + } + fmt.Fprintf(&checks, " for (int offset = 0; offset < 16; ++offset) {\n if (compare(reference%d,%s,memory+offset,UINT64_C(%d),%s)) return 1;\n }\n", count-1, name, mask, expectedReadable) + for _, offset := range []int{0, 1, bits / 8, 16, 32, 64} { + expected := x86PackedIntegerCompareMemoryExpectation(opIndex < 8, width, bits/8, masked, broadcast, mask, offset, offset%16 == 0) + fmt.Fprintf(&checks, " if (compare(reference%d,%s,boundary-%d,UINT64_C(%d),%d)) return 1;\n", count-1, name, offset, mask, expected) + } + expectedNull := x86PackedIntegerCompareMemoryExpectation(opIndex < 8, width, bits/8, masked, broadcast, mask, 0, true) + fmt.Fprintf(&checks, " if (compare(reference%d,%s,0,UINT64_C(%d),%d)) return 1;\n", count-1, name, mask, expectedNull) + } + } + } + } + x86LEAActualTargetGoObject(t, "amd64", triple, source.String()) + ir, err := x86PackedIntegerCompareAPI(t, source.String(), declarations.String(), "amd64", triple, "Go") + if err != nil { + t.Fatal(err) + } + program := `#include +#include +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + references.String() + x86PackedIntegerCompareFaultStatusOracle + ` +int main(void) { + struct rlimit noCore = {0,0}; + if (setrlimit(RLIMIT_CORE,&noCore)) return 76; + unsigned a,b,c,d; + if (!__get_cpuid(1,&a,&b,&c,&d) || !(c&(1u<<19)) || !(c&(1u<<20))) { + fprintf(stderr,"packed compare fault root requires actual SSE4.1 and SSE4.2\n"); + return 77; + } + __asm__ volatile("pxor %%xmm0,%%xmm0\n\t" + "pcmpeqq %%xmm0,%%xmm0\n\t" + "pcmpgtq %%xmm0,%%xmm0" : : : "xmm0"); + printf("packed compare fault preflight: CPUID SSE4.1=1 SSE4.2=1; actual PCMPEQQ/PCMPGTQ executed\n"); + long page = sysconf(_SC_PAGESIZE); + if (page <= 0 || page%64) return 1; + uint8_t *memory = mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON,-1,0); + if (memory == MAP_FAILED) return 1; + memset(memory,0x87,(size_t)page); + if (mprotect(memory,(size_t)page,PROT_READ) || + mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary = memory+page; +` + checks.String() + ` + if (munmap(memory,(size_t)page*2)) return 1; + return 0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-compare-fault", triple, text, program, prefix) + }) + } + t.Logf("%d dead-result programs at mod16/NULL/read-only guard pages, exact native legacy and independent E4 lane-read status at O0/O2; not native EVEX execution", count) +} + +// Expected status comes from the independently constructed fixture accesses, +// not a production descriptor. 0 requires normal exit0; 1 requires a memory +// fault. Only readable, unaligned legacy sources use 2: preserve the original +// machine's exact alignment behavior without guessing a vendor's fault policy. +func x86PackedIntegerCompareMemoryExpectation(legacy bool, width, laneBytes int, masked, broadcast bool, mask uint64, available int, aligned bool) int { + if legacy { + if available < 16 { + return 1 + } + if !aligned { + return 2 + } + return 0 + } + readEnd := 0 + for lane := 0; lane < width/laneBytes; lane++ { + if masked && mask&(uint64(1)< readEnd { + readEnd = end + } + } + if readEnd > available { + return 1 + } + return 0 +} + +const x86PackedIntegerCompareFaultStatusOracle = ` +typedef struct { int signal; int code; } Status; + +static int observe(void (*fn)(const uint8_t*,uint64_t),const uint8_t *p, + uint64_t mask,Status *out) { + pid_t child = fork(); + if (child < 0) return 1; + if (child == 0) { + struct rlimit limit = {0,0}; + if (setrlimit(RLIMIT_CORE,&limit)) _exit(3); + fn(p,mask); + _exit(0); + } + int status; + if (waitpid(child,&status,0) != child) return 1; + if (WIFSIGNALED(status)) { + *out = (Status){WTERMSIG(status),0}; + return 0; + } + if (WIFEXITED(status)) { + *out = (Status){0,WEXITSTATUS(status)}; + return 0; + } + return 1; +} + +static int compare(void (*original)(const uint8_t*,uint64_t), + void (*translated)(const uint8_t*,uint64_t), + const uint8_t *p,uint64_t mask,int expectation) { + Status a,b; + if (observe(original,p,mask,&a) || observe(translated,p,mask,&b)) return 1; + int allowedA = a.code == 0 && (a.signal == 0 || a.signal == SIGSEGV || a.signal == SIGBUS); + int allowedB = b.code == 0 && (b.signal == 0 || b.signal == SIGSEGV || b.signal == SIGBUS); + if (!allowedA || !allowedB || expectation < 0 || expectation > 2 || + (expectation == 0 && (a.signal || b.signal)) || + (expectation == 1 && (!a.signal || !b.signal)) || + a.signal != b.signal || a.code != b.code) { + fprintf(stderr,"mod16=%lu mask=%llx original=(%d,%d) translated=(%d,%d)\n", + (unsigned long)((uintptr_t)p%16),(unsigned long long)mask, + a.signal,a.code,b.signal,b.code); + return 1; + } + return 0; +} +` + +func TestX86PackedIntegerCompareFaultStatusRejectsSymmetricUnexpectedExit(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + // The child observations are real, but these deliberately raised signals + // test the oracle's status domain, not a memory fault or native opcode. + program := `#include +#include +#include +#include +#include +#include +` + x86PackedIntegerCompareFaultStatusOracle + ` +static int selectedSignal; + +static void unexpected(const uint8_t *p,uint64_t mask) { + if (!selectedSignal) { + _exit(7); + } + raise(selectedSignal); +} + +static void complete(const uint8_t *p,uint64_t mask) {} + +int main(void) { + const int rejected[] = {SIGILL,SIGABRT,SIGFPE,SIGTRAP,0}; + for (unsigned i = 0; i < sizeof(rejected)/sizeof(rejected[0]); ++i) { + selectedSignal = rejected[i]; + if (!compare(unexpected,unexpected,0,0,2)) { + fprintf(stderr,"symmetric unexpected status accepted: signal=%d\n",selectedSignal); + return 1; + } + } + return compare(complete,complete,0,0,0); +} +` + ir := "target triple = \"" + triple + "\"\ndefine void @status_tripwire_object() { ret void }\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-compare-status-tripwire", triple, ir, program, prefix) +} + +func TestX86PackedIntegerCompareFaultStatusRequiresSuccessfulReadableControl(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + // Deliberately raised memory signals are classification tripwires. They do + // not replace the independently mapped native-width fixtures above. + program := `#include +#include +#include +#include +#include +#include +` + x86PackedIntegerCompareFaultStatusOracle + ` +static void unexpectedMemoryFault(const uint8_t *p,uint64_t mask) { + raise(SIGSEGV); +} + +static void complete(const uint8_t *p,uint64_t mask) {} + +int main(void) { + if (!compare(unexpectedMemoryFault,unexpectedMemoryFault,0,0,0)) { + fprintf(stderr,"symmetric memory fault replaced required successful control\n"); + return 1; + } + if (compare(unexpectedMemoryFault,unexpectedMemoryFault,0,0,1)) return 2; + return compare(complete,complete,0,0,0); +} +` + ir := "target triple = \"" + triple + "\"\ndefine void @readable_tripwire_object() { ret void }\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-compare-readable-tripwire", triple, ir, program, prefix) +} + +func TestX86PackedIntegerCompareFaultExpectedAccessMatrix(t *testing.T) { + for _, test := range []struct { + name string + legacy, masked, broadcast bool + width, bytes, available int + mask uint64 + aligned bool + want int + }{ + {"legacy-aligned-readable", true, false, false, 16, 8, 16, 0, true, 0}, + {"legacy-unaligned-readable", true, false, false, 16, 8, 16, 0, false, 2}, + {"legacy-guard-overlap", true, false, false, 16, 8, 15, 0, true, 1}, + {"vex-full-readable-unaligned", false, false, false, 32, 4, 32, 0, false, 0}, + {"vex-full-guard-overlap", false, false, false, 32, 4, 31, 0, false, 1}, + {"evex-zero-mask-null", false, true, false, 64, 1, 0, 0, true, 0}, + {"evex-low-lane-readable", false, true, false, 64, 1, 1, 1, false, 0}, + {"evex-high-lane-guard", false, true, false, 64, 1, 63, uint64(1) << 63, false, 1}, + {"evex-high-lane-readable", false, true, false, 64, 1, 64, uint64(1) << 63, false, 0}, + {"broadcast-zero-mask-null", false, true, true, 64, 8, 0, 0, false, 0}, + {"broadcast-readable", false, true, true, 64, 8, 8, 0xff, false, 0}, + {"broadcast-guard-overlap", false, true, true, 64, 8, 7, 0xff, false, 1}, + {"broadcast-irrelevant-mask-null", false, true, true, 16, 8, 0, 4, false, 0}, + } { + t.Run(test.name, func(t *testing.T) { + got := x86PackedIntegerCompareMemoryExpectation(test.legacy, test.width, test.bytes, + test.masked, test.broadcast, test.mask, test.available, test.aligned) + if got != test.want { + t.Fatalf("independent expected access status=%d, want=%d", got, test.want) + } + }) + } +} diff --git a/amd64_packed_integer_compare_forms.go b/amd64_packed_integer_compare_forms.go new file mode 100644 index 000000000..11a7389e0 --- /dev/null +++ b/amd64_packed_integer_compare_forms.go @@ -0,0 +1,157 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedIntegerCompareSpec struct { + laneBits int + greater bool + legacy bool + mmx bool +} + +var x86PackedIntegerCompareSpecs = map[Op]x86PackedIntegerCompareSpec{ + "PCMPEQB": {8, false, true, true}, + "PCMPEQW": {16, false, true, true}, + "PCMPEQL": {32, false, true, true}, + "PCMPEQQ": {64, false, true, false}, + "PCMPGTB": {8, true, true, true}, + "PCMPGTW": {16, true, true, true}, + "PCMPGTL": {32, true, true, true}, + "PCMPGTQ": {64, true, true, false}, + "VPCMPEQB": {8, false, false, false}, + "VPCMPEQW": {16, false, false, false}, + "VPCMPEQD": {32, false, false, false}, + "VPCMPEQQ": {64, false, false, false}, + "VPCMPGTB": {8, true, false, false}, + "VPCMPGTW": {16, true, false, false}, + "VPCMPGTD": {32, true, false, false}, + "VPCMPGTQ": {64, true, false, false}, +} + +type x86PackedIntegerCompareForm struct { + x86PackedBinaryForm + spec x86PackedIntegerCompareSpec + source Operand + maskResult bool +} + +// Properties describe the Go operand rows and physical mode, not initialized +// inputs, native backing, memory bounds, or caller authority. The raw decoder's +// eight-entry encoding registry remains distinct from these sixteen Go names. +func x86PackedIntegerCompareProperties(op Op, ins Instr, arch string) (x86PackedIntegerCompareForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedIntegerCompareSpecs[Op(base)] + if !known { + return x86PackedIntegerCompareForm{}, false, nil + } + form := x86PackedIntegerCompareForm{spec: spec} + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return form, true, fmt.Errorf("packed integer compare dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + local.Args = append([]Operand(nil), ins.Args...) + fail := func() (x86PackedIntegerCompareForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go packed integer compare grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" { + return fail() + } + if spec.legacy || len(local.Args) == 3 && local.Args[2].Kind == OpReg && !amd64IsKOperand(local.Args[2]) { + encoding := x86PackedLogicalVEX + if spec.legacy { + encoding = x86PackedLogicalLegacyFP + if spec.mmx { + encoding = x86PackedLogicalLegacyInteger + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return form, true, err + } + } + } + binary, err := x86PackedBinaryProperties(local, arch, x86PackedBinarySpec{encoding: encoding}) + form.x86PackedBinaryForm = binary + if err != nil { + return form, true, err + } + } else { + // _yvpcmpeqb EVEX rows produce K0-7, not an X/Y/Z destination; + // their optional input mask is K1-7. There is no .Z form. + form.maskResult = true + form.x86PackedBinaryForm = x86PackedBinaryForm{ + first: 0, second: 1, destination: len(local.Args) - 1, + mask: -1, memory: -1, + } + if len(local.Args) != 3 && len(local.Args) != 4 || dotted && suffix != "BCST" { + return fail() + } + if !amd64IsKOperand(local.Args[form.destination]) { + return fail() + } + width, valid := amd64PackedCompareWidth(local.Args[1]) + if !valid || !amd64GoPackedVectorMoveRegister(local.Args[1], width, "amd64") { + return fail() + } + form.byteWidth = width + first := local.Args[0] + if first.Kind == OpReg { + if !amd64GoPackedVectorMoveRegister(first, width, "amd64") { + return fail() + } + } else if !x86PackedShiftMemoryValid(first, arch) { + return fail() + } else { + form.memory, form.memoryBytes = 0, width + } + if dotted { + if spec.laneBits < 32 || form.memory < 0 { + return fail() + } + form.broadcast, form.memoryBytes = true, spec.laneBits/8 + } + if len(local.Args) == 4 { + if !amd64NonzeroKOperand(local.Args[2]) || arch == "386" && !local.x86Encoded { + return fail() + } + form.mask = 2 + } + } + form.source = local.Args[form.first] + if err := x86Mode32VectorRegisterContext(local, arch); err != nil { + return form, true, err + } + if form.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + return form, true, fmt.Errorf("%w: packed integer compare has no bound address-size override", ErrProbeNeedsContext) + } + return form, true, nil +} + +func x86TypedPackedIntegerCompareByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedIntegerCompareProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + firstWidth := form.byteWidth + if form.memory >= 0 { + firstWidth = form.memoryBytes + } + operands := []x86TypedByteOperand{ + {index: form.first, operand: form.source, width: int64(firstWidth), read: true}, + {index: form.second, operand: ins.Args[form.second], width: int64(form.byteWidth), read: true}, + } + if form.mask >= 0 { + operands = append(operands, x86TypedByteOperand{index: form.mask, operand: ins.Args[form.mask], width: 8, read: true}) + } + width := form.byteWidth + if form.maskResult { + width = 8 + } + operands = append(operands, x86TypedByteOperand{ + index: form.destination, operand: ins.Args[form.destination], width: int64(width), write: true, + }) + return x86TypedByteEffect{operands: operands}, true, nil +} diff --git a/amd64_packed_integer_compare_forms_test.go b/amd64_packed_integer_compare_forms_test.go new file mode 100644 index 000000000..596c55162 --- /dev/null +++ b/amd64_packed_integer_compare_forms_test.go @@ -0,0 +1,341 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86PackedIntegerCompareCompleteActualGoRowsAndAttributes(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + read := func(name string) string { + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + return strings.Join(strings.Fields(string(data)), "") + } + legacy, vector := read("asm6.go"), read("avx_optabs.go") + for _, definition := range []string{ + "var ymm = []ytab{ {Zm_r_xm, 1, argList{Ymm, Ymr}}, {Zm_r_xm, 2, argList{Yxm, Yxr}}, }", + "var yxm_q4 = []ytab{ {Zm_r, 1, argList{Yxm, Yxr}}, }", + } { + if !strings.Contains(legacy, strings.Join(strings.Fields(definition), "")) { + t.Fatal("actual selected SDK legacy operand rows changed", definition) + } + } + rows := []string{ + "{zcase:Zvex_rm_v_r,zoffset:2,args:argList{Yxm,Yxr,Yxr}}", + "{zcase:Zvex_rm_v_r,zoffset:2,args:argList{Yym,Yyr,Yyr}}", + "{zcase:Zevex_rm_v_r,zoffset:0,args:argList{YxmEvex,YxrEvex,Yk}}", + "{zcase:Zevex_rm_v_k_r,zoffset:3,args:argList{YxmEvex,YxrEvex,Yknot0,Yk}}", + "{zcase:Zevex_rm_v_r,zoffset:0,args:argList{YymEvex,YyrEvex,Yk}}", + "{zcase:Zevex_rm_v_k_r,zoffset:3,args:argList{YymEvex,YyrEvex,Yknot0,Yk}}", + "{zcase:Zevex_rm_v_r,zoffset:0,args:argList{Yzm,Yzr,Yk}}", + "{zcase:Zevex_rm_v_k_r,zoffset:3,args:argList{Yzm,Yzr,Yknot0,Yk}}", + } + if !strings.Contains(vector, "var_yvpcmpeqb=[]ytab{"+strings.Join(rows, ",")+",}") { + t.Fatal("actual selected SDK ordered eight-row compare table changed") + } + opcodes := []int{0x74, 0x75, 0x76, 0x29, 0x64, 0x65, 0x66, 0x37} + bits := []int{8, 16, 32, 64, 8, 16, 32, 64} + for index, op := range x86PackedIntegerCompareTestNames[:8] { + table, prefix, code := "ymm", "Py1", fmt.Sprintf("0x%x,Pe,0x%x", opcodes[index], opcodes[index]) + if bits[index] == 64 { + table, prefix, code = "yxm_q4", "Pq4", fmt.Sprintf("0x%x", opcodes[index]) + } + if !strings.Contains(legacy, fmt.Sprintf("{A%s,%s,%s,opBytes{%s}}", op, table, prefix, code)) { + t.Fatal("actual selected SDK legacy opcode attributes changed", op) + } + } + for index, op := range x86PackedIntegerCompareTestNames[8:] { + mapName, evexW, broadcast := "0F", "W0", "" + if bits[index] == 64 { + mapName, evexW, broadcast = "0F38", "W1", "|evexBcstN8" + } else if bits[index] == 32 { + broadcast = "|evexBcstN4" + } + var attributes strings.Builder + for _, width := range []int{128, 256} { + fmt.Fprintf(&attributes, "avxEscape|vex%d|vex66|vex%s|vexW0,0x%X,", width, mapName, opcodes[index]) + } + for _, width := range []int{128, 256, 512} { + fmt.Fprintf(&attributes, "avxEscape|evex%d|evex66|evex%s|evex%s,evexN%d%s,0x%X,", width, mapName, evexW, width/8, broadcast, opcodes[index]) + } + if !strings.Contains(vector, "{as:A"+op+",ytab:_yvpcmpeqb,prefix:Pavx,op:opBytes{"+attributes.String()+"}},") { + t.Fatal("actual selected SDK complete five encoding attributes changed", op) + } + } + actual := map[string]int{} + for _, source := range []string{legacy, vector} { + for _, match := range regexp.MustCompile(`\{(?:as:)?A(V?PCMP(?:EQ|GT)[BWLQD]),`).FindAllStringSubmatch(source, -1) { + actual[match[1]]++ + } + } + if len(actual) != 16 || len(x86PackedIntegerCompareSpecs) != 16 { + t.Fatal("complete Go named-family denominator changed", actual) + } + for index, op := range x86PackedIntegerCompareTestNames { + spec := x86PackedIntegerCompareSpecs[Op(op)] + if actual[op] != 1 || spec.laneBits != bits[index%8] || spec.greater != (index%8 >= 4) || spec.legacy != (index < 8) || spec.mmx != (index < 8 && bits[index%8] < 64) { + t.Fatal("Go registration and typed semantics differ", op, spec) + } + } +} + +func TestX86PackedIntegerCompareLogicalFPSpanRequiresEveryByteEvenMaskZero(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames[8:] { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + if broadcast && bits < 32 { + continue + } + span, instruction := width, op + if broadcast { + span, instruction = bits/8, op+".BCST" + } + t.Run(fmt.Sprintf("%s/%d/bcst=%t", op, width, broadcast), func(t *testing.T) { + var parameters strings.Builder + for index := 0; index < span; index++ { + if index != 0 { + parameters.WriteByte(',') + } + fmt.Fprintf(¶meters, "a%d uint8", index) + } + pkg := mustGoPackage(t, "compare", "package compare\nfunc query("+parameters.String()+")\n") + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + source := fmt.Sprintf("TEXT ·query(SB),4,$0-%d\nMOVQ $0,AX\nKMOVQ AX,K1\n%s a0+0(FP),%s1,K1,K2\nRET\n", span, instruction, bank) + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs} + if _, err := Translate(file, options); err != nil { + t.Fatal("complete logical FP fields", err) + } + for missing := 0; missing < span; missing++ { + t.Run(fmt.Sprint(missing), func(t *testing.T) { + sig := sigs["query"] + sig.Frame.Params = append([]FrameSlot(nil), sig.Frame.Params...) + sig.Frame.Params = append(sig.Frame.Params[:missing], sig.Frame.Params[missing+1:]...) + mutated := options + mutated.Sigs = map[string]FuncSig{"query": sig} + if ir, err := Translate(file, mutated); !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("mask0 borrowed absent logical FP byte%d: %v", missing, err) + } + }) + } + }) + } + } + } +} + +func TestX86PackedIntegerCompareMMXActualEncodedBankAndDefinedness(t *testing.T) { + for opIndex, op := range x86PackedIntegerCompareTestNames[:8] { + if opIndex%4 == 3 { + continue + } + opcode := []byte{0x74, 0x75, 0x76, 0x29, 0x64, 0x65, 0x66, 0x37}[opIndex] + for index, spelling := range []string{"M0", "M1", "M2", "M3", "M4", "M5", "M6", "M7", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + t.Run(op+"/"+spelling, func(t *testing.T) { + source := "TEXT probe(SB),4,$0-0\n" + op + " " + spelling + ",M7\nRET\n" + want := []byte{0x0f, opcode, 0xf8 | byte(index&7), 0xc3} + if index >= 16 { + want = append([]byte{0x41}, want...) + } + if got := x86PackedShiftActualGoBytes(t, "amd64", source); !bytes.Equal(got, want) { + t.Fatalf("actual Go GP spelling changed native MMX identity: %x != %x", got, want) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || effect.operands[0].operand.Reg != Reg(fmt.Sprintf("M%d", index&7)) || effect.operands[0].width != 8 { + t.Fatal("pure query confused GP initialization with MMX", effect, err) + } + state := x86ResultState{flags: x86ByteFlagsAll} + for i := range state.gp { + state.gp[i] = 0xff + } + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("numeric GP bytes supplied unproved MMX state") + } + }) + } + } + for _, op := range x86PackedIntegerCompareTestNames[:8] { + ins := Instr{Op: Op(op), Args: []Operand{{Kind: OpReg, Reg: "X1"}, {Kind: OpReg, Reg: "X0"}}} + for input := 0; input < 2; input++ { + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/input%d/byte%d", op, input, missing), func(t *testing.T) { + state := x86ResultState{flags: x86ByteFlagsAll} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + state.xmm[1-input] &^= uint16(1) << uint(missing) + if x86ResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("compare borrowed undefined input byte") + } + }) + } + } + } +} + +type x86PackedIntegerCompareCase struct { + op, operands string + width, bits int + legacy, mmx bool + mask, memory bool + broadcast bool + first, second, destination int +} + +func x86PackedIntegerCompareCases(arch string) []x86PackedIntegerCompareCase { + var cases []x86PackedIntegerCompareCase + for index, op := range x86PackedIntegerCompareTestNames { + bits := []int{8, 16, 32, 64}[index%4] + if index < 8 { + for _, row := range []struct { + operands string + first, second int + memory bool + }{ + {"X0,X1", 0, 1, false}, {"X1,X0", 1, 0, false}, {"X0,X0", 0, 0, false}, {"(BX),X1", 0, 1, true}, + } { + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: row.operands, width: 16, bits: bits, legacy: true, memory: row.memory, first: row.first, second: row.second, destination: row.second}) + } + if arch == "amd64" && bits < 64 { + for _, memory := range []bool{false, true} { + operand := "M0,M1" + if memory { + operand = "(BX),M1" + } + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: operand, width: 8, bits: bits, legacy: true, mmx: true, memory: memory, first: 0, second: 1, destination: 1}) + } + } + continue + } + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + if width < 64 { + for _, alias := range [][3]int{{0, 1, 2}, {0, 1, 0}, {0, 1, 1}, {0, 0, 0}} { + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: fmt.Sprintf("%s%d,%s%d,%s%d", bank, alias[0], bank, alias[1], bank, alias[2]), width: width, bits: bits, first: alias[0], second: alias[1], destination: alias[2]}) + } + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: fmt.Sprintf("(BX),%s1,%s2", bank, bank), width: width, bits: bits, memory: true, first: 0, second: 1, destination: 2}) + } + for _, masked := range []bool{false, true} { + if arch == "386" && masked { + continue + } + for _, memory := range []bool{false, true} { + for _, alias := range [][3]int{{0, 1, 0}, {0, 1, 2}, {1, 0, 1}, {0, 0, 7}} { + first := fmt.Sprintf("%s%d", bank, alias[0]) + if memory { + first = "(BX)" + } + mask := "" + if masked { + mask = "K2," + } + cases = append(cases, x86PackedIntegerCompareCase{op: op, operands: fmt.Sprintf("%s,%s%d,%sK%d", first, bank, alias[1], mask, alias[2]), width: width, bits: bits, mask: masked, memory: memory, first: alias[0], second: alias[1], destination: alias[2]}) + } + if memory && bits >= 32 { + mask := "" + if masked { + mask = "K2," + } + cases = append(cases, x86PackedIntegerCompareCase{op: op + ".BCST", operands: fmt.Sprintf("(BX),%s1,%sK2", bank, mask), width: width, bits: bits, mask: masked, memory: true, broadcast: true, first: 0, second: 1, destination: 2}) + } + } + } + } + } + return cases +} + +func TestX86PackedIntegerCompareThreeAPIsFiveTargetsO0O2Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + cases := x86PackedIntegerCompareCases(target.arch) + for index, row := range cases { + fmt.Fprintf(&source, "TEXT ·compare%d(SB),4,$0-0\n%s %s\nRET\n", index, row.op, row.operands) + fmt.Fprintf(&declarations, "func compare%d()\n", index) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86PackedIntegerCompareAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + x86PackedStringCompileObject(t, llc, target.triple, ir, level) + } + }) + } + t.Logf("%d complete legal physical forms, actual Go object + three API × O0/O2 LLVM objects", len(cases)) + }) + } +} + +func TestX86PackedIntegerCompareDescriptorPermissionsAndMalformedForms(t *testing.T) { + for _, row := range x86PackedIntegerCompareCases("amd64") { + file, err := Parse(ArchAMD64, "TEXT query(SB),4,$0-0\n"+row.op+" "+row.operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86PackedIntegerCompareProperties(ins.Op, ins, "amd64") + if !known || err != nil || form.byteWidth != row.width || form.spec.laneBits != row.bits || form.mmx != row.mmx || form.broadcast != row.broadcast || (form.mask >= 0) != row.mask || (form.memory >= 0) != row.memory { + t.Fatal("complete form lost an independent axis", row, form, err) + } + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, "amd64") + if !known || err != nil || effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatal("pure descriptor lost unchanged flags", effect, err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": 0xff}, masks: map[Reg]bool{"K2": true}} + before := state + before.vectors, before.masks = x86CloneCallerVectorState(state.vectors, state.masks) + if x86CallerVectorMoveKnown(ins) || x86CallerVectorMoveStep(&ins, &state, nil) || x86SourceStackInstructionKnown(ins, OpMOVQ, "Q", "amd64") || !reflect.DeepEqual(state, before) || ins.x86CallerMemory != nil { + t.Fatal("query descriptor invented caller input/permission", row) + } + for _, mutate := range []func(*Instr){ + func(i *Instr) { i.Op += ".Z" }, + func(i *Instr) { i.Args = append(i.Args, Operand{Kind: OpReg, Reg: "X0"}) }, + func(i *Instr) { i.Args[len(i.Args)-1] = Operand{Kind: OpReg, Reg: "AX"} }, + func(i *Instr) { i.Args[0] = Operand{Kind: OpImm, Imm: 3} }, + } { + local := ins + local.Args = append([]Operand(nil), ins.Args...) + mutate(&local) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + before, temporary := output.String(), ctx.tmp + known, _, err := ctx.lowerPackedIntegerCompare(local.Op, local) + if !known || err == nil || output.String() != before || ctx.tmp != temporary { + t.Fatal("malformed form emitted before rejection", row, local, err) + } + } + } +} diff --git a/amd64_packed_integer_compare_go_oracle_test.go b/amd64_packed_integer_compare_go_oracle_test.go new file mode 100644 index 000000000..e5dd18874 --- /dev/null +++ b/amd64_packed_integer_compare_go_oracle_test.go @@ -0,0 +1,277 @@ +package plan9asm + +import ( + "fmt" + "go/format" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func x86PackedIntegerCompareRunOriginalGo(t *testing.T, declaration, assembly, testSource string) { + x86PackedIntegerCompareOriginalGoDriver(t, declaration, assembly, testSource, false) +} + +func x86PackedIntegerCompareOriginalGoDriver(t *testing.T, declaration, assembly, testSource string, buildOnly bool) { + t.Helper() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declaration, + "value_amd64.s": assembly, "value_test.go": testSource, + } { + data := []byte(source) + if strings.HasSuffix(name, ".go") { + var err error + data, err = format.Source(data) + if err != nil { + t.Fatalf("first-party original fixture syntax: %v", err) + } + } + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } + } + driver, _ := x86LEAActualSDKDriver(t) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + arguments := []string{"test", "-p=2", "-vet=off", "-count=1"} + if buildOnly { + arguments = append(arguments, "-run=^$") + } + arguments = append(arguments, ".") + command := exec.CommandContext(ctx, driver, arguments...) + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go packed compare execution: %v\n%s", err, output) + } +} + +func x86PackedIntegerCompareOriginalWideSources() (string, string, string) { + rows := x86PackedIntegerCompareCases("amd64") + declaration, assembly, _ := x86PackedIntegerCompareRuntimeFixture(rows) + var checks strings.Builder + for index, row := range rows { + maskResult := !row.legacy && strings.Contains(row.operands, "K") + fmt.Fprintf(&checks, " checkWide(t,compare%d,%q,%d,%d,%t,%t,%t,%t,%t,%t,%t,%d,%d,%d)\n", + index, fmt.Sprintf("compare%d", index), row.width, row.bits/8, strings.Contains(row.op, "GT"), row.legacy, row.mmx, + maskResult, row.mask, row.memory, row.broadcast, row.first, row.second, row.destination) + } + tests := `package oracle + +import ( + "encoding/binary" + "testing" + "unsafe" +) + +func wideLane(p []byte, bytes int) uint64 { + var result uint64 + for j := 0; j < bytes; j++ { result |= uint64(p[j]) << uint(8*j) } + return result +} + +func checkWide(t *testing.T,fn func(*byte,*byte,*byte,uint64,uint32),name string, + width,bytes int,greater,legacy,mmx,maskResult,masked,memory,broadcast bool, + firstBank,secondBank,destination int) { + t.Helper() + masks := [4]uint64{0,1,0x55aa55aa55aa55aa,^uint64(0)} + for round := 0; round < 12; round++ { + for _,mask := range masks { + var aStorage,bStorage [96]byte + aOffset := int((16-uintptr(unsafe.Pointer(&aStorage[0]))%16)%16) + bOffset := int((16-uintptr(unsafe.Pointer(&bStorage[0]))%16)%16) + for i := range aStorage { aStorage[i],bStorage[i] = 0xa7,0xb9 } + a,b := aStorage[aOffset:aOffset+64],bStorage[bOffset:bOffset+64] + for i := range a { a[i],b[i] = byte(i*17+round*31),byte(i*23+round*53) } + if round < 4 { + patterns := [4][2]byte{{0xff,1},{0x80,0x7f},{1,0},{0,0xff}} + for i := range a { a[i],b[i] = patterns[round][0],patterns[round][1] } + } + originalA,originalB := aStorage,bStorage + var out [424]byte + for i := range out { out[i] = 0xc3 } + csr := uint32(0x1f80 | (round%4)<<13 | round) + fn(&out[1],&a[0],&b[0],mask,csr) + expectedFlags := uint64(0x895) + if mask&3 == 0 { expectedFlags = 0 } + if mask&3 == 1 { expectedFlags = 0x55 } + if binary.LittleEndian.Uint64(out[409:417])&0x8d5 != expectedFlags || out[417] != byte(2|(expectedFlags&0xd5)) || + binary.LittleEndian.Uint32(out[418:422]) != csr || out[0] != 0xc3 || + out[422] != 0xc3 || out[423] != 0xc3 || originalA != aStorage || originalB != bStorage { + t.Fatalf("%s flags/CSR/full input/output canaries: %x",name,out) + } + var expected [3][64]byte + copy(expected[0][:],a) + copy(expected[1][:],b) + copy(expected[2][:],b) + first,second := a,b + if !memory && firstBank != 0 { first = b } + if secondBank == 0 { second = a } + var result [64]byte + var resultMask uint64 + for i := 0; i < width; i += bytes { + offset := i + if broadcast { offset = 0 } + left,right := wideLane(first[offset:],bytes),wideLane(second[i:],bytes) + selected := left == right + if greater { + sign := uint64(1) << uint(bytes*8-1) + selected = (right^sign) > (left^sign) + } + if selected { + for j := 0; j < bytes; j++ { result[i+j] = 0xff } + if !masked || (mask>>uint(i/bytes))&1 != 0 { resultMask |= uint64(1)< (left^sign) + } + want := byte(0) + if selected { want = 0xff } + for j := 0; j < bytes; j++ { + if out[i+j] != want { + t.Fatalf("%s round%d byte%d got%x want%x",name,round,i+j,out[i+j],want) + } + } + } + for i := width; i < 16; i++ { + if out[i] != 0xc3 { t.Fatalf("%s narrow write changed canary",name) } + } + } +} + +func TestOriginalCompare(t *testing.T) { +` + checks.String() + "}\n" + x86PackedIntegerCompareRunOriginalGo(t, declarations.String(), assembly.String(), tests) + t.Logf("%d actual original Go SSE/MMX kernels ×12 initialized numerical/6flag/LAHF/MXCSR/full-buffer cases; MMX bit results do not certify x87 native state", count) +} diff --git a/amd64_packed_integer_compare_linux_amd64_test.go b/amd64_packed_integer_compare_linux_amd64_test.go new file mode 100644 index 000000000..1e679f0cf --- /dev/null +++ b/amd64_packed_integer_compare_linux_amd64_test.go @@ -0,0 +1,43 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedIntegerCompareOriginalNativeWideBits(t *testing.T) { + // No emulator, Rosetta, host-inapplicable Skip, or object-only substitute. + // The shared preflight binds native AMD64 and enabled AVX512F/VL state. + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + dir := t.TempDir() + input, output := filepath.Join(dir, "compare-isa.c"), filepath.Join(dir, "compare-isa") + const program = `#include +#include +#include + +int main(void) { + struct rlimit noCore = {0,0}; + if (setrlimit(RLIMIT_CORE,&noCore)) return 77; + unsigned a,b,c,d; + if (!__get_cpuid_count(7,0,&a,&b,&c,&d)) return 78; + printf("AVX2=%u AVX512BW=%u\n",(b>>5)&1,(b>>30)&1); + return (b&(1u<<5)) && (b&(1u<<30)) ? 0 : 79; +} +` + if err := os.WriteFile(input, []byte(program), 0600); err != nil { + t.Fatal(err) + } + compile := append([]string(nil), execution.compiler...) + compile = append(compile, "-O2", input, "-o", output) + x86FloatCompareCaptureCommand(t, exec.Command(compile[0], compile[1:]...)) + run := execution.command(output) + t.Logf("native packed integer ISA supplement: %s", x86FloatCompareCaptureCommand(t, exec.Command(run[0], run[1:]...))) + declaration, assembly, tests := x86PackedIntegerCompareOriginalWideSources() + x86LEAActualTargetGoObject(t, "amd64", execution.triple, assembly) + x86PackedIntegerCompareRunOriginalGo(t, declaration, assembly, tests) + t.Log("532 native original Go forms ×48 independent scalar lane/full-bank/K/flags/CSR/full-buffer observations; execution requires native Linux AMD64 AVX2+AVX512F/VL/BW") +} diff --git a/amd64_packed_integer_compare_runtime_test.go b/amd64_packed_integer_compare_runtime_test.go new file mode 100644 index 000000000..8e87d9184 --- /dev/null +++ b/amd64_packed_integer_compare_runtime_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Every input is initialized by this first-party fixture. Its declared pointer +// transport is not a caller-region or native-frame permission certificate. +func x86PackedIntegerCompareRuntimeFixture(rows []x86PackedIntegerCompareCase) (string, string, string) { + var declarations, source, prototypes strings.Builder + for index, row := range rows { + name := fmt.Sprintf("compare%d", index) + fmt.Fprintf(&declarations, "func %s(out,a,b *byte, mask uint64, csr uint32)\n", name) + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t);\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-36\nMOVQ out+0(FP),DI\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nLDMXCSR csr+32(FP)\n", name) + source.WriteString("VMOVUPS (BX),Z0\nVMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\n") + for mask := 0; mask < 8; mask++ { + fmt.Fprintf(&source, "KMOVQ DX,K%d\n", mask) + } + if row.mmx { + source.WriteString("MOVQ (BX),M0\nMOVQ (CX),M1\n") + } + source.WriteString("MOVL DX,SI\nANDL $3,SI\nCMPL SI,$0\nJE compare_zero\nCMPL SI,$1\nJE compare_one\n" + + "MOVL $0x7fffffff,SI\nADDL $1,SI\nSTC\nJMP compare_flags_done\n" + + "compare_zero:\nMOVL $1,SI\nADDL $1,SI\nJMP compare_flags_done\n" + + "compare_one:\nMOVL $-1,SI\nADDL $1,SI\ncompare_flags_done:\n") + fmt.Fprintf(&source, "%s %s\n", row.op, row.operands) + for bank := 0; bank < 3; bank++ { + fmt.Fprintf(&source, "MOVOU X%d,%d(DI)\nVMOVUPS Y%d,%d(DI)\nVMOVUPS Z%d,%d(DI)\n", bank, bank*112, bank, bank*112+16, bank, bank*112+48) + } + for mask := 0; mask < 8; mask++ { + fmt.Fprintf(&source, "KMOVQ K%d,%d(DI)\n", mask, 336+mask*8) + } + if row.mmx { + source.WriteString("MOVQ M1,400(DI)\nEMMS\n") + } else { + source.WriteString("MOVQ $0,400(DI)\n") + } + source.WriteString("PUSHFQ\nPOPQ DX\nMOVQ DX,408(DI)\nLAHF\nMOVB AH,416(DI)\nSTMXCSR 417(DI)\nRET\n") + } + return declarations.String(), source.String(), prototypes.String() +} + +const x86PackedIntegerCompareScalarOracle = ` +static uint64_t compareLane(const uint8_t *p, int bytes) { + uint64_t value = 0; + for (int j = 0; j < bytes; ++j) { + value |= (uint64_t)p[j] << (8*j); + } + return value; +} + +static int compareValue(uint64_t first, uint64_t second, int bytes, int greater) { + if (!greater) { + return first == second; + } + uint64_t sign = UINT64_C(1) << (bytes*8-1); + return (second ^ sign) > (first ^ sign); +} + +__attribute__((noinline)) static int checkCompare( + void (*fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint32_t), + const char *name, int width, int bytes, int greater, int legacy, int mmx, + int maskResult, int masked, int memory, int broadcast, + int firstBank, int secondBank, int destination) { + const uint64_t masks[] = {0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; + for (int round = 0; round < 12; ++round) { + for (int m = 0; m < 4; ++m) { + _Alignas(64) uint8_t aStorage[96], bStorage[96], output[424]; + uint8_t *a = aStorage+16; + uint8_t *b = bStorage+16; + memset(aStorage,0xa7,sizeof(aStorage)); + memset(bStorage,0xb9,sizeof(bStorage)); + memset(output,0xc3,sizeof(output)); + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(i*17+round*31); + b[i] = (uint8_t)(i*23+round*53); + } + if (round < 4) { + memset(a,round == 0 ? 0xff : round == 1 ? 0x80 : round == 2 ? 1 : 0,64); + memset(b,round == 0 ? 1 : round == 1 ? 0x7f : round == 2 ? 0 : 0xff,64); + } + uint8_t originalA[96], originalB[96]; + memcpy(originalA,aStorage,96); + memcpy(originalB,bStorage,96); + uint64_t mask = masks[m]; + uint32_t saved, csr = 0x1f80 | ((round%4)<<13) | round, after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(csr)); + fn(output+1,a,b,mask,csr); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + uint64_t expectedFlags = (mask&3) == 0 ? 0 : (mask&3) == 1 ? 0x55 : 0x895; + if (after != csr || compareLane(output+418,4) != csr || + (compareLane(output+409,8)&0x8d5) != expectedFlags || output[417] != (2|(expectedFlags&0xd5)) || + output[0] != 0xc3 || output[422] != 0xc3 || output[423] != 0xc3 || + memcmp(originalA,aStorage,96) || memcmp(originalB,bStorage,96)) { + fprintf(stderr,"%s CSR/flags/full-input/output canary mismatch\n",name); + return 1; + } + uint8_t expected[3][64]; + memcpy(expected[0],a,64); + memcpy(expected[1],b,64); + memcpy(expected[2],b,64); + const uint8_t *first = memory || firstBank == 0 ? a : b; + const uint8_t *second = secondBank == 0 ? a : b; + uint64_t resultMask = 0; + uint8_t result[64] = {0}; + for (int i = 0; i < width; i += bytes) { + int selected = compareValue(compareLane(first+(broadcast ? 0 : i),bytes), + compareLane(second+i,bytes),bytes,greater); + if (selected && (!masked || ((mask>>(i/bytes))&1))) { + resultMask |= UINT64_C(1) << (i/bytes); + } + memset(result+i,selected ? 0xff : 0,bytes); + } + if (!maskResult && !mmx) { + memcpy(expected[destination],result,width); + if (!legacy) { + memset(expected[destination]+width,0,64-width); + } + } + for (int bank = 0; bank < 3; ++bank) { + for (int view = 0; view < 3; ++view) { + int offset = bank*112+(view == 0 ? 0 : view == 1 ? 16 : 48); + int length = view == 0 ? 16 : view == 1 ? 32 : 64; + if (memcmp(output+1+offset,expected[bank],length)) { + fprintf(stderr,"%s round%d mask%llx bank%d view%d mismatch\n", + name,round,(unsigned long long)mask,bank,view); + return 1; + } + } + } + for (int k = 0; k < 8; ++k) { + uint64_t want = maskResult && k == destination ? resultMask : mask; + if (compareLane(output+337+k*8,8) != want) { + fprintf(stderr,"%s K%d mismatch\n",name,k); + return 1; + } + } + for (int i = 0; i < 8; ++i) { + if (output[401+i] != (mmx ? result[i] : 0)) { + fprintf(stderr,"%s MMX bit-result mismatch\n",name); + return 1; + } + } + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedIntegerCompareCompleteScalarSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + rows := x86PackedIntegerCompareCases("amd64") + declarations, source, prototypes := x86PackedIntegerCompareRuntimeFixture(rows) + x86LEAActualTargetGoObject(t, "amd64", triple, source) + ir, err := x86PackedIntegerCompareAPI(t, source, declarations, "amd64", triple, "Go") + if err != nil { + t.Fatal(err) + } + var checks strings.Builder + for index, row := range rows { + maskResult := !row.legacy && strings.Contains(row.operands, "K") + fmt.Fprintf(&checks, " if (checkCompare(compare%d,\"compare%d\",%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)) return 1;\n", + index, index, row.width, row.bits/8, x86FloatAddSubBoolInt(strings.Contains(row.op, "GT")), + x86FloatAddSubBoolInt(row.legacy), x86FloatAddSubBoolInt(row.mmx), x86FloatAddSubBoolInt(maskResult), + x86FloatAddSubBoolInt(row.mask), x86FloatAddSubBoolInt(row.memory), x86FloatAddSubBoolInt(row.broadcast), + row.first, row.second, row.destination) + } + program := "#include \n#include \n#include \n" + + "#if !defined(_WIN32)\n#include \n#endif\n" + + prototypes + x86PackedIntegerCompareScalarOracle + "\nint main(void) {\n" + + "#if !defined(_WIN32)\n struct rlimit noCore = {0,0};\n if (setrlimit(RLIMIT_CORE,&noCore)) return 3;\n#endif\n" + + checks.String() + " return 0;\n}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-integer-compare", triple, text, program, prefix) + }) + } + t.Logf("%d forms ×48 independent signed/equal/mask/full-bank/full flags/CSR/input+output canaries at O0/O2; portable VEX/EVEX, not native AVX512", len(rows)) +} diff --git a/amd64_packed_integer_compare_typed_test.go b/amd64_packed_integer_compare_typed_test.go new file mode 100644 index 000000000..9f58c002e --- /dev/null +++ b/amd64_packed_integer_compare_typed_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86PackedIntegerCompareTestNames = []string{ + "PCMPEQB", "PCMPEQW", "PCMPEQL", "PCMPEQQ", + "PCMPGTB", "PCMPGTW", "PCMPGTL", "PCMPGTQ", + "VPCMPEQB", "VPCMPEQW", "VPCMPEQD", "VPCMPEQQ", + "VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ", +} + +func x86PackedIntegerCompareAPI(t *testing.T, source, declaration, arch, triple, api string) (string, error) { + t.Helper() + pkg := mustGoPackage(t, "compare", "package compare\n"+declaration) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + ctx := llvm.NewContext() + defer ctx.Dispose() + if api == "Go" { + module, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, ResolveSym: resolve, + }) + if module != nil && !module.Module.IsNil() { + defer module.Module.Dispose() + return module.Module.String(), err + } + return "", err + } + file, err := Parse(ArchAMD64, source) + if err != nil { + return "", err + } + signatures, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + return "", err + } + options := Options{Goarch: arch, TargetTriple: triple, ResolveSym: resolve, Sigs: signatures} + if api == "text" { + return Translate(file, options) + } + if api != "module" { + t.Fatal("unknown packed comparison consumer", api) + } + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + return module.String(), err + } + return "", err +} + +func TestX86PackedIntegerCompareMode32HighBankActualSourceNeedsContext(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames[8:] { + for _, operands := range []string{ + "X8,X1,X2", "X1,X8,X2", "X1,X2,X15", + "Y15,Y1,Y2", "Y1,Y15,Y2", "Y1,Y2,Y8", + "X20,X1,K2", "X1,X20,K2", "Y20,Y1,K2", "Y1,Y20,K2", + } { + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(op+"/"+operands+"/"+triple, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-0\n" + op + " " + operands + "\nRET\n" + x86LEAActualTargetGoObject(t, "386", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86PackedIntegerCompareAPI(t, source, "func query()\n", "386", triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("actual Go acceptance fabricated a physical high bank: IR=%d err=%v", len(ir), err) + } + }) + } + }) + } + } + } +} + +func TestX86PackedIntegerCompareDispatchIdentityBeforeEmission(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames { + operands := []Operand{{Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}} + if strings.HasPrefix(op, "V") { + operands = append(operands, Operand{Kind: OpReg, Reg: "X2"}) + } + for _, identity := range []Op{"", Op(op), Op(strings.ToLower(op)), "MOVOU", "UNKNOWN"} { + t.Run(op+"/"+string(identity), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = "amd64" + ins := Instr{Op: identity, Args: operands} + before, temporary := output.String(), ctx.tmp + original := fmt.Sprintf("%#v", ins) + known, terminated, err := ctx.lowerPackedIntegerCompare(Op(op), ins) + invalid := identity != "" && !strings.EqualFold(string(identity), op) + if !known || terminated || (err != nil) != invalid || + invalid && (before != output.String() || temporary != ctx.tmp) || original != fmt.Sprintf("%#v", ins) { + t.Fatalf("dispatch identity changed source or emitted before rejection: %v", err) + } + }) + } + } +} + +func TestX86PackedIntegerCompareLegacyNativeOpcodeAndVirtualBoundaries(t *testing.T) { + for _, op := range x86PackedIntegerCompareTestNames[:8] { + for _, target := range x86RawReturnTargets { + t.Run(op+"/"+target.triple, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-0\n" + op + " (AX),X1\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + ir, err := x86PackedIntegerCompareAPI(t, source, "func query()\n", target.arch, target.triple, "text") + mnemonic := strings.ToLower(strings.ReplaceAll(op, "L", "D")) + if err != nil || !strings.Contains(ir, "asm sideeffect \""+mnemonic+" ($2), $0\"") { + t.Fatalf("legacy m128 erased original opcode/fault semantics: %v\n%s", err, ir) + } + }) + } + for _, operand := range []string{"a+0(FP)", "16(SP)", "16(BP)"} { + t.Run(op+"/virtual/"+operand, func(t *testing.T) { + source := "TEXT ·query(SB),4,$64-16\n" + op + " " + operand + ",X1\nRET\n" + x86LEAActualTargetGoObject(t, "amd64", "x86_64-unknown-linux-gnu", source) + ir, err := x86PackedIntegerCompareAPI(t, source, "func query(a,b uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("virtual fields acquired native legacy m128 alignment: IR=%d err=%v", len(ir), err) + } + }) + } + } +} diff --git a/amd64_packed_logical_forms.go b/amd64_packed_logical_forms.go new file mode 100644 index 000000000..6059a906b --- /dev/null +++ b/amd64_packed_logical_forms.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedLogicalEncoding uint8 + +const ( + x86PackedLogicalLegacyFP x86PackedLogicalEncoding = iota + x86PackedLogicalLegacyInteger + x86PackedLogicalVEX + x86PackedLogicalVEXOrEVEX + x86PackedLogicalEVEX +) + +type x86PackedLogicalSpec struct { + laneBits int + mode amd64PackedLogicalMode + encoding x86PackedLogicalEncoding + // These are opcode attributes, not inferred from a shared ytab. Go's + // avx_optabs supplies N4 for PS/D and N8 for PD/Q, and zeroing for each + // EVEX row. VEX-only integer and legacy forms have neither attribute. + broadcastBytes int + zeroing bool +} + +var x86PackedLogicalSpecs = map[Op]x86PackedLogicalSpec{ + "ANDPS": {32, amd64PackedLogicalAnd, x86PackedLogicalLegacyFP, 0, false}, + "ANDPD": {64, amd64PackedLogicalAnd, x86PackedLogicalLegacyFP, 0, false}, + "ANDNPS": {32, amd64PackedLogicalAndNot, x86PackedLogicalLegacyFP, 0, false}, + "ANDNPD": {64, amd64PackedLogicalAndNot, x86PackedLogicalLegacyFP, 0, false}, + "ORPS": {32, amd64PackedLogicalOr, x86PackedLogicalLegacyFP, 0, false}, + "ORPD": {64, amd64PackedLogicalOr, x86PackedLogicalLegacyFP, 0, false}, + "XORPS": {32, amd64PackedLogicalXor, x86PackedLogicalLegacyFP, 0, false}, + "XORPD": {64, amd64PackedLogicalXor, x86PackedLogicalLegacyFP, 0, false}, + "PAND": {8, amd64PackedLogicalAnd, x86PackedLogicalLegacyInteger, 0, false}, + "PANDN": {8, amd64PackedLogicalAndNot, x86PackedLogicalLegacyInteger, 0, false}, + "POR": {8, amd64PackedLogicalOr, x86PackedLogicalLegacyInteger, 0, false}, + "PXOR": {8, amd64PackedLogicalXor, x86PackedLogicalLegacyInteger, 0, false}, + "VANDPS": {32, amd64PackedLogicalAnd, x86PackedLogicalVEXOrEVEX, 4, true}, + "VANDPD": {64, amd64PackedLogicalAnd, x86PackedLogicalVEXOrEVEX, 8, true}, + "VANDNPS": {32, amd64PackedLogicalAndNot, x86PackedLogicalVEXOrEVEX, 4, true}, + "VANDNPD": {64, amd64PackedLogicalAndNot, x86PackedLogicalVEXOrEVEX, 8, true}, + "VORPS": {32, amd64PackedLogicalOr, x86PackedLogicalVEXOrEVEX, 4, true}, + "VORPD": {64, amd64PackedLogicalOr, x86PackedLogicalVEXOrEVEX, 8, true}, + "VXORPS": {32, amd64PackedLogicalXor, x86PackedLogicalVEXOrEVEX, 4, true}, + "VXORPD": {64, amd64PackedLogicalXor, x86PackedLogicalVEXOrEVEX, 8, true}, + "VPAND": {8, amd64PackedLogicalAnd, x86PackedLogicalVEX, 0, false}, + "VPANDN": {8, amd64PackedLogicalAndNot, x86PackedLogicalVEX, 0, false}, + "VPOR": {8, amd64PackedLogicalOr, x86PackedLogicalVEX, 0, false}, + "VPXOR": {8, amd64PackedLogicalXor, x86PackedLogicalVEX, 0, false}, + "VPANDD": {32, amd64PackedLogicalAnd, x86PackedLogicalEVEX, 4, true}, + "VPANDQ": {64, amd64PackedLogicalAnd, x86PackedLogicalEVEX, 8, true}, + "VPANDND": {32, amd64PackedLogicalAndNot, x86PackedLogicalEVEX, 4, true}, + "VPANDNQ": {64, amd64PackedLogicalAndNot, x86PackedLogicalEVEX, 8, true}, + "VPORD": {32, amd64PackedLogicalOr, x86PackedLogicalEVEX, 4, true}, + "VPORQ": {64, amd64PackedLogicalOr, x86PackedLogicalEVEX, 8, true}, + "VPXORD": {32, amd64PackedLogicalXor, x86PackedLogicalEVEX, 4, true}, + "VPXORQ": {64, amd64PackedLogicalXor, x86PackedLogicalEVEX, 8, true}, +} + +type x86PackedLogicalForm struct { + x86PackedBinaryForm + spec x86PackedLogicalSpec + source Operand +} + +// This is a pure numerical-byte dependency, shared by independent consumers. +// A validated form and initialized mask are prerequisites; the result grants +// no memory, frame, source-selection or caller permission. Higher physical +// bytes are updated separately according to the consumer's encoding model. +func x86PackedLogicalDefinedBytes(ins Instr, form x86PackedLogicalForm, first, second, old uint64) uint64 { + width := x86CallerVectorByteMask(int64(form.byteWidth)) + known := first & second & width + if form.memory < 0 && ins.Args[form.first].Reg == ins.Args[form.second].Reg && + (form.spec.mode == amd64PackedLogicalXor || form.spec.mode == amd64PackedLogicalAndNot) { + known = width + } + if form.mask >= 0 && !form.zeroing { + known &= old // A defined mask is not necessarily all ones. + } + return known +} + +// These fields describe only an operand grammar. Arithmetic, interleave and +// bitwise effects remain in their own typed operation specs and consumers. +type x86PackedBinarySpec struct { + encoding x86PackedLogicalEncoding + broadcastBytes int + zeroing bool +} + +type x86PackedBinaryForm struct { + byteWidth, first, second, destination int + mask, memory, memoryBytes int + legacy, mmx, broadcast, zeroing bool +} + +// Go accepts some X/Y high-bank spellings in mode32 without proving their +// physical register identity. Apply this only after the family's syntax has +// been validated; it grants no initialized values, ABI or memory permissions. +func x86Mode32VectorRegisterContext(ins Instr, goarch string) error { + if goarch != "386" { + return nil + } + for _, operand := range ins.Args { + index, valid := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + if operand.Kind == OpReg && valid && index >= 8 { + return fmt.Errorf("%w: 386 %s register %s has no proved physical mode32 binding", ErrProbeNeedsContext, ins.Op, operand.Reg) + } + } + return nil +} + +// This grammar supplies no initialized values, address bounds or caller +// privileges. The emitter and the private source/CFG proof share only forms. +func x86PackedLogicalProperties(ins Instr, goarch string) (x86PackedLogicalForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86PackedLogicalSpecs[Op(base)] + if !recognized { + return x86PackedLogicalForm{}, false, nil + } + local := ins + if spec.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(ins, goarch) + if err != nil { + return x86PackedLogicalForm{}, true, err + } + } + form, err := x86PackedBinaryProperties(local, goarch, x86PackedBinarySpec{ + encoding: spec.encoding, broadcastBytes: spec.broadcastBytes, zeroing: spec.zeroing, + }) + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + result := x86PackedLogicalForm{x86PackedBinaryForm: form, spec: spec} + if err == nil { + result.source = local.Args[form.first] + } + return result, true, err +} + +func x86PackedBinaryProperties(ins Instr, goarch string, spec x86PackedBinarySpec) (x86PackedBinaryForm, error) { + _, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + form := x86PackedBinaryForm{first: 0, second: 1, destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86PackedBinaryForm, error) { + return form, fmt.Errorf("%s %s operands/suffix outside Go packed binary grammar: %q", goarch, ins.Op, ins.Raw) + } + if goarch != "386" && goarch != "amd64" { + return fail() + } + form.legacy = spec.encoding == x86PackedLogicalLegacyFP || spec.encoding == x86PackedLogicalLegacyInteger + if form.legacy { + if dotted || len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return fail() + } + form.byteWidth = 16 + _, form.mmx = amd64ParseMReg(ins.Args[1].Reg) + if form.mmx { + if spec.encoding != x86PackedLogicalLegacyInteger || goarch != "amd64" { + return fail() + } + form.byteWidth = 8 + } + } else { + switch suffix { + case "": + if dotted { + return fail() + } + case "Z": + form.zeroing = true + case "BCST": + form.broadcast = true + case "BCST.Z": + form.broadcast, form.zeroing = true, true + default: + return fail() + } + if len(ins.Args) != 3 && len(ins.Args) != 4 || form.broadcast && spec.broadcastBytes == 0 || form.zeroing && !spec.zeroing { + return fail() + } + if len(ins.Args) == 4 { + if goarch == "386" || spec.encoding == x86PackedLogicalVEX || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + form.mask = 2 + } else if form.zeroing { + return fail() + } + form.byteWidth = amd64VectorByteWidth(ins.Args[form.destination].Reg) + } + register := func(operand Operand) bool { + if form.mmx { + _, valid := amd64ParseMReg(operand.Reg) + return operand.Kind == OpReg && valid + } + if form.legacy { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + if spec.encoding == x86PackedLogicalVEX { + return (form.byteWidth == 16 || form.byteWidth == 32) && amd64VEXVectorRegister(operand, form.byteWidth) + } + return amd64GoPackedVectorMoveRegister(operand, form.byteWidth, goarch) + } + if !register(ins.Args[form.destination]) || !register(ins.Args[form.second]) { + return fail() + } + first := ins.Args[form.first] + if isAMD64MemoryOperand(first) { + if first.Kind == OpSym { + symbol := strings.TrimSpace(first.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if first.Kind == OpMem { + // Go asmandsz requires signed disp32 in amd64 mode. Its historical + // 386 rule additionally admits uint32 spellings for the same bits; + // packed logical rows have no LEAL-only overflow exception. + displacement := first.Mem.Off + if displacement != int64(int32(displacement)) && + (goarch != "386" || displacement != int64(uint32(displacement))) { + return fail() + } + if !x86MemoryRegistersValidForArch(first.Mem, goarch) || first.Mem.Index != "" && + (first.Mem.Index == SP || first.Mem.Scale != 1 && first.Mem.Scale != 2 && first.Mem.Scale != 4 && first.Mem.Scale != 8) { + return fail() + } + } + form.memory, form.memoryBytes = form.first, form.byteWidth + if form.broadcast { + form.memoryBytes = spec.broadcastBytes + } + } else if form.broadcast || !register(first) { + return fail() + } + return form, nil +} diff --git a/amd64_packed_logical_forms_test.go b/amd64_packed_logical_forms_test.go new file mode 100644 index 000000000..87bd76873 --- /dev/null +++ b/amd64_packed_logical_forms_test.go @@ -0,0 +1,360 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" +) + +func x86LogicalOpcodeNames() []Op { + names := make([]Op, 0, len(x86PackedLogicalSpecs)) + for opcode := range x86PackedLogicalSpecs { + names = append(names, opcode) + } + sort.Slice(names, func(i, j int) bool { return names[i] < names[j] }) + return names +} + +func TestX86PackedLogicalSpecsBindEveryGoOpcodeAttribute(t *testing.T) { + if len(x86PackedLogicalSpecs) != 32 { + t.Fatal("incomplete packed logical family") + } + if !goToolchainAtLeast(runtime.Version(), 1, 27) { + return + } + dir := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86") + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, opcode := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[opcode] + if spec.encoding <= x86PackedLogicalLegacyInteger { + row := "{A" + string(opcode) + ", " + start := strings.Index(string(legacy), row) + if start < 0 { + t.Fatalf("Go omits %s", opcode) + } + line := strings.SplitN(string(legacy)[start:], "\n", 2)[0] + table := "yxm" + if spec.encoding == x86PackedLogicalLegacyInteger { + table = "ymm" + } + if !strings.Contains(line, ", "+table+",") { + t.Fatalf("Go legacy row differs: %s", line) + } + continue + } + prefix := "{as: A" + string(opcode) + ", ytab: " + start := strings.Index(string(avx), prefix) + if start < 0 { + t.Fatalf("Go omits %s", opcode) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + table := "_yvandnpd" + if spec.encoding == x86PackedLogicalVEX { + table = "_yvaddsubpd" + } + if spec.encoding == x86PackedLogicalEVEX { + table = "_yvblendmpd" + } + if !strings.HasPrefix(row, prefix+table+",") { + t.Fatalf("Go ytab changed for %s", opcode) + } + if spec.encoding == x86PackedLogicalVEX { + if strings.Contains(row, "evex") || spec.broadcastBytes != 0 || spec.zeroing { + t.Fatalf("VEX attributes changed: %s", row) + } + continue + } + for _, line := range strings.Split(row, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + if !strings.Contains(line, fmt.Sprintf("evexBcstN%d", spec.broadcastBytes)) || !strings.Contains(line, "evexZeroingEnabled") || + strings.Contains(line, "evexSae") || strings.Contains(line, "evexRounding") { + t.Fatalf("%s opcode attributes disagree with Go: %s", opcode, line) + } + } + } +} + +// Rows come from the actual yxm/ymm, _yvaddsubpd, _yvandnpd and +// _yvblendmpd tables; memory and all opcode-specific suffix axes are included. +func x86LogicalAcceptedForms(opcode Op, arch string) []string { + spec := x86PackedLogicalSpecs[opcode] + if spec.encoding <= x86PackedLogicalLegacyInteger { + last := 15 + if arch == "386" { + last = 7 + } + rows := []string{fmt.Sprintf("%s X0,X%d", opcode, last), fmt.Sprintf("%s 8(AX),X%d", opcode, last)} + if spec.encoding == x86PackedLogicalLegacyInteger && arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s M0,M7", opcode), fmt.Sprintf("%s 8(AX),M7", opcode)) + } + return rows + } + var rows []string + widths := []string{"X", "Y", "Z"} + if spec.encoding == x86PackedLogicalVEX { + widths = widths[:2] + } + for _, width := range widths { + last := 31 + if spec.encoding == x86PackedLogicalVEX { + last = 15 + } + if arch == "386" && width == "Z" { + last = 7 + } + dest := fmt.Sprintf("%s%d", width, last) + rows = append(rows, fmt.Sprintf("%s %s0,%s1,%s", opcode, width, width, dest), fmt.Sprintf("%s 8(AX),%s1,%s", opcode, width, dest)) + if spec.broadcastBytes == 0 { + continue + } + rows = append(rows, fmt.Sprintf("%s.BCST 8(AX),%s1,%s", opcode, width, dest)) + if arch == "386" { + continue + } + rows = append(rows, + fmt.Sprintf("%s %s0,%s1,K7,%s", opcode, width, width, dest), + fmt.Sprintf("%s.Z %s0,%s1,K1,%s", opcode, width, width, dest), + fmt.Sprintf("%s 8(AX),%s1,K7,%s", opcode, width, dest), + fmt.Sprintf("%s.Z 8(AX),%s1,K1,%s", opcode, width, dest), + fmt.Sprintf("%s.BCST 8(AX),%s1,K7,%s", opcode, width, dest), + fmt.Sprintf("%s.BCST.Z 8(AX),%s1,K1,%s", opcode, width, dest)) + } + return rows +} + +func TestX86PackedLogicalCompleteGoRowsAndLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source, goSource strings.Builder + source.WriteString("TEXT logicalforms(SB),$0-0\n") + goSource.WriteString("TEXT logicalforms(SB),$0-0\n") + for _, op := range x86LogicalOpcodeNames() { + for _, row := range x86LogicalAcceptedForms(op, target.arch) { + fmt.Fprintln(&goSource, row) + if target.arch == "386" { + // Preserve Go's full source grammar independently. LLVM + // object generation requires physically bound registers. + row = strings.NewReplacer("X31", "X7", "Y31", "Y7", "X15", "X7", "Y15", "Y7").Replace(row) + } + fmt.Fprintln(&source, row) + parsed, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, recognized, err := x86PackedLogicalProperties(parsed.Funcs[0].Instrs[1], target.arch); !recognized || err != nil { + t.Fatalf("%s: %v", row, err) + } + } + } + source.WriteString("RET\n") + goSource.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, goSource.String(), true) + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"logicalforms": {Name: "logicalforms", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "logical.ll", "logical.o", ir) + }) + } +} + +func TestX86PackedLogicalRejectsInvalidFormsBeforeEmission(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, opcode := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[opcode] + args := "X0,X1,X2" + if spec.encoding <= x86PackedLogicalLegacyInteger { + args = "X0,X1" + } + rows := []string{string(opcode) + ".SAE " + args, string(opcode) + ".RN_SAE " + args, string(opcode) + ".Z.Z " + args, + string(opcode) + " $3," + args, string(opcode) + " X0,(AX)", string(opcode) + " X0,Y1", string(opcode) + " X0,X1,X2,X3"} + memoryTail := ",X0,X1" + if spec.encoding <= x86PackedLogicalLegacyInteger { + memoryTail = ",X0" + } + rows = append(rows, string(opcode)+" (AX)(CX*3)"+memoryTail, string(opcode)+" (AX)(SP*1)"+memoryTail) + if spec.encoding <= x86PackedLogicalLegacyInteger { + rows = append(rows, string(opcode)+" X16,X0", string(opcode)+".BCST 8(AX),X0") + if spec.encoding == x86PackedLogicalLegacyFP { + rows = append(rows, string(opcode)+" M0,M1") + } + } else { + rows = append(rows, string(opcode)+".Z X0,X1,X2", string(opcode)+" X0,X1,K0,X2", string(opcode)+".BCST X0,X1,X2", + string(opcode)+".Z.BCST (AX),X0,K1,X1") + if spec.encoding == x86PackedLogicalVEX { + rows = append(rows, string(opcode)+" Z0,Z1,Z2", string(opcode)+" X16,X1,X2", string(opcode)+".BCST (AX),X0,X1", string(opcode)+" X0,X1,K1,X2") + } + } + if arch == "386" { + rows = append(rows, string(opcode)+" 8(R9),"+args) + } + for _, row := range rows { + t.Run(arch+"/"+row, func(t *testing.T) { + assembly := "TEXT bad(SB),$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, arch, assembly, false) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + return + } + instruction := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(instruction.Op, instruction) + if !ok || term || err == nil || out.String() != before { + t.Fatalf("invalid form emitted (%v,%v,%v)", ok, term, err) + } + }) + } + } + } +} + +func TestX86PackedLogicalDispatchIdentityBeforeEmission(t *testing.T) { + for _, opcode := range x86LogicalOpcodeNames() { + for _, sourceOp := range []Op{"", opcode, Op(strings.ToLower(string(opcode))), "UNKNOWN", "MOVQ"} { + row := x86LogicalAcceptedForms(opcode, "amd64")[0] + file, err := Parse(ArchAMD64, "TEXT one(SB),$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + instruction.Op = sourceOp + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(opcode, instruction) + if sourceOp == "UNKNOWN" || sourceOp == "MOVQ" { + if !ok || term || err == nil || out.String() != before { + t.Fatalf("%s/%s conflicting helper identity emitted", opcode, sourceOp) + } + } else if !ok || term || err != nil { + t.Fatalf("%s/%s helper compatibility lost: %v", opcode, sourceOp, err) + } + if instruction.Op != sourceOp { + t.Fatal("helper rewrote source identity") + } + } + } +} + +func TestX86PackedLogicalSymbolAddressIsNotMemory(t *testing.T) { + for _, opcode := range x86LogicalOpcodeNames() { + t.Run(string(opcode), func(t *testing.T) { + args := "X0,X1" + if x86PackedLogicalSpecs[opcode].encoding <= x86PackedLogicalLegacyInteger { + args = "X0" + } + source := "TEXT bad(SB),$0-0\n" + string(opcode) + " $symbol(SB)," + args + "\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + instruction := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(instruction.Op, instruction) + if !ok || term || err == nil || out.String() != before { + t.Fatalf("symbol address acquired memory load/emission: (%v,%v,%v)", ok, term, err) + } + }) + } +} + +func TestX86PackedLogicalMemoryDisplacementMatchesGoEncoding(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, opcode := range x86LogicalOpcodeNames() { + args := "X0,X1" + if x86PackedLogicalSpecs[opcode].encoding <= x86PackedLogicalLegacyInteger { + args = "X0" + } + invalid := []int64{-2147483649, 4294967296} + if arch == "amd64" { + invalid = append(invalid, 2147483648, 4294967295) + } + for _, displacement := range invalid { + t.Run(fmt.Sprintf("%s/%s/%d", arch, opcode, displacement), func(t *testing.T) { + source := fmt.Sprintf("TEXT bad(SB),$0-0\n%s %d(AX)(CX*2),%s\nRET\n", opcode, displacement, args) + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := out.String() + ok, term, err := ctx.lowerPackedLogicalFamily(instruction.Op, instruction) + if !ok || term || err == nil || out.String() != before { + t.Fatalf("unencodable displacement emitted: (%v,%v,%v)", ok, term, err) + } + }) + } + } + } +} + +func TestX86PackedLogicalBoundaryDisplacementsGoAndLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT boundaries(SB),$0-0\n") + displacements := []int64{-2147483648, 2147483647} + if target.arch == "386" { + displacements = append(displacements, 2147483648, 4294967295) + } + for _, opcode := range x86LogicalOpcodeNames() { + args := "X0,X1" + if x86PackedLogicalSpecs[opcode].encoding <= x86PackedLogicalLegacyInteger { + args = "X0" + } + for _, displacement := range displacements { + fmt.Fprintf(&source, "%s %d(AX)(CX*2),%s\n", opcode, displacement, args) + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"boundaries": {Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "boundary.ll", "boundary.o", ir) + }) + } +} diff --git a/amd64_packed_logical_oracle_test.go b/amd64_packed_logical_oracle_test.go new file mode 100644 index 000000000..5c81c0f6d --- /dev/null +++ b/amd64_packed_logical_oracle_test.go @@ -0,0 +1,73 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestX86LogicalRuntimeBatchesRetainCompleteInventory(t *testing.T) { + fixtures := x86LogicalRuntimeFixtures(t) + if len(fixtures) != 472 { + t.Fatalf("complete opcode/width/memory/mask inventory: got %d, want 472", len(fixtures)) + } + seen := make(map[string]bool, len(fixtures)) + for index, batch := range x86LogicalRuntimeBatches(fixtures) { + if len(batch) == 0 || len(batch) > 64 { + t.Errorf("batch %d has %d fixtures, require 1..64", index, len(batch)) + } + for _, fixture := range batch { + if seen[fixture.name] { + t.Fatalf("duplicate fixture %s", fixture.name) + } + seen[fixture.name] = true + } + } + if len(seen) != len(fixtures) { + t.Fatalf("batch union lost fixtures: got %d, want %d", len(seen), len(fixtures)) + } + for _, fixture := range fixtures { + if !seen[fixture.name] { + t.Errorf("missing fixture %s", fixture.name) + } + } +} + +// This is the real numerical oracle with a deliberately wrong implementation, +// not an instruction-semantics pass. Its former return code was 256, which a +// Unix parent observes as success. A large case ID must remain diagnostic data. +func TestX86LogicalRuntimeOracleCannotWrapFailureToSuccess(t *testing.T) { + fixture := x86LogicalRuntimeFixtures(t)[245] + program := x86LogicalRuntimeC([]x86LogicalRuntimeFixture{fixture}) + fmt.Sprintf(` +void %s(uint8_t *out, const uint8_t *a, const uint8_t *b, uint64_t mask) { + memset(out, 0, 65); +} +`, fixture.name) + requireX86GeneratedOracleFailure(t, program, fixture.name+" numerical/canary failure") +} + +func requireX86GeneratedOracleFailure(t *testing.T, program, diagnostic string) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + dir := t.TempDir() + source, executable := filepath.Join(dir, "mutation.c"), filepath.Join(dir, "mutation") + if err := os.WriteFile(source, []byte(program), 0600); err != nil { + t.Fatal(err) + } + execution.compileC(t, source, executable) + arguments := execution.command(executable) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, arguments[0], arguments[1:]...)) + var failure *exec.ExitError + if !errors.As(err, &failure) || failure.ExitCode() != 1 { + t.Fatalf("wrong result must terminate with failure 1, got %v; output=%s", err, output) + } + if !strings.Contains(string(output), diagnostic) { + t.Fatalf("failure diagnostic lost the exact fixture: %s", output) + } +} diff --git a/amd64_packed_logical_runtime_test.go b/amd64_packed_logical_runtime_test.go new file mode 100644 index 000000000..f5949c65c --- /dev/null +++ b/amd64_packed_logical_runtime_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// The numerical oracle derives semantics from the source mnemonic, not the +// emitter's mode field; a wrong opcode-to-mode registration must still fail. +func x86LogicalIndependentCOperator(t *testing.T, opcode Op) string { + t.Helper() + name := string(opcode) + switch { + case strings.Contains(name, "ANDN"): + return "aByte & (uint8_t)~b[i]" + case strings.Contains(name, "AND"): + return "aByte & b[i]" + case strings.Contains(name, "XOR"): + return "aByte ^ b[i]" + case strings.Contains(name, "OR"): + return "aByte | b[i]" + default: + t.Fatalf("unknown numerical oracle opcode %s", opcode) + return "" + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedLogicalCompleteSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires Linux/amd64 runtime counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 llc/clang missing") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + fixtures := x86LogicalRuntimeFixtures(t) + for index, batch := range x86LogicalRuntimeBatches(fixtures) { + t.Run(fmt.Sprintf("batch-%02d", index), func(t *testing.T) { + var assembly strings.Builder + sigs := make(map[string]FuncSig, len(batch)) + for _, fixture := range batch { + assembly.WriteString(fixture.assembly) + sigs[fixture.name] = fixture.signature + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "logical_complete_semantics", + triple, ir, x86LogicalRuntimeC(batch), prefix) + }) + } + t.Logf("%d exact opcode/width/memory/mask programs; 16 independently expected numerical/canary inputs each", len(fixtures)) +} + +type x86LogicalRuntimeFixture struct { + name, assembly, declaration, checks string + signature FuncSig +} + +func x86LogicalRuntimeBatches(fixtures []x86LogicalRuntimeFixture) [][]x86LogicalRuntimeFixture { + const batchSize = 64 + var batches [][]x86LogicalRuntimeFixture + for start := 0; start < len(fixtures); start += batchSize { + end := start + batchSize + if end > len(fixtures) { + end = len(fixtures) + } + batches = append(batches, fixtures[start:end]) + } + return batches +} + +func x86LogicalRuntimeC(fixtures []x86LogicalRuntimeFixture) string { + var declarations, checks, calls strings.Builder + for _, fixture := range fixtures { + declarations.WriteString(fixture.declaration) + // Keep independent reference loops out of one giant optimizer graph. + // noinline also prevents -O2 from reconstructing that graph in main. + fmt.Fprintf(&checks, "static __attribute__((noinline)) int check_%s(void) {\n", fixture.name) + checks.WriteString("const uint64_t masks[]={0,1,0x55aa55aa55aa55aaULL,~0ULL};\n") + checks.WriteString(fixture.checks) + checks.WriteString("return 0;\n}\n") + // Case IDs belong in diagnostics, not process exit codes: Unix keeps + // only eight bits and would report return 256 as successful execution. + fmt.Fprintf(&calls, "if (check_%s()) { fprintf(stderr, \"%s numerical/canary failure\\n\"); return 1; }\n", + fixture.name, fixture.name) + } + return "#include \n#include \n#include \n" + + declarations.String() + checks.String() + + "int main(void) {\n" + calls.String() + "return 0;\n}\n" +} + +func x86LogicalRuntimeFixtures(t *testing.T) []x86LogicalRuntimeFixture { + t.Helper() + var fixtures []x86LogicalRuntimeFixture + count := 0 + for _, opcode := range x86LogicalOpcodeNames() { + spec := x86PackedLogicalSpecs[opcode] + widths := []int{16, 32, 64} + if spec.encoding <= x86PackedLogicalLegacyInteger { + widths = widths[:1] + } + if spec.encoding == x86PackedLogicalVEX { + widths = widths[:2] + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, memory := range []string{"register", "memory", "broadcast"} { + if memory == "broadcast" && spec.broadcastBytes == 0 { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if mode != "unmasked" && spec.broadcastBytes == 0 { + continue + } + var assembly, declarations, checks strings.Builder + name := fmt.Sprintf("logical%d", count) + count++ + legacy := spec.encoding <= x86PackedLogicalLegacyInteger + first := bank + "1" + if memory != "register" { + first = "(BX)" + } + operation := string(opcode) + if memory == "broadcast" { + operation += ".BCST" + } + if mode == "zero" { + operation += ".Z" + } + row := operation + " " + first + "," + bank + "0" + destination := 2 + if legacy { + destination = 0 + } else { + if mode != "unmasked" { + row += ",K1" + } + row += "," + bank + "2" + } + fmt.Fprintf(&assembly, "TEXT %s(SB),$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (CX),Z2\n", name) + if memory == "register" { + assembly.WriteString("VMOVUPS (BX),Z1\n") + } + assembly.WriteString("KMOVQ DX,K1\nSTC\n" + row + "\n") + fmt.Fprintf(&assembly, "VMOVUPS Z%d,(AX)\nSETCS 64(AX)\nRET\n", destination) + signature := FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + operator := x86LogicalIndependentCOperator(t, opcode) + maskExpression := "1" + if mode != "unmasked" { + maskExpression = fmt.Sprintf("((mask>>(i/%d))&1)", spec.laneBits/8) + } + inactive := "b[i]" + if mode == "zero" { + inactive = "0" + } + upper := "0" + if legacy { + upper = "b[i]" + } + index := "i" + if memory == "broadcast" { + index = fmt.Sprintf("i%%%d", spec.broadcastBytes) + } + aOffset := 1 + if legacy && memory == "memory" { + // The original legacy m128 instruction requires 16-byte + // alignment. Keep other source forms deliberately unaligned; + // the independent native fault root checks legacy misalignment. + aOffset = 16 + } + fmt.Fprintf(&checks, ` + for(int round=0;round<4;++round) for(int m=0;m<4;++m) { + uint64_t mask=masks[m]; _Alignas(16) uint8_t aStorage[96]; uint8_t bStorage[68],output[67]; uint8_t *a=aStorage+%[9]d,*b=bStorage+3; + memset(aStorage,0xa7,96); memset(bStorage,0xb9,68); memset(output,0xc3,67); + for(int i=0;i<64;++i) { a[i]=(uint8_t)(i*19+round*83+7); b[i]=(uint8_t)(i*31+round*59+13); } + %[1]s(output+1,a,b,mask); + if(output[0]!=0xc3 || output[66]!=0xc3 || output[65]!=1) return %[2]d; + for(int i=0;i<64;++i) { + uint8_t aByte=a[%[3]s]; uint8_t want=i<%[4]d ? ((%[5]s) ? (%[6]s) : (%[7]s)) : (%[8]s); + if(output[i+1]!=want || a[i]!=(uint8_t)(i*19+round*83+7) || b[i]!=(uint8_t)(i*31+round*59+13)) return %[2]d; + } + for(int i=0;i<96;++i) if((i<%[9]d || i>=%[9]d+64) && aStorage[i]!=0xa7) return %[2]d; + if(bStorage[2]!=0xb9 || bStorage[67]!=0xb9) return %[2]d; + } +`, name, count+10, index, width, maskExpression, operator, inactive, upper, aOffset) + if mode != "unmasked" && memory != "register" { + // A zero relevant mask suppresses every memory read, + // including broadcast. NULL is never dereferenced. + fmt.Fprintf(&checks, `{ uint8_t b[64],out[65]; memset(b,0x5a,64); %[1]s(out,0,b,0); + for(int i=0;i<64;++i) { uint8_t want=i<%[2]d ? (%[3]s) : 0; if(out[i]!=want) return %[4]d; } } +`, name, width, map[string]string{"merge": "0x5a", "zero": "0"}[mode], count+10) + } + fixtures = append(fixtures, x86LogicalRuntimeFixture{ + name: name, assembly: assembly.String(), declaration: declarations.String(), + checks: checks.String(), signature: signature, + }) + } + } + } + } + return fixtures +} + +func TestX86PackedLogicalXWritesClearWiderAliases(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires amd64 or Rosetta; Linux counterpart executes the same oracle") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple, prefix = "x86_64-apple-darwin", []string{"/usr/bin/arch", "-x86_64"} + } + for _, opcode := range []string{"VXORPS", "VPXOR", "VPXORD"} { + t.Run(opcode, func(t *testing.T) { + source := "TEXT logicalalias(SB),$0-16\nMOVQ out+0(FP),AX\nMOVQ in+8(FP),BX\nVMOVUPS (BX),Z0\n" + opcode + " X0,X0,X0\nVMOVUPS Z0,(AX)\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{ + "logicalalias": {Name: "logicalalias", Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}}, + }}) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void logicalalias(uint8_t *, const uint8_t *); +int main(void) { + uint8_t in[66], out[66]; + for (int k=0; k<66; ++k) { in[k]=(uint8_t)(k*19+7); out[k]=0xa5; } + logicalalias(out+1,in+1); + if(out[0]!=0xa5 || out[65]!=0xa5) return 2; + for(int k=1; k<65; ++k) if(out[k]!=0) return k+10; + for(int k=0; k<66; ++k) if(in[k]!=(uint8_t)(k*19+7)) return 3; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, strings.ToLower(opcode)+"_alias", triple, ir, mainC, prefix) + }) + } +} diff --git a/amd64_packed_low_multiply_forms.go b/amd64_packed_low_multiply_forms.go new file mode 100644 index 000000000..f0ea55f40 --- /dev/null +++ b/amd64_packed_low_multiply_forms.go @@ -0,0 +1,65 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedLowMultiplySpec struct { + x86PackedBinarySpec + laneBits int +} + +// PMULLW/VPMULLW already use the word family's typed grammar, including +// its distinct MMX source bank. D/Q share that binary grammar, not its bank. +var x86PackedLowMultiplySpecs = map[Op]x86PackedLowMultiplySpec{ + "PMULLD": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 32}, + "VPMULLD": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32}, + "VPMULLQ": {x86PackedBinarySpec{encoding: x86PackedLogicalEVEX, broadcastBytes: 8, zeroing: true}, 64}, +} + +type x86PackedLowMultiplyForm struct { + x86PackedBinaryForm + spec x86PackedLowMultiplySpec +} + +func x86PackedLowMultiplyProperties(op Op, ins Instr, arch string) (x86PackedLowMultiplyForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86PackedLowMultiplySpecs[Op(base)] + if !known { + return x86PackedLowMultiplyForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedLowMultiplyForm{}, true, fmt.Errorf("packed low multiply dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + grammarArch := arch + if arch == "386" && local.x86Encoded && spec.encoding != x86PackedLogicalLegacyFP { + // Only the decoder can supply this physical identity. Go's named + // mode32 frontend has a separate three-operand limit. + grammarArch = "amd64" + } + binary, err := x86PackedBinaryProperties(local, grammarArch, spec.x86PackedBinarySpec) + if err == nil && grammarArch != arch && binary.memory >= 0 && !x86PackedShiftMemoryValid(local.Args[binary.memory], arch) { + err = fmt.Errorf("386 %s physical source has an unbound address", op) + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && binary.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: packed low multiply has no bound address-size override", ErrProbeNeedsContext) + } + return x86PackedLowMultiplyForm{binary, spec}, true, err +} + +func x86TypedPackedLowMultiplyByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, known, err := x86PackedLowMultiplyProperties(op, ins, arch) + if !known || err != nil || !form.legacy { + return x86TypedByteEffect{}, known && (err != nil || form.legacy), err + } + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: form.first, operand: ins.Args[form.first], width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), read: true, write: true}, + }}, true, nil +} diff --git a/amd64_packed_low_multiply_forms_test.go b/amd64_packed_low_multiply_forms_test.go new file mode 100644 index 000000000..64f9203bb --- /dev/null +++ b/amd64_packed_low_multiply_forms_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86PackedLowMultiplyCompleteGoEncoderRows(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APMULLD, yxm_q4, Pq4, opBytes{0x40}}", + "{Zm_r, 1, argList{Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual legacy Go row changed", row) + } + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if len(x86PackedLowMultiplySpecs) != 3 { + t.Fatal("incomplete D/Q low-product inventory") + } + for _, name := range []string{"VPMULLD", "VPMULLQ"} { + table, w, broadcast, rows := "_yvandnpd", 0, 4, 5 + if name == "VPMULLQ" { + table, w, broadcast, rows = "_yvblendmpd", 1, 8, 3 + } + start := strings.Index(string(avx), "{as: A"+name+",") + if start < 0 { + t.Fatal("missing Go opcode", name) + } + record := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(record, "ytab: "+table+", prefix: Pavx") || strings.Count(record, "avxEscape") != rows { + t.Fatal("Go opcode row inventory changed", record) + } + if name == "VPMULLD" { + for _, width := range []int{128, 256} { + want := fmt.Sprintf("avxEscape | vex%d | vex66 | vex0F38 | vexW0, 0x40,", width) + if !strings.Contains(record, want) { + t.Fatal("Go VEX attributes changed", want) + } + } + } + for _, width := range []int{128, 256, 512} { + want := fmt.Sprintf("avxEscape | evex%d | evex66 | evex0F38 | evexW%d, evexN%d | evexBcstN%d | evexZeroingEnabled, 0x40,", width, w, width/8, broadcast) + if !strings.Contains(record, want) { + t.Fatal("Go EVEX attributes changed", want) + } + } + start = strings.Index(string(avx), "var "+table+" =") + if start < 0 { + t.Fatal("missing Go operand table", table) + } + operands := strings.SplitN(string(avx)[start:], "\n}", 2)[0] + var expected []string + if name == "VPMULLD" { + expected = append(expected, + "{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr, Yxr}}", + "{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yym, Yyr, Yyr}}") + } + for _, classes := range [][2]string{{"YxmEvex", "YxrEvex"}, {"YymEvex", "YyrEvex"}, {"Yzm", "Yzr"}} { + expected = append(expected, + fmt.Sprintf("{zcase: Zevex_rm_v_r, zoffset: 0, args: argList{%s, %s, %s}}", classes[0], classes[1], classes[1]), + fmt.Sprintf("{zcase: Zevex_rm_v_k_r, zoffset: 3, args: argList{%s, %s, Yknot0, %s}}", classes[0], classes[1], classes[1])) + } + if strings.Count(operands, "argList{") != len(expected) { + t.Fatal("Go operand row inventory changed", operands) + } + for _, row := range expected { + if !strings.Contains(operands, row) { + t.Fatal("Go operand classes/encoding changed", row) + } + } + } +} + +func x86LowMultiplyFormatRows(arch string) []string { + var rows []string + for _, source := range []string{"X0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, "PMULLD "+source+",X1") + } + for _, op := range []string{"VPMULLD", "VPMULLQ"} { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + for _, broadcast := range []bool{false, true} { + if broadcast && source == bank+"0" { + continue + } + for mode := 0; mode < 3; mode++ { + if arch == "386" && mode != 0 { + continue + } + name, args := op, source+","+bank+"1" + if broadcast { + name += ".BCST" + } + if mode != 0 { + args += ",K7" + } + if mode == 2 { + name += ".Z" + } + rows = append(rows, name+" "+args+","+bank+"2") + } + } + } + } + } + return rows +} + +func TestX86PackedLowMultiplyGoFormatsAndConsumerShapes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86LowMultiplyFormatRows(arch) { + t.Run(arch+"/"+row, func(t *testing.T) { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch) + if !known || err != nil { + t.Fatal("valid Go shape lost", known, err) + } + if !x86CallerVectorMoveKnown(ins) { + t.Fatal("caller consumer lost valid family") + } + if form.memory >= 0 { + width, write, known := x86InstructionMemoryShape(ins, form.memory, arch) + want := form.byteWidth + if strings.Contains(row, ".BCST") { + want = 4 + if strings.HasPrefix(row, "VPMULLQ") { + want = 8 + } + } + if !known || write || width != int64(want) { + t.Fatal("read tuple differs from Go", width, write, known, want) + } + } + }) + } + } +} + +func TestX86PackedLowMultiplyMode32HighBanksRemainContext(t *testing.T) { + for _, name := range []string{"VPMULLD", "VPMULLQ"} { + for _, bank := range []string{"X", "Y"} { + for _, register := range []int{8, 15, 16, 31} { + row := fmt.Sprintf("%s %s%d,%s1,%s2", name, bank, register, bank, bank) + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", true) + for _, target := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "386", TargetTriple: target, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go syntax invented a physical high bank", row, target, err) + } + } + } + } + } +} + +func TestX86PackedLowMultiplyResultBytes(t *testing.T) { + for _, row := range []string{"PMULLW X0,X1", "VPMULLW X0,X1,X2", "PMULLD X0,X1", "VPMULLD X0,X1,X2", "VPMULLQ X0,X1,X2"} { + for _, arch := range []string{"amd64", "386"} { + for missing := -1; missing < 16; missing++ { + ins := x86ScalarLaneInstruction(t, row) + state := x86ResultState{flags: 0x3f} + state.xmm[0], state.xmm[1] = 0xffff, 0xffff + state.vectorUpper[1], state.vectorUpper[2] = 0xffff0000, 0xff000000 + if missing >= 0 { + state.xmm[0] &^= 1 << uint(missing) + } + if !x86ResultTransfer(ins, &state, nil, 0, arch) { + t.Fatalf("registered low multiply result effect missing: %s/%s", arch, row) + } + laneBytes := 4 + if strings.Contains(row, "LLW") { + laneBytes = 2 + } else if strings.Contains(row, "LLQ") { + laneBytes = 8 + } + want := uint64(0xffff) + if missing >= 0 { + want &^= (uint64(1)< len(rows) { + end = len(rows) + } + t.Run(fmt.Sprint(begin), func(t *testing.T) { + var source, declarations strings.Builder + for index, row := range rows[begin:end] { + if strings.HasPrefix(row, "PMULLD ") && (strings.Contains(row, "(FP)") || strings.Contains(row, "(SP)")) { + // These original-Go forms have an explicit rejection + // regression below, not a successful object claim. + continue + } + fmt.Fprintf(&declarations, "func Low%d(value [64]byte)\n", index) + fmt.Fprintf(&source, "TEXT ·Low%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "low-multiply-"+api) + x86ResultInitializationObject(t, llc, target.triple, x86MulDivRoundOptimize(t, ir), "low-multiply-"+api+"-optimized") + } + }) + } + }) + } +} + +func TestX86PackedLowMultiplyVirtualLegacyStorageRemainsContext(t *testing.T) { + for _, row := range []string{"PMULLD value+0(FP),X1", "PMULLD local-16(SP),X1", "PMULLD (SP),X1", "PMULLD (BP),X1"} { + source := "TEXT ·Low(SB),4,$64-64\n" + row + "\nRET\n" + for _, target := range x86RawReturnTargets { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "func Low(value [64]byte)", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("logical storage acquired original SSE alignment", row, target.triple, api, err) + } + } + } + } +} + +func TestX86PackedLowMultiplyResultMemoryAndMasks(t *testing.T) { + for _, op := range []string{"VPMULLW", "VPMULLD", "VPMULLQ"} { + for _, broadcast := range []bool{false, true} { + if broadcast && op == "VPMULLW" { + continue + } + row, width := op, 16 + if broadcast { + row, width = row+".BCST", 4 + if op == "VPMULLQ" { + width = 8 + } + } + row += ".Z value+0(FP),X1,K1,X2" + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$0-16\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + for missing := -1; missing < 16; missing++ { + for _, maskKnown := range []bool{false, true} { + state := x86ResultState{fp: make([]bool, 16)} + owned := make([]bool, 16) + for i := range owned { + owned[i], state.fp[i] = true, i != missing + } + state.xmm[1], state.masks[1] = 0xffff, maskKnown + accepted := x86ResultTransfer(ins, &state, owned, 16, "amd64") + want := maskKnown && (missing < 0 || missing >= width) + if accepted != want || accepted && state.vectorBytes(2) != ^uint64(0) || len(ins.x86CallerMemory) != 0 { + t.Fatal("mask/tuple observation invented bytes or caller permission", row, missing, maskKnown, accepted) + } + } + } + } + } +} + +func TestX86PackedLowMultiplyResultThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var source, declarations strings.Builder + for index, row := range []string{"PMULLW X0,X1", "VPMULLW X0,X1,X1", "PMULLD X0,X1", "VPMULLD X0,X1,X1", "VPMULLQ X0,X1,X1"} { + fmt.Fprintf(&declarations, "func Low%d() uint32\n", index) + fmt.Fprintf(&source, "TEXT ·Low%d(SB),4,$0-4\nMOVL $0x87654321,AX\n", index) + for _, reg := range []string{"X0", "X1"} { + for lane := 0; lane < 4; lane++ { + fmt.Fprintf(&source, "PINSRD $%d,AX,%s\n", lane, reg) + } + } + fmt.Fprintf(&source, "%s\nPEXTRD $0,X1,AX\nMOVL AX,ret0+0(FP)\nRET\n", row) + } + for _, target := range x86RawReturnTargets { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(target, api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "low-multiply-result-"+api) + } + } +} diff --git a/amd64_packed_low_multiply_raw_test.go b/amd64_packed_low_multiply_raw_test.go new file mode 100644 index 000000000..55135061d --- /dev/null +++ b/amd64_packed_low_multiply_raw_test.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86PackedLowMultiplyRawWIGAndPhysicalMasks(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, length := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + code := []byte{0xc4, 0xe2, 0x71 | length | w, 0x40, 0xd0} + ins, size, known, err := decodedX86PackedMultiplyLowInstruction(code, mode) + if !known || err != nil || size != len(code) || ins.Op != "VPMULLD" { + t.Fatalf("VEX.WIG does not select PMULLQ: mode=%d code=%x known=%v size=%d err=%v", mode, code, known, size, err) + } + } + } + for _, qword := range []bool{false, true} { + for length := byte(0); length < 3; length++ { + for mask := byte(1); mask < 8; mask++ { + for _, zero := range []bool{false, true} { + code := encodeX86EVEXPackedMultiplyLow(qword, length, int(mask), zero, 2, 1, 0) + ins, size, known, err := decodedX86PackedMultiplyLowInstruction(code, mode) + if !known || err != nil || size != len(code) || !ins.x86Encoded { + t.Fatalf("physical EVEX must not inherit named operand limit: mode=%d code=%x known=%v size=%d err=%v", mode, code, known, size, err) + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + if _, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch); !known || err != nil { + t.Fatal("typed consumer lost decoded physical form", ins.Raw, err) + } + ins.x86Encoded = false + if _, _, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch); mode == 32 && err == nil { + t.Fatal("named mode32 instruction borrowed decoder identity") + } + } + } + } + } + } +} + +func TestX86PackedLowMultiplyRawThreeAPIsFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var encodings [][]byte + for _, length := range []byte{0, 4} { + for _, w := range []byte{0, 0x80} { + encodings = append(encodings, []byte{0xc4, 0xe2, 0x71 | length | w, 0x40, 0xd0}) + } + } + for _, qword := range []bool{false, true} { + for length := byte(0); length < 3; length++ { + for _, mask := range []int{1, 7} { + for _, zero := range []bool{false, true} { + encodings = append(encodings, encodeX86EVEXPackedMultiplyLow(qword, length, mask, zero, 2, 1, 0)) + } + } + } + } + var source, declarations strings.Builder + for i, code := range encodings { + fmt.Fprintf(&declarations, "func RawLow%d()\n", i) + fmt.Fprintf(&source, "TEXT ·RawLow%d(SB),4,$0-0\n", i) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("RET\n") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "raw-low-multiply-"+api) + } + }) + } +} diff --git a/amd64_packed_low_multiply_result.go b/amd64_packed_low_multiply_result.go new file mode 100644 index 000000000..1f53f552e --- /dev/null +++ b/amd64_packed_low_multiply_result.go @@ -0,0 +1,61 @@ +package plan9asm + +// This numerical observer shares the complete W/D/Q operand grammars. It +// supplies no caller region, native backing, pointer or raw-return permission. +func x86ResultPackedLowMultiplyStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + low, known, err := x86PackedLowMultiplyProperties(ins.Op, ins, arch) + form, bits := low.x86PackedBinaryForm, low.spec.laneBits + if !known { + word, wordKnown, wordErr := x86PackedWordMultiplyProperties(ins.Op, ins, arch) + if !wordKnown || word.spec.mode != amd64PackedWordMultiplyLow { + return false, false + } + form, bits, err = word.x86PackedBinaryForm, 16, wordErr + } + if err != nil || form.mmx { + return false, true + } + return x86ResultPackedLaneArithmeticStep(ins, form, bits, state, owned, arch), true +} + +// A lane is numerical only when every byte of both input lanes is known. +// This dependency rule is shared by low products and wrapping/saturating +// arithmetic, not their operations or their independent memory permissions. +func x86ResultPackedLaneArithmeticStep(ins Instr, form x86PackedBinaryForm, bits int, state *x86ResultState, owned []bool, arch string) bool { + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false + } + } + if form.memory >= 0 { + memory := ins.Args[form.memory] + // Logical FP/AUTO bytes can establish numerical dependence, not the + // original native alignment. Legacy lowering checks that separately. + if !x86ResultRead(memory, int64(form.memoryBytes), state, owned, arch) { + return false + } + } + bytes := func(index int) uint64 { + if index == form.memory { + return x86CallerVectorByteMask(int64(form.byteWidth)) + } + register, _ := amd64VectorRegisterIndex(ins.Args[index].Reg, form.byteWidth) + return state.vectorBytes(register) + } + first, second := bytes(form.first), bytes(form.second) + var initialized uint64 + laneBytes := bits / 8 + for index := 0; index < form.byteWidth; index += laneBytes { + lane := x86CallerVectorByteMask(int64(laneBytes)) << uint(index) + if first&lane == lane && second&lane == lane { + initialized |= lane + } + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + if form.mask >= 0 && !form.zeroing { + initialized &= state.vectorBytes(destination) + } + state.writeVector(destination, form.byteWidth, initialized, form.legacy) + return true +} diff --git a/amd64_packed_low_multiply_runtime_test.go b/amd64_packed_low_multiply_runtime_test.go new file mode 100644 index 000000000..f025042e0 --- /dev/null +++ b/amd64_packed_low_multiply_runtime_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +const x86LowMultiplyReference = ` +#include +#include +#include +#include + +typedef void (*kernel)(uint8_t *, const uint8_t *, const uint8_t *, + uint64_t *, uint32_t *, uint64_t, uint64_t); + +static uint64_t little(const uint8_t *p, int bytes) { + uint64_t value = 0; + for (int i = 0; i < bytes; ++i) { + value |= (uint64_t)p[i] << (8 * i); + } + return value; +} + +static int check(kernel fn, const char *name, int width, int bytes, + int legacy, int masking, int broadcast, int memory, int dst) { + _Alignas(64) uint8_t a[64], b[64], out[192], oldA[64], oldB[64]; + const uint64_t masks[] = {0, UINT64_MAX, 0x5555, UINT64_C(1) << 63}; + for (int sample = 0; sample < 65; ++sample) { + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(sample * 127 + i * 53); + b[i] = (uint8_t)(sample * 17 - i * 91); + } + memcpy(oldA, a, 64); + memcpy(oldB, b, 64); + for (int seedIndex = 0; seedIndex < 2; ++seedIndex) { + uint64_t seed = seedIndex ? 0x8d5 : 0; + for (unsigned mi = 0; mi < sizeof(masks)/sizeof(masks[0]); ++mi) { + uint64_t mask = masks[mi]; + uint64_t relevant = mask & ((UINT64_C(1) << (width / bytes)) - 1); + const uint8_t *input = memory && masking && !relevant ? NULL : a; + struct { uint64_t before, value, after; } flags = {11, 0, 13}; + struct { uint32_t before, value, after; } csr = {17, 0, 19}; + uint32_t initialCSR = _mm_getcsr(); + memset(out, 0xa7, sizeof(out)); + fn(out + 64, input, b, &flags.value, &csr.value, seed, mask); + if ((flags.value & 0x8d5) != seed || flags.before != 11 || flags.after != 13 || + csr.value != initialCSR || csr.before != 17 || csr.after != 19 || + memcmp(a, oldA, 64) || memcmp(b, oldB, 64)) { + fprintf(stderr, "%s state/input/canary mismatch\n", name); + return 1; + } + for (int i = 0; i < 192; ++i) { + unsigned want = 0xa7; + int offset = i - 64; + if (offset >= 0 && offset < 64) { + const uint8_t *old = dst == 0 ? a : b; + want = legacy ? old[offset] : 0; + if (offset < width) { + int lane = offset / bytes; + int active = !masking || ((mask >> lane) & 1); + if (active) { + uint64_t x = little(a + (broadcast ? 0 : lane * bytes), bytes); + uint64_t y = little(b + lane * bytes, bytes); + want = (uint8_t)((x * y) >> (8 * (offset % bytes))); + } else { + want = masking == 2 ? 0 : old[offset]; + } + } + } + if (out[i] != want) { + fprintf(stderr, "%s sample=%d mask=%llx byte=%d got=%x want=%x\n", + name, sample, (unsigned long long)mask, i, out[i], want); + return 1; + } + } + } + } + } + return 0; +} +` + +// This executes the portable LLVM result, not original AVX512 machine code. +// Independent full-byte observations catch stale upper aliases and merging; +// NULL with inactive/high-only masks catches eager E4 memory accesses. +func TestCrossLinuxRuntimeMatrixX86PackedLowMultiplySoftware(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + for _, destination := range []int{0, 1} { + t.Run(fmt.Sprint("destination-", destination), func(t *testing.T) { + var declarations, assembly, reference, checks strings.Builder + declarations.WriteString("package oracle\n") + reference.WriteString(x86LowMultiplyReference) + count := 0 + for _, name := range []string{"PMULLD", "VPMULLD", "VPMULLQ"} { + legacy, bits := 0, 32 + widths := []int{16, 32, 64} + if name == "PMULLD" { + legacy, widths = 1, []int{16} + } + if name == "VPMULLQ" { + bits = 64 + } + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for source := 0; source < 3; source++ { + for masking := 0; masking < 3; masking++ { + if legacy != 0 && (masking != 0 || source == 2 || destination == 0) { + continue + } + fn := fmt.Sprint("Low", count) + count++ + row, first, broadcast, memory := name, bank+"0", 0, 0 + if source != 0 { + first, memory = "(DI)", 1 + } + if source == 2 { + row, broadcast = row+".BCST", 1 + } + if masking == 2 { + row += ".Z" + } + row += " " + first + "," + bank + "1" + if masking != 0 { + row += ",K1" + } + if legacy == 0 { + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\nVMOVUPS (SI),Z1\n", fn) + if source == 0 || destination == 0 { + // For memory forms Z0's merge source uses B; do not + // read the possibly inactive source address early. + base := "DI" + if source != 0 { + base = "SI" + } + fmt.Fprintf(&assembly, "VMOVUPS (%s),Z0\n", base) + } + assembly.WriteString("MOVQ mask+48(FP),AX\nKMOVQ AX,K1\nPUSHQ seed+40(FP)\nPOPFQ\n") + fmt.Fprintf(&assembly, "%s\nVMOVUPS Z%d,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", row, destination) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", fn) + old := destination + if memory != 0 { + old = 1 + } + fmt.Fprintf(&checks, " if (check(%s,%q,%d,%d,%d,%d,%d,%d,%d)) return 1;\n", fn, row, width, bits/8, legacy, masking, broadcast, memory, old) + } + } + } + } + if count > 96 { + t.Fatal("runtime batch exceeds its resource bound", count) + } + reference.WriteString("int main(void) {\n" + checks.String() + " return 0;\n}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sigs[name] = x86WordMultiplySignature(t, pkg, name) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, + Sigs: sigs, ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "low-multiply", ir, reference.String(), level) + }) + } + t.Logf("%d kernels, 65 inputs, two FLAGS seeds, four masks; full aliases, canaries and inactive NULL sources", count) + }) + } +} diff --git a/amd64_packed_move_zero_store_test.go b/amd64_packed_move_zero_store_test.go new file mode 100644 index 000000000..33ef13cab --- /dev/null +++ b/amd64_packed_move_zero_store_test.go @@ -0,0 +1,315 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +// These are Go's ten EVEX packed-move opcodes, independently of the shared +// implementation map. VMOVDQA/DQU are VEX-only and do not have this form. +var x86PackedZeroStoreOpcodes = []string{ + "VMOVAPD", "VMOVAPS", "VMOVUPD", "VMOVUPS", + "VMOVDQA32", "VMOVDQA64", "VMOVDQU8", "VMOVDQU16", "VMOVDQU32", "VMOVDQU64", +} + +func x86PackedZeroStoreSource() (string, map[string]FuncSig, []string) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var names []string + for _, opcode := range x86PackedZeroStoreOpcodes { + for _, bank := range []string{"X", "Y", "Z"} { + name := "fault" + opcode + bank + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s.Z %s0,K1,(AX)\nRET\n", name, opcode, bank) + sigs[name] = FuncSig{Name: name, Ret: Void} + names = append(names, name) + } + } + return source.String(), sigs, names +} + +func TestX86PackedZeroMemoryStorePreservesUndefinedOpcode(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + source, sigs, names := x86PackedZeroStoreSource() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.arch, source, true) + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + trap := `call void asm sideeffect "ud2", "~{memory}"()` + if strings.Count(ir, trap) != len(names) || strings.Contains(ir, "call void @llvm.masked.store.") { + t.Fatalf("Go-accepted EVEX.z memory stores must deliver #UD before reading registers or memory; traps=%d want=%d", strings.Count(ir, trap), len(names)) + } + compileLLVMToObject(t, llc, target.triple, "zero-store.ll", "zero-store.o", ir) + }) + } +} + +func TestX86PackedZeroMemoryStoreFaultPrecedesFPBoundsAndMaskReads(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + instruction := Instr{Op: "VMOVUPS.Z", Args: []Operand{ + {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "K1"}, + {Kind: OpFP, FPOffset: 1 << 20}, + }} + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.Len() + recognized, terminated, err := ctx.lowerPackedVectorMove(instruction.Op, instruction) + if !recognized || !terminated || err != nil || output.String()[before:] != " call void asm sideeffect \"ud2\", \"~{memory}\"()\n unreachable\n" { + t.Fatalf("%s invalid encoding evaluated nonexistent FP/mask/register data before #UD: %t/%t/%v", arch, recognized, terminated, err) + } + } +} + +func x86PackedFirstGoInstructionBytes(t *testing.T, arch, instruction string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, arch, "TEXT probe(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatalf("actual Go packed-move object: %v\n%s", err, listing) + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + bytes := make([]byte, 6) + for index := range bytes { + value, err := strconv.ParseUint(fields[index+1], 16, 8) + if err != nil { + t.Fatalf("actual Go object byte: %v\n%s", err, listing) + } + bytes[index] = byte(value) + } + return bytes + } + t.Fatalf("actual Go object lacks packed-move bytes:\n%s", listing) + return nil +} + +func TestX86PackedZeroMemoryStoreActualGoAndLLVM22Encoding(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM 22 llvm-mc not found") + } + for _, arch := range []string{"amd64", "386"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, opcode := range x86PackedZeroStoreOpcodes { + for _, bank := range []string{"X", "Y", "Z"} { + instruction := fmt.Sprintf("%s %s0,K1,(AX)", opcode, bank) + normal := x86PackedFirstGoInstructionBytes(t, arch, instruction) + zero := x86PackedFirstGoInstructionBytes(t, arch, strings.Replace(instruction, " ", ".Z ", 1)) + if normal[0] != 0x62 || normal[3]&0x80 != 0 || zero[3]&0x80 == 0 { + t.Fatalf("%s %s lacks independent EVEX.z witness: normal=%x zero=%x", arch, instruction, normal, zero) + } + wantW := strings.HasSuffix(opcode, "64") || opcode == "VMOVDQU16" || strings.HasSuffix(opcode, "PD") + if (normal[2]&0x80 != 0) != wantW { + t.Fatalf("actual Go object changed the native element selector: %s %s bytes=%x", arch, instruction, normal) + } + for index := range normal { + wanted := normal[index] + if index == 3 { + wanted |= 0x80 + } + if zero[index] != wanted { + t.Fatalf(".Z changed more than the native EVEX.z bit: %s %s", arch, instruction) + } + } + for _, test := range []struct { + bytes []byte + invalid bool + }{{normal, false}, {zero, true}} { + var input strings.Builder + for _, value := range test.bytes { + fmt.Fprintf(&input, "0x%02x ", value) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, + "--mattr=+avx512f,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(input.String()) + command.WaitDelay = 2 * time.Second + output, err := command.CombinedOutput() + cancel() + // llvm-mc can return zero while diagnosing rejected bytes. + invalid := strings.Contains(string(output), "invalid instruction encoding") + if err != nil || invalid != test.invalid || !test.invalid && !strings.Contains(strings.ToUpper(string(output)), "VMOV") { + t.Fatalf("%s %s invalid=%t: LLVM 22 diagnostic mismatch: %v\n%s", arch, instruction, test.invalid, err, output) + } + // LLVM 22's 32-bit decoder aliases some W1 move spellings + // to W0 names. Verify the independent W byte above and the + // exact opcode in 64-bit decode, not the aliased 32-bit name. + if !test.invalid { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=x86_64-unknown-linux-gnu", + "--mattr=+avx512f,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(input.String()) + command.WaitDelay = 2 * time.Second + output, err := command.CombinedOutput() + cancel() + if err != nil || !strings.Contains(strings.ToUpper(string(output)), opcode) || strings.Contains(string(output), "invalid instruction encoding") { + t.Fatalf("actual Go object does not decode as the exact opcode: %s %s: %v\n%s", arch, instruction, err, output) + } + } + } + } + } + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedZeroMemoryStoreFault(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires the separately required Linux/amd64 packed-move fault oracle") + } + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM 22 llc/clang not found") + } + source, sigs, names := x86PackedZeroStoreSource() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + var oracle strings.Builder + oracle.WriteString(`#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#include +#endif + +`) + for _, name := range names { + fmt.Fprintf(&oracle, "extern void %s(void);\n", name) + } + oracle.WriteString("\nstatic void (*fault_functions[])(void) = {\n") + for _, name := range names { + fmt.Fprintf(&oracle, " %s,\n", name) + } + oracle.WriteString(`}; + +int main(int argc, char **argv) { +#ifdef _WIN32 + SetErrorMode(SEM_NOGPFAULTERRORBOX | SEM_FAILCRITICALERRORS); + if (argc == 2) { + unsigned index = (unsigned)strtoul(argv[1], 0, 10); + if (index >= sizeof(fault_functions)/sizeof(fault_functions[0])) return 6; + fault_functions[index](); + return 3; + } + for (unsigned i = 0; i < sizeof(fault_functions)/sizeof(fault_functions[0]); i++) { + char index[16]; + snprintf(index, sizeof(index), "%u", i); + intptr_t status = _spawnl(_P_WAIT, argv[0], argv[0], index, NULL); + if ((uint32_t)status != UINT32_C(0xc000001d)) return 5; + } +#else + (void)argc; + (void)argv; + struct rlimit no_core = {0, 0}; + if (setrlimit(RLIMIT_CORE, &no_core)) return 1; + for (unsigned i = 0; i < sizeof(fault_functions)/sizeof(fault_functions[0]); i++) { + pid_t child = fork(); + if (child < 0) return 2; + if (!child) { fault_functions[i](); _exit(3); } + int status; + if (waitpid(child, &status, 0) != child) return 4; + if (!WIFSIGNALED(status) || WTERMSIG(status) != SIGILL) return 5; + } +#endif + return 0; +} +`) + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-zero-store-fault", triple, ir, oracle.String(), runner) + + // No preceding valid AVX512 instruction is executed. EVEX.z=1 with a + // memory destination is always #UD, irrespective of mask/address values. + var mainGo strings.Builder + mainGo.WriteString("package main\nimport \"os\"\n") + for _, name := range names { + fmt.Fprintf(&mainGo, "func %s()\n", name) + } + mainGo.WriteString("func main() { switch os.Args[1] {\n") + for _, name := range names { + fmt.Fprintf(&mainGo, "case %q: %s()\n", name, name) + } + mainGo.WriteString("} }\n") + dir := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module example.com/packed-zero-store\n\ngo 1.20\n", + "main.go": mainGo.String(), + "fault_amd64.s": strings.ReplaceAll(source, "TEXT fault", "TEXT ·fault"), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original-packed-zero-store") + if runtime.GOOS == "windows" { + executable += ".exe" + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + build := exec.CommandContext(ctx, "go", "build", "-o", executable, ".") + build.Dir, build.Env = dir, append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0", "GOARCH=amd64") + build.WaitDelay = 2 * time.Second + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("original Go packed-store build: %v\n%s", err, output) + } + for _, name := range names { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + args := append(append([]string(nil), runner...), executable, name) + command := exec.CommandContext(ctx, args[0], args[1:]...) + command.Env = append(os.Environ(), "GOTRACEBACK=single") + command.WaitDelay = 2 * time.Second + output, err := command.CombinedOutput() + cancel() + exit, ok := err.(*exec.ExitError) + expectedDiagnostic := "SIGILL: illegal instruction" + if runtime.GOOS == "windows" { + expectedDiagnostic = "Exception 0xc000001d" + } + if !ok || exit.ExitCode() != 2 || !strings.Contains(string(output), expectedDiagnostic) { + t.Fatalf("original Go %s did not deliver SIGILL: %v\n%s", name, err, output) + } + } +} diff --git a/amd64_packed_narrow_forms.go b/amd64_packed_narrow_forms.go new file mode 100644 index 000000000..731446e78 --- /dev/null +++ b/amd64_packed_narrow_forms.go @@ -0,0 +1,71 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedNarrowSpec struct { + x86PackedBinarySpec + inputBits, outputBits int + mode amd64PackedNarrowMode +} + +// Go's ymm rows admit MMX only in amd64 mode; PACKUSDW is yxm_q4. +// All four V rows use _yvandnpd, with N4 broadcast only for dword inputs. +// Memory exception behavior is E4NF (SSWB/USWB: E4NF.nb), independently +// of their output-lane writemask. No initialized bank or ABI is granted here. +var x86PackedNarrowSpecs = map[Op]x86PackedNarrowSpec{ + "PACKSSLW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 32, 16, amd64PackedNarrowSigned}, + "PACKSSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, 8, amd64PackedNarrowSigned}, + "PACKUSDW": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP}, 32, 16, amd64PackedNarrowUnsigned}, + "PACKUSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalLegacyInteger}, 16, 8, amd64PackedNarrowUnsigned}, + "VPACKSSDW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32, 16, amd64PackedNarrowSigned}, + "VPACKSSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, 8, amd64PackedNarrowSigned}, + "VPACKUSDW": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, broadcastBytes: 4, zeroing: true}, 32, 16, amd64PackedNarrowUnsigned}, + "VPACKUSWB": {x86PackedBinarySpec{encoding: x86PackedLogicalVEXOrEVEX, zeroing: true}, 16, 8, amd64PackedNarrowUnsigned}, +} + +type x86PackedNarrowForm struct { + x86PackedBinaryForm + spec x86PackedNarrowSpec + source Operand +} + +func x86PackedNarrowProperties(ins Instr, goarch string) (x86PackedNarrowForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, known := x86PackedNarrowSpecs[Op(base)] + if !known { + return x86PackedNarrowForm{}, false, nil + } + local := ins + local.Args = append([]Operand(nil), ins.Args...) + if spec.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(local, goarch) + if err != nil { + return x86PackedNarrowForm{}, true, err + } + } + grammarArch := goarch + if goarch == "386" && ins.x86Encoded && spec.encoding == x86PackedLogicalVEXOrEVEX { + // Validated physical bytes may have masks which Go's mode32 named + // frontend rejects. The private marker is not JSON-exportable. + grammarArch = "amd64" + } + form, err := x86PackedBinaryProperties(local, grammarArch, spec.x86PackedBinarySpec) + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + if err == nil && grammarArch != goarch && form.memory >= 0 && !x86PackedShiftMemoryValid(local.Args[form.memory], goarch) { + err = fmt.Errorf("386 %s physical source has an unbound address", ins.Op) + } + if err == nil && form.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: packed narrow has no bound address-size override", ErrProbeNeedsContext) + } + result := x86PackedNarrowForm{x86PackedBinaryForm: form, spec: spec} + if err == nil { + result.source = local.Args[form.first] + } + return result, true, err +} diff --git a/amd64_packed_narrow_independent_runtime_test.go b/amd64_packed_narrow_independent_runtime_test.go new file mode 100644 index 000000000..2869cdab3 --- /dev/null +++ b/amd64_packed_narrow_independent_runtime_test.go @@ -0,0 +1,501 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// The independent oracle describes the ISA, not production properties. Go's +// reversed operands put its second source in the low half of each 128-bit lane. +type x86NarrowOracleFamily struct { + legacy, vector string + inputBytes int + unsigned bool +} + +type x86NarrowOracleRow struct { + family x86NarrowOracleFamily + bank string + width, source, alias int + masking int // 0: none, 1: merge, 2: zero + legacy bool +} + +func x86NarrowOracleRows() []x86NarrowOracleRow { + var rows []x86NarrowOracleRow + for _, family := range x86NarrowOracleFamilies { + for _, bank := range []string{"X", "M"} { + if bank == "M" && family.legacy == "PACKUSDW" { + continue + } + width := 16 + if bank == "M" { + width = 8 + } + for source := 0; source < 2; source++ { + for alias := 0; alias < 2; alias++ { + rows = append(rows, x86NarrowOracleRow{family, bank, width, source, alias, 0, true}) + } + } + } + for widthIndex, bank := range []string{"X", "Y", "Z"} { + for source := 0; source < 3; source++ { + if source == 2 && family.inputBytes != 4 { + continue + } + for alias := 0; alias < 3; alias++ { + for mask := 0; mask < 3; mask++ { + rows = append(rows, x86NarrowOracleRow{family, bank, 16 << widthIndex, source, alias, mask, false}) + } + } + } + } + } + return rows +} + +func (row x86NarrowOracleRow) instruction() (string, string) { + first, destination := row.bank+"0", row.bank+fmt.Sprint(row.alias) + if row.legacy { + destination = row.bank + "1" + if row.alias == 1 { + first = destination + } + } + if row.source != 0 { + first = "(CX)" + if row.legacy && row.alias == 1 { + first = "(DX)" + } + } + if row.legacy { + return row.family.legacy + " " + first + "," + destination, destination + } + op := row.family.vector + if row.source == 2 { + op += ".BCST" + } + if row.masking == 2 { + op += ".Z" + } + mask := "" + if row.masking != 0 { + mask = "K1," + } + return op + " " + first + "," + row.bank + "1," + mask + destination, destination +} + +// Each scalar is signed before clamping, including the unsigned output forms. +// This reference does not call or reproduce production's vector/typed helpers. +const x86NarrowIndependentCReference = ` +static const int32_t edges[]={ + INT32_MIN,-65537,-32769,-32768,-32767,-257,-256,-129,-128,-127,-1,0,1, + 126,127,128,129,254,255,256,257,32766,32767,32768,32769, + 65534,65535,65536,65537,INT32_MAX-1,INT32_MAX}; +static void fill(uint8_t *out,int bytes,unsigned sample,unsigned salt) { + for(int lane=0;lane<64/bytes;++lane) { + int32_t value=edges[(sample+salt+lane*7)%(sizeof(edges)/sizeof(edges[0]))]; + if(bytes==4) memcpy(out+lane*bytes,&value,4); + else {int16_t small=(int16_t)value;memcpy(out+lane*bytes,&small,2);} + } +} +static void independent_pack(uint8_t *out,const uint8_t *first,const uint8_t *second, + int width,int inputBytes,int unsignedOutput,int broadcast) { + int chunk=width==8?8:16,outputBytes=inputBytes/2; + int64_t minimum=unsignedOutput?0:-(INT64_C(1)<<(outputBytes*8-1)); + int64_t maximum=unsignedOutput?(INT64_C(1)<<(outputBytes*8))-1:(INT64_C(1)<<(outputBytes*8-1))-1; + for(int start=0;startmaximum) value=maximum; + uint32_t result=(uint32_t)value; + int destination=start+source*(chunk/2)+lane*outputBytes; + for(int byte=0;byte>(byte*8)); + } + } +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedNarrowIndependentWholeBytes(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + rows := x86NarrowOracleRows() + var assembly, declarations, table, main strings.Builder + declarations.WriteString("package narroworacle\n") + main.WriteString(`#include +#include +#include +typedef void (*kernel)(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t); +`) + for index, row := range rows { + name := fmt.Sprintf("numeric%d", index) + fmt.Fprintf(&declarations, "func %s(out *[80]byte,a,b,old *[64]byte,mask,flags uint64,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-52\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\n", name) + if row.bank == "M" { + assembly.WriteString("MOVQ (CX),M0\nMOVQ (DX),M1\n") + } else { + assembly.WriteString("VMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\n") + } + instruction, destination := row.instruction() + fmt.Fprintf(&assembly, "LDMXCSR csr+48(FP)\nPUSHQ flags+40(FP)\nPOPFQ\n%s\n", instruction) + if row.bank == "M" { + fmt.Fprintf(&assembly, "MOVQ %s,(AX)\nEMMS\n", destination) + } else { + fmt.Fprintf(&assembly, "VMOVUPS Z%s,(AX)\n", destination[1:]) + } + assembly.WriteString("PUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nSTMXCSR 72(AX)\nRET\n") + fmt.Fprintf(&main, "extern void %s(void*,const void*,const void*,const void*,uint64_t,uint64_t,uint32_t);\n", name) + fmt.Fprintf(&table, "{%s,%d,%d,%t,%d,%d,%d,%t},\n", name, row.width, row.family.inputBytes, + row.family.unsigned, row.source, row.alias, row.masking, row.legacy) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + main.WriteString("static const struct {kernel run;int width,bytes,uns,source,alias,masking,legacy;} cases[]={\n" + table.String() + "};\n") + main.WriteString(x86NarrowIndependentCReference) + main.WriteString(` +int main(void) { + unsigned saved;__asm__ volatile("stmxcsr %0":"=m"(saved)); + const uint32_t profiles[]={0x1f80,0x3fc1,0x5f82,0x9fe5}; + unsigned long long observations=0; + for(unsigned row=0;row>(i/(cases[row].bytes/2))&1)) + value=cases[row].masking==2?0:previous[i]; + } + want[1+i]=value; + } + uint64_t observedFlags;uint32_t observedCSR; + memcpy(&observedFlags,output+65,8);memcpy(want+65,&observedFlags,8); + memcpy(&observedCSR,output+73,4);memcpy(want+73,&observedCSR,4); + ++observations; + if((observedFlags&0x8d5)!=(flags&0x8d5)||observedCSR!=profiles[profile]|| + memcmp(output,want,sizeof(output))||memcmp(a,beforeA,64)||memcmp(b,beforeB,64)||memcmp(old,beforeOld,64)) { + fprintf(stderr,"PACK row=%u width=%d inputBytes=%d unsigned=%d source=%d alias=%d masking=%d legacy=%d sample=%u mask=%llx flags=%llx csr=%x\n", + row,cases[row].width,cases[row].bytes,cases[row].uns,cases[row].source,cases[row].alias,cases[row].masking, + cases[row].legacy,sample,(unsigned long long)mask,(unsigned long long)observedFlags,observedCSR); + for(unsigned i=0;imaximum {value=maximum} + for byteIndex:=0;byteIndex>(byteIndex*8)) + } + } + } + return out +} +func TestOriginal(t *testing.T) { + runtime.LockOSThread();defer runtime.UnlockOSThread() + profiles:=[]uint32{0x1f80,0x3fc1,0x5f82,0x9fe5} + var aa,bb [128]byte + a,b:=aligned(&aa),aligned(&bb) + observations:=0 + for row,entry:=range cases {for sample:=0;sample<32;sample++ {for _,csr:=range profiles {for seed:=0;seed<2;seed++ { + for i:=range aa {aa[i]=0xa7;bb[i]=0xb9} + fill(a,entry.bytes,sample,0);fill(b,entry.bytes,sample,13) + beforeA,beforeB:=aa,bb + var output,want [34]byte + for i:=range output {output[i]=0xc3;want[i]=0xc3} + flags:=uint64(2);if seed!=0 {flags|=0x8d5} + entry.run((*[32]byte)(unsafe.Pointer(&output[1])),a,b,flags,csr) + first:=a;if entry.alias!=0 {first=b} + packed:=reference(first,b,entry.width,entry.bytes,entry.unsigned) + copy(want[1:1+entry.width],packed[:entry.width]) + observed:=binary.LittleEndian.Uint64(output[17:25]);copy(want[17:29],output[17:29]) + actualCSR:=binary.LittleEndian.Uint32(output[25:29]) + observations++ + if output!=want||aa!=beforeA||bb!=beforeB||observed&0x8d5!=flags&0x8d5||actualCSR!=csr { + t.Fatalf("original PACK row%d sample%d width%d signedInputBytes%d flags%x csr%x got%x want%x",row,sample,entry.width,entry.bytes,observed,actualCSR,output,want) + } + }}}} + t.Logf("original SSE/MMX kernels=%d observations=%d sixFlags/fullCSR/input-output-canaries",len(cases),observations) +} +` + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module narroworacle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": tests, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "original") + command := exec.Command("go", "test", "-c", "-p=2", "-vet=off", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH=amd64", "CGO_ENABLED=0", + "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOMAXPROCS=2") + if output := x86FloatCompareCaptureCommand(t, command); len(output) != 0 { + t.Logf("actual original Go build diagnostic (unchanged bytes):\n%s", output) + } + output := execution.run(t, executable, "-test.v", "-test.run=^TestOriginal$", "-test.count=1") + want := fmt.Sprintf("original SSE/MMX kernels=%d observations=%d sixFlags/fullCSR/input-output-canaries", count, count*32*4*2) + if !strings.Contains(string(output), want) || !strings.Contains(string(output), "--- PASS: TestOriginal") || + strings.Contains(string(output), "SKIP") { + t.Fatalf("incomplete original-Go oracle denominator: %q", output) + } + t.Log(strings.TrimSpace(string(output))) + t.Logf("%d actual stock Go SSE/MMX kernels; no VEX/EVEX original code executed", count) +} + +var x86NarrowOracleFamilies = []x86NarrowOracleFamily{ + {"PACKSSLW", "VPACKSSDW", 4, false}, + {"PACKSSWB", "VPACKSSWB", 2, false}, + {"PACKUSDW", "VPACKUSDW", 4, true}, + {"PACKUSWB", "VPACKUSWB", 2, true}, +} + +func x86NarrowOracleIR(t *testing.T, source, declarations, triple string) string { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "narroworacle", declarations) + sigs := make(map[string]FuncSig) + for _, function := range file.Funcs { + name := strings.TrimPrefix(function.Sym, "·") + object, ok := pkg.Types.Scope().Lookup(name).(*types.Func) + if !ok { + t.Fatal("independent source lacks its declaration", name) + } + sig, err := DeriveGoFuncABI0Signature(name, object, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, Options{ + Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + return module.String() +} + +// Intel SDM Vol. 2A, Table 2-50 and the PACK chapters classify these complete +// memory tuples as E4NF/E4NF.nb: a zero output mask does not suppress the read. +// No AVX/AVX512 original instruction is executed by this software LLVM oracle. +func TestCrossLinuxRuntimeMatrixX86PackedNarrowIndependentE4NF(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var assembly, declarations, table strings.Builder + declarations.WriteString("package narroworacle\n") + count := 0 + for _, family := range x86NarrowOracleFamilies { + for widthIndex, bank := range []string{"X", "Y", "Z"} { + for _, broadcast := range []bool{false, true} { + if broadcast && family.inputBytes != 4 { + continue + } + for _, zero := range []bool{false, true} { + name := fmt.Sprintf("fault%d", count) + count++ + op := family.vector + bytes := 16 << widthIndex + if broadcast { + op += ".BCST" + bytes = 4 + } + if zero { + op += ".Z" + } + fmt.Fprintf(&declarations, "func %s(input *byte)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-8\nMOVQ input+0(FP),AX\nVPXORD %s0,%s0,%s0\nMOVL $0,CX\nKMOVQ CX,K1\n%s (AX),%s0,K1,%s1\nRET\n", + name, bank, bank, bank, op, bank, bank) + fmt.Fprintf(&table, "{%s,%d,\"%s %s\"},\n", name, bytes, op, bank) + } + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + ir := x86NarrowOracleIR(t, assembly.String(), declarations.String(), execution.triple) + var main strings.Builder + main.WriteString(`#include +#include +#include +#include +#include +#include +#include +`) + for index := 0; index < count; index++ { + fmt.Fprintf(&main, "extern void fault%d(const uint8_t*);\n", index) + } + main.WriteString("static const struct {void (*run)(const uint8_t*); int bytes; const char *name;} cases[]={\n" + table.String() + "};\n") + main.WriteString(` +static int observe(void (*run)(const uint8_t*),const uint8_t *input,int fault,const char *name) { + pid_t child=fork(); + if(child<0) return 1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + run(input); + _exit(0); + } + int status=0; + if(waitpid(child,&status,0)!=child) return 1; + int correct=fault ? WIFSIGNALED(status)&&(WTERMSIG(status)==SIGSEGV||WTERMSIG(status)==SIGBUS) + : WIFEXITED(status)&&WEXITSTATUS(status)==0; + if(!correct) fprintf(stderr,"E4NF %s input=%p requiredFault=%d waitStatus=%d\n",name,input,fault,status); + return !correct; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 2; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED) return 3; + for(long i=0;i 3 && !ins.x86Encoded { + return fail() + } + local := ins + local.Args = append([]Operand(nil), ins.Args...) + shift := 0 + if spec.kind != amd64PackedSADBlockSum { + if len(local.Args) == 0 || !amd64UnsignedImmediate(local.Args[0], 8) { + return fail() + } + result.immediate = uint8(local.Args[0].Imm) + local.Args = local.Args[1:] + shift = 1 + } + grammar := x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP} + if spec.form == amd64PackedSADVEXXY { + grammar.encoding = x86PackedLogicalVEX + } else if spec.form == amd64PackedSADEVEXXYZ { + grammar.encoding = x86PackedLogicalVEXOrEVEX + } + if spec.kind == amd64PackedSADDoubleBlock { + grammar.encoding, grammar.zeroing = x86PackedLogicalEVEX, true + } + // Go's shared table accepts high vector spellings on 386, but does not + // establish their physical bank. Validate the table first and return + // Context below, rather than misclassifying accepted syntax as invalid. + // Address register/disp32 checks retain the actual mode independently. + for i, operand := range local.Args { + if operand.Kind != OpMem { + continue + } + if !x86MemoryRegistersValidForArch(operand.Mem, goarch) { + return fail() + } + if goarch == "386" && operand.Mem.Off == int64(uint32(operand.Mem.Off)) { + local.Args[i].Mem.Off = int64(int32(operand.Mem.Off)) + } + } + form, err := x86PackedBinaryProperties(local, "amd64", grammar) + form.first += shift + form.second += shift + form.destination += shift + if form.memory >= 0 { + form.memory += shift + } + if form.mask >= 0 { + form.mask += shift + } + result.x86PackedBinaryForm = form + if err == nil && goarch == "386" { + for _, operand := range ins.Args { + index, valid := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + if operand.Kind == OpReg && valid && index >= 8 && (form.legacy || amd64VectorByteWidth(operand.Reg) == 64) { + return fail() + } + } + } + if err == nil && spec.kind != amd64PackedSADDoubleBlock && form.mask >= 0 { + return fail() + } + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + if err == nil && form.memory >= 0 && ins.x86AddressBits != 0 { + err = fmt.Errorf("%w: %s lacks its original native address-size binding", ErrProbeNeedsContext, ins.Op) + } + return result, true, err +} + +// Required byte masks for one output word. PSADBW's other six bytes in +// each qword are independent known zero; MPS/DBPS compute all output words. +func (form x86PackedSADForm) wordInputs(outputByte int) (first, second uint64) { + switch form.spec.kind { + case amd64PackedSADBlockSum: + if outputByte%8 != 0 { + return 0, 0 + } + return uint64(255) << uint(outputByte), uint64(255) << uint(outputByte) + case amd64PackedSADMultiple: + lane := outputByte / 16 * 16 + control := form.immediate + if lane != 0 { + control >>= 3 + } + firstOffset := lane + int(control&3)*4 + secondOffset := lane + int(control>>2&1)*4 + outputByte%16/2 + return uint64(15) << uint(firstOffset), uint64(15) << uint(secondOffset) + default: + block := outputByte / 8 * 8 + word := outputByte % 8 / 2 + secondOffset := block + word/2*4 + for i := 0; i < 4; i++ { + shuffled := block + word + i + lane := shuffled / 16 * 16 + selector := form.immediate >> uint(2*(shuffled%16/4)) & 3 + first |= uint64(1) << uint(lane+int(selector)*4+shuffled%4) + } + return first, uint64(15) << uint(secondOffset) + } +} diff --git a/amd64_packed_sad_forms_test.go b/amd64_packed_sad_forms_test.go new file mode 100644 index 000000000..08b3ffdae --- /dev/null +++ b/amd64_packed_sad_forms_test.go @@ -0,0 +1,346 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86SADRows(arch string) []string { + rows := []string{"PSADBW X0,X7", "PSADBW (AX),X7", "MPSADBW $0,X0,X7", "MPSADBW $255,16(AX)(BX*2),X7"} + if arch == "amd64" { + rows = append(rows, "PSADBW X15,X0", "PSADBW data(SB),X1", "PSADBW (FS),X1", "MPSADBW $128,(GS),X15") + } + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", "16(AX)(BX*2)"} { + rows = append(rows, fmt.Sprintf("VPSADBW %s,%s1,%s7", source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("VPSADBW %s,%s31,%s30", source, bank, bank)) + for _, imm := range []int{0, 1, 7, 8, 37, 228, 255} { + rows = append(rows, + fmt.Sprintf("VDBPSADBW $%d,%s,%s31,%s30", imm, source, bank, bank), + fmt.Sprintf("VDBPSADBW $%d,%s,%s31,K1,%s30", imm, source, bank, bank), + fmt.Sprintf("VDBPSADBW.Z $%d,%s,%s31,K7,%s30", imm, source, bank, bank)) + } + if bank != "Z" { + rows = append(rows, fmt.Sprintf("VMPSADBW $37,%s,%s1,%s15", source, bank, bank)) + } + } + } + } + return rows +} + +func x86SADGoPackage(t *testing.T, arch, declaration string) GoPackage { + t.Helper() + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "sad.go", "package sad\nfunc sad"+declaration+"\n", 0) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("sad", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "sad", Types: typed, Syntax: []*ast.File{syntax}} +} + +func TestX86PackedSADActualGoCompleteOptabs(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{"{APSADBW, yxm, Pq, opBytes{0xf6}}", "{AMPSADBW, yxshuf, Pq, opBytes{0x3a, 0x42, 0}}"} { + if !strings.Contains(string(legacy), row) { + t.Fatal("actual Go legacy registration changed: " + row) + } + } + for name, expected := range map[string][]string{ + "yxm": {"Yxm,Yxr"}, + "yxshuf": {"Yu8,Yxm,Yxr"}, + "_yvaesdec": {"Yxm,Yxr,Yxr", "Yym,Yyr,Yyr", "YxmEvex,YxrEvex,YxrEvex", "YymEvex,YyrEvex,YyrEvex", "Yzm,Yzr,Yzr"}, + "_yvblendpd": {"Yu8,Yxm,Yxr,Yxr", "Yu8,Yym,Yyr,Yyr"}, + "_yvalignd": {"Yu8,YxmEvex,YxrEvex,YxrEvex", "Yu8,YxmEvex,YxrEvex,Yknot0,YxrEvex", "Yu8,YymEvex,YyrEvex,YyrEvex", "Yu8,YymEvex,YyrEvex,Yknot0,YyrEvex", "Yu8,Yzm,Yzr,Yzr", "Yu8,Yzm,Yzr,Yknot0,Yzr"}, + } { + text := string(avx) + if !strings.HasPrefix(name, "_") { + text = string(legacy) + } + section := regexp.MustCompile(`(?s)var ` + name + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(text) + if len(section) != 2 { + t.Fatal("actual table missing " + name) + } + var got []string + for _, row := range regexp.MustCompile(`argList\{([^}]*)\}`).FindAllStringSubmatch(section[1], -1) { + got = append(got, strings.ReplaceAll(row[1], " ", "")) + } + if !reflect.DeepEqual(got, expected) { + t.Fatalf("actual %s rows %q != %q", name, got, expected) + } + } + for _, row := range []struct { + op, table string + count int + }{{"VPSADBW", "_yvaesdec", 5}, {"VMPSADBW", "_yvblendpd", 2}, {"VDBPSADBW", "_yvalignd", 3}} { + start := strings.Index(string(avx), "{as: A"+row.op+",") + if start < 0 { + t.Fatal("missing actual " + row.op) + } + entry := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(entry, "ytab: "+row.table) || strings.Count(entry, "avxEscape") != row.count || strings.Contains(entry, "Bcst") || strings.Contains(entry, "Sae") || strings.Contains(entry, "Rounding") { + t.Fatalf("actual SAD registration changed: %s", entry) + } + if strings.Contains(entry, "ZeroingEnabled") != (row.op == "VDBPSADBW") { + t.Fatal("actual SAD zeroing attribute changed " + row.op) + } + } +} + +func TestX86PackedSADCompleteRowsThreeAPIsFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + source := "TEXT sad(SB),4,$0-0\n" + strings.Join(x86SADRows(arch), "\n") + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"sad": {Name: "sad", Ret: Void}}} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "sad-text.ll", "sad-text.o", ir) + compileLLVMToObject(t, llc, triple, "sad-O2.ll", "sad-O2.o", x86MulDivRoundOptimize(t, ir)) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "sad-module.ll", "sad-module.o", module.String()) + translated, err := translateGoModuleInContext(ctx, x86SADGoPackage(t, arch, "()"), []byte(strings.Replace(source, "sad(SB)", "·sad(SB)", 1)), GoModuleOptions{GOARCH: arch, TargetTriple: triple}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "sad-go.ll", "sad-go.o", translated.Module.String()) + }) + } +} + +func TestX86PackedSADDispatchMalformedAndPhysicalBounds(t *testing.T) { + for _, row := range []string{"PSADBW X0,X1", "VPSADBW Z0,Z1,Z2", "MPSADBW $5,X0,X1", "VMPSADBW $37,Y0,Y1,Y2", "VDBPSADBW.Z $228,Z0,Z1,K1,Z2"} { + file, _ := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + ins := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", ins.Op, Op(strings.ToLower(string(ins.Op))), "UNKNOWN", "PSADBW.Z"} { + local := ins + local.Op = identity + original := local + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerPackedSumAbsoluteDifferences(ins.Op, local) + bad := identity != "" && !strings.EqualFold(string(identity), string(ins.Op)) + if !known || term || (err != nil) != bad || bad && output.String() != before || !reflect.DeepEqual(local, original) { + t.Fatalf("dispatch %s/%s: %v", ins.Op, identity, err) + } + } + } + for _, row := range []string{"PSADBW M0,M1", "PSADBW Y0,Y1", "PSADBW $1,X0", "VPSADBW X0,Y1,Y2", "VPSADBW X0,X1,K1,X2", "VPSADBW.Z X0,X1,X2", "VPSADBW.BCST (AX),Z0,Z1", "MPSADBW $-1,X0,X1", "MPSADBW $256,X0,X1", "VMPSADBW $1,Z0,Z1,Z2", "VMPSADBW $1,Y16,Y1,Y2", "VDBPSADBW $1,X0,X1,K0,X2", "VDBPSADBW.Z $1,X0,X1,X2", "VDBPSADBW.BCST $1,(AX),X0,X1", "PSADBW (AX)(SP*2),X0", "VPSADBW 2147483648(AX),Z0,Z1", "MPSADBW $1,$data(SB),X0"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + if row == "VDBPSADBW.Z $1,X0,X1,X2" { + x86SADReservedZeroMaskGoObservation(t, source) + } else { + goAssemblerOracleOutput(t, "amd64", source, "", false) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + if known, _, err := ctx.lowerPackedSumAbsoluteDifferences(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]); !known || err == nil || output.String() != before { + t.Fatalf("malformed SAD emitted IR %s: %v", row, err) + } + } + for _, row := range []string{"PSADBW X8,X0", "MPSADBW $1,X0,X15", "VPSADBW Y0,Y1,Y20", "VPSADBW Z0,Z1,Z31"} { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + rejected := !strings.HasPrefix(row, "V") || strings.Contains(row, "Z31") + goAssemblerOracleOutput(t, "386", source, "", !rejected) + file, _ := Parse(ArchAMD64, source) + _, _, err := x86PackedSADProperties(file.Funcs[0].Instrs[1], "386") + if rejected && err == nil || !rejected && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("386 accepted spelling acquired physical register %s: %v", row, err) + } + } +} + +func x86SADReservedZeroMaskGoObservation(t *testing.T, source string) { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "sad", "-o", object, asm) + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatalf("actual reserved zero-mask observation: %v\n%s", failure, output) + } + if err != nil { + if !bytes.Contains(output, []byte("mask register must be specified for .Z instructions")) { + t.Fatalf("unexpected original-Go diagnostic: %s", output) + } + t.Log("actual Go rejected unchanged source without its nonzero mask") + return + } + // Intel SDM Vol. 2A Table 2-40 defines EVEX.z=1/aaa=0 as #UD. + // Older Go accepts this source and emits precisely that reserved pattern; + // acceptance is an observation, not authorization to execute or translate it. + if !bytes.Contains(output, []byte("62 f3 75 88 42 d0 01 c3")) { + t.Fatalf("actual Go accepted reserved-body bytes changed: %s", output) + } + code := []byte{0x62, 0xf3, 0x75, 0x88, 0x42, 0xd0, 0x01} + if _, _, known, err := decodedX86RawPackedSADInstruction(code, 64); !known || err == nil { + t.Fatal("actual reserved bytes acquired valid raw semantics") + } + t.Log("actual Go accepted reserved EVEX.z/aaa=0 bytes; named and raw validators still reject them") +} + +func TestX86PackedSADFPSpanAndNativeMemoryBoundary(t *testing.T) { + for _, row := range []string{"VPSADBW source+0(FP),Z0,Z1", "VMPSADBW $37,source+0(FP),Y0,Y1", "VDBPSADBW.Z $228,source+0(FP),Z0,K1,Z1"} { + source := "TEXT sad(SB),4,$0-64\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, _ := Parse(ArchAMD64, source) + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSADProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, missing := range []int{-1, 0, form.memoryBytes - 1} { + sig := x86WordMaddFPType(t) + if missing >= 0 { + for i, slot := range sig.Frame.Params { + if slot.Offset == int64(missing) { + sig.Frame.Params = append(sig.Frame.Params[:i], sig.Frame.Params[i+1:]...) + break + } + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + _, _, err := ctx.lowerPackedSumAbsoluteDifferences(ins.Op, ins) + if (err != nil) != (missing >= 0) || missing >= 0 && output.String() != before { + t.Fatalf("FP complete native-width span %s missing%d: %v", row, missing, err) + } + } + } + for _, row := range []string{"PSADBW source+0(FP),X0", "MPSADBW $1,(SP),X0", "PSADBW 8(BP),X0", "MPSADBW $1,(AX)(BP*2),X0"} { + source := "TEXT f(SB),4,$0-64\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, _ := Parse(ArchAMD64, source) + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], x86WordMaddFPType(t), nil) + before := output.String() + if _, _, err := ctx.lowerPackedSumAbsoluteDifferences(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1]); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("virtual slots became native storage %s: %v", row, err) + } + } +} + +func TestX86PackedSADLowerEffectAndFullReadClassificationAgree(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range x86SADRows(arch) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, known, err := x86PackedSADProperties(ins, arch) + if !known || err != nil || !x86CallerVectorMoveKnown(ins) { + t.Fatalf("lowered accepted SAD lacks registered source effect %s: %v", row, err) + } + for index := range ins.Args { + width, write, valid := x86CallerMoveMemoryShape(ins, index) + if valid != (index == form.memory) || valid && (write || width != int64(form.byteWidth)) { + t.Fatalf("source/native read classifications disagree %s operand%d: %d/%t/%t", row, index, width, write, valid) + } + } + } + } +} + +func TestX86PackedSADDecoderPhysicalRowsDoNotGrantSourcePermissions(t *testing.T) { + for _, encoded := range []Op{"VPSADBW", "VPSADBW.EVEX", "VMPSADBW", "VDBPSADBW"} { + for width := 0; width < 3; width++ { + if width == 2 && (encoded == "VMPSADBW" || encoded == "VPSADBW") { + continue + } + code := encodeX86RawPackedSAD(encoded, width, 0, 1, 2, 0, false, false) + for _, mode := range []int{32, 64} { + ins, length, known, err := decodedX86RawPackedSADInstruction(code, mode) + if !known || err != nil || length != len(code) { + t.Fatalf("physical %x mode%d: %v", code, mode, err) + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + if _, known, err := x86PackedSADProperties(ins, arch); !known || err != nil { + t.Fatalf("actual physical decoder row %x: %v", code, err) + } + if mode == 32 && len(ins.Args) > 3 { + local := ins + local.x86Encoded = false + if _, _, err := x86PackedSADProperties(local, arch); err == nil { + t.Fatal("named/public source borrowed physical frontend permission") + } + } + // Byte identity is not initialized input or pointer permission. + body := x86SourceStackState{vectors: make(map[Reg]uint64), masks: make(map[Reg]bool)} + form, _, _ := x86PackedSADProperties(ins, "amd64") + if !x86CallerSADStep(&ins, form, &body, nil) { + t.Fatal("pure register effect unexpectedly failed") + } + knownBytes := body.vectors[x86CallerVectorRoot(ins.Args[form.destination].Reg)] + if knownBytes&3 != 0 { + t.Fatal("raw producer invented initialized SAD input bytes") + } + } + } + } +} diff --git a/amd64_packed_sad_reference_test.go b/amd64_packed_sad_reference_test.go new file mode 100644 index 000000000..0e6c18a33 --- /dev/null +++ b/amd64_packed_sad_reference_test.go @@ -0,0 +1,47 @@ +package plan9asm + +// Test-only unsigned byte mathematics. It knows no producer/parser/LLVM +// helper and is also used to construct the separate original-Go oracle. +func x86SADReference(op string, width, control int, first, second []byte) []uint16 { + result := make([]uint16, width/2) + difference := func(a, b byte) uint16 { + if a >= b { + return uint16(a) - uint16(b) + } + return uint16(b) - uint16(a) + } + if op == "PSADBW" || op == "VPSADBW" { + for byteIndex := 0; byteIndex < width; byteIndex++ { + result[byteIndex/8*4] += difference(first[byteIndex], second[byteIndex]) + } + return result + } + if op == "MPSADBW" || op == "VMPSADBW" { + for lane := 0; lane < width/16; lane++ { + selector := control >> (lane * 3) + block := lane*16 + (selector&3)*4 + window := lane*16 + (selector>>2&1)*4 + for word := 0; word < 8; word++ { + for byteIndex := 0; byteIndex < 4; byteIndex++ { + result[lane*8+word] += difference(first[block+byteIndex], second[window+word+byteIndex]) + } + } + } + return result + } + for lane := 0; lane < width/16; lane++ { + var permuted [16]byte + for word := 0; word < 4; word++ { + copy(permuted[word*4:word*4+4], first[lane*16+(control>>(2*word)&3)*4:]) + } + for word := 0; word < 8; word++ { + block := word / 4 * 8 + window := block + word%4 + other := lane*16 + block + (word%4/2)*4 + for byteIndex := 0; byteIndex < 4; byteIndex++ { + result[lane*8+word] += difference(permuted[window+byteIndex], second[other+byteIndex]) + } + } + } + return result +} diff --git a/amd64_packed_sad_runtime_test.go b/amd64_packed_sad_runtime_test.go new file mode 100644 index 000000000..7902d92e8 --- /dev/null +++ b/amd64_packed_sad_runtime_test.go @@ -0,0 +1,324 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86SADNumericCase struct { + op, bank string + width, imm, masking int + memory bool +} + +func x86SADNumericCases() []x86SADNumericCase { + var cases []x86SADNumericCase + for _, op := range []string{"PSADBW", "VPSADBW", "MPSADBW", "VMPSADBW", "VDBPSADBW"} { + for widthIndex, bank := range []string{"X", "Y", "Z"} { + if !strings.HasPrefix(op, "V") && bank != "X" || op == "VMPSADBW" && bank == "Z" { + continue + } + controls := []int{0} + if strings.Contains(op, "MPS") || op == "VDBPSADBW" { + controls = []int{0, 1, 2, 3, 4, 5, 6, 7, 27, 37, 85, 170, 228, 255} + } + for _, control := range controls { + for _, memory := range []bool{false, true} { + for mask := 0; mask < 3; mask++ { + if mask != 0 && op != "VDBPSADBW" { + continue + } + cases = append(cases, x86SADNumericCase{op, bank, 16 << widthIndex, control, mask, memory}) + } + } + } + } + } + return cases +} + +func (row x86SADNumericCase) instruction() string { + first := row.bank + "0" + if row.memory { + first = "(CX)" + } + op := row.op + if row.masking == 2 { + op += ".Z" + } + if op == "PSADBW" { + return op + " " + first + ",X2" + } + if op == "MPSADBW" { + return fmt.Sprintf("%s $%d,%s,X2", op, row.imm, first) + } + if op == "VPSADBW" { + return fmt.Sprintf("%s %s,%s1,%s2", op, first, row.bank, row.bank) + } + mask := "" + if row.masking != 0 { + mask = "K1," + } + return fmt.Sprintf("%s $%d,%s,%s1,%s%s2", op, row.imm, first, row.bank, mask, row.bank) +} + +func x86SADOriginalLegacyOracle(t *testing.T) { + t.Helper() + var declarations, assembly, tests strings.Builder + declarations.WriteString("package sad\n") + tests.WriteString(`package sad +import("testing";"encoding/binary") +func TestOriginal(t *testing.T) { + var a,b [64]byte + for sample:=0;sample<32;sample++ { + for i:=range a {a[i]=byte(i*17+sample*31);b[i]=byte(i*73-sample*19)} +`) + // Actual stock Go executes every legacy imm8, in both register and + // m128 forms. It does not execute AVX512 or the Z-alias observation program. + for _, op := range []string{"PSADBW", "MPSADBW"} { + controls := 1 + if op == "MPSADBW" { + controls = 256 + } + for imm := 0; imm < controls; imm++ { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("original%d", strings.Count(declarations.String(), "func ")) + fmt.Fprintf(&declarations, "func %s(out *[24]byte,a,b *[64]byte,flags uint64)\n", name) + row := x86SADNumericCase{op: op, bank: "X", width: 16, imm: imm, memory: memory} + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVUPS (CX),X0\nMOVUPS (DX),X2\nPUSHQ flags+24(FP)\nPOPFQ\n%s\nMOVUPS X2,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,16(AX)\nRET\n", name, row.instruction()) + fmt.Fprintf(&tests, `{ + var out [24]byte + beforeA,beforeB:=a,b + flags:=uint64(2) + if sample&1!=0 {flags|=0x8d5} + %s(&out,&a,&b,flags) + want:=[8]uint16{} +`, name) + if op == "PSADBW" { + tests.WriteString("for i:=0;i<16;i++ {d:=int(a[i])-int(b[i]);if d<0 {d=-d};want[i/8*4]+=uint16(d)}\n") + } else { + fmt.Fprintf(&tests, "for word:=0;word<8;word++ {for i:=0;i<4;i++ {d:=int(a[%d+i])-int(b[%d+word+i]);if d<0 {d=-d};want[word]+=uint16(d)}}\n", (imm&3)*4, (imm>>2&1)*4) + } + fmt.Fprintf(&tests, "for i,w:=range want {if g:=binary.LittleEndian.Uint16(out[i*2:]);g!=w {t.Fatalf(\"%s sample%%d word%%d got%%d want%%d\",sample,i,g,w)}}\n", name) + tests.WriteString("if binary.LittleEndian.Uint64(out[16:])&0x8d5!=flags&0x8d5 || a!=beforeA || b!=beforeB {t.Fatal(\"flags/input canary\")}\n}\n") + } + } + } + tests.WriteString("}\n}\n") + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module sad\n\ngo 1.20\n", "sad.go": declarations.String(), "sad_amd64.s": assembly.String(), "sad_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil { + t.Fatalf("actual original legacy SAD oracle: %v\n%s", err, output) + } + t.Logf("actual original SSE %d kernels x 32 initialized samples; full status flags and unchanged input canaries", strings.Count(declarations.String(), "func ")) +} + +func TestCrossLinuxRuntimeMatrixX86PackedSADOriginalVEXUpperAliases(t *testing.T) { + x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package sad\n") + tests.WriteString(`package sad +import("testing";"encoding/binary") +func TestOriginal(t *testing.T) { + var a,b,old [64]byte + for sample:=0;sample<32;sample++ { + for i:=range a {a[i]=byte(i*17+sample*31);b[i]=byte(i*73-sample*19);old[i]=byte(0x91+i*11)} +`) + kernels := 0 + for _, op := range []string{"PSADBW", "MPSADBW", "VPSADBW", "VMPSADBW"} { + controls := []int{0} + if strings.Contains(op, "MPS") { + controls = []int{0, 1, 2, 3, 4, 5, 6, 7, 27, 37, 85, 170, 228, 255} + } + for _, imm := range controls { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("alias%d", kernels) + kernels++ + row := x86SADNumericCase{op: op, bank: "X", width: 16, imm: imm, memory: memory} + fmt.Fprintf(&declarations, "func %s(out *[48]byte,a,b,old *[64]byte,flags uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-40\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVUPS (CX),X0\nMOVUPS (DX),X1\nVMOVUPS (R8),Y2\n", name) + legacy := !strings.HasPrefix(op, "V") + if legacy { + assembly.WriteString("MOVUPS (DX),X2\n") + } + fmt.Fprintf(&assembly, "PUSHQ flags+32(FP)\nPOPFQ\n%s\nVMOVUPS Y2,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,32(AX)\nRET\n", row.instruction()) + fmt.Fprintf(&tests, "{\n var out,want [48]byte\n for i:=range out {out[i]=0xc3;want[i]=0xc3}\n beforeA,beforeB,beforeOld:=a,b,old\n flags:=uint64(2)\n if sample&1!=0 {flags|=0x8d5}\n %s(&out,&a,&b,&old,flags)\n sums:=[8]uint16{}\n", name) + if strings.Contains(op, "MPS") { + fmt.Fprintf(&tests, "for word:=0;word<8;word++ {for i:=0;i<4;i++ {d:=int(a[%d+i])-int(b[%d+word+i]);if d<0 {d=-d};sums[word]+=uint16(d)}}\n", (imm&3)*4, (imm>>2&1)*4) + } else { + tests.WriteString("for i:=0;i<16;i++ {d:=int(a[i])-int(b[i]);if d<0 {d=-d};sums[i/8*4]+=uint16(d)}\n") + } + tests.WriteString("for i,w:=range sums {binary.LittleEndian.PutUint16(want[2*i:],w)}\n") + if legacy { + tests.WriteString("copy(want[16:32],old[16:32])\n") + } else { + tests.WriteString("for i:=16;i<32;i++ {want[i]=0}\n") + } + tests.WriteString("observed:=binary.LittleEndian.Uint64(out[32:40])\n binary.LittleEndian.PutUint64(want[32:40],observed)\n") + fmt.Fprintf(&tests, "if out!=want || observed&0x8d5!=flags&0x8d5 || a!=beforeA || b!=beforeB || old!=beforeOld {t.Fatalf(\"%s sample%%d actual%%x want%%x flags%%x\",sample,out,want,observed)}\n}\n", name) + } + } + } + tests.WriteString("}\n}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + dir := t.TempDir() + for name, source := range map[string]string{"go.mod": "module sad\n\ngo 1.20\n", "sad.go": declarations.String(), "sad_amd64.s": assembly.String(), "sad_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + output, err := runtimeOracleCombinedOutput(ctx, command) + if err != nil { + t.Fatalf("actual original SSE/VEX upper-alias oracle: %v\n%s", err, output) + } + t.Logf("actual original SSE/VEX %d kernels x 32 initialized samples; all 32 output bytes, six status flags and canaries; no EVEX instruction executed", kernels) +} + +func TestCrossLinuxRuntimeMatrixX86PackedSADIndependentWholeBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + x86SADOriginalLegacyOracle(t) + var declarations, assembly, table strings.Builder + declarations.WriteString("package sad\n") + cases := x86SADNumericCases() + for index, row := range cases { + name := fmt.Sprintf("sad%d", index) + fmt.Fprintf(&declarations, "func %s(out *[72]byte,a,b,old *[64]byte,mask,flags uint64)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-48\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),CX\nMOVQ b+16(FP),DX\nMOVQ old+24(FP),R8\nMOVQ mask+32(FP),R9\nVMOVUPS (CX),Z0\nVMOVUPS (DX),Z1\nVMOVUPS (R8),Z2\nKMOVQ R9,K1\n", name) + if !strings.HasPrefix(row.op, "V") { + assembly.WriteString("MOVUPS (DX),X2\n") + } + // Isolate the SAD preservation contract from arithmetic flag producers. + // Both seeds still demand all six defined status bits, including AF. + fmt.Fprintf(&assembly, "PUSHQ flags+40(FP)\nPOPFQ\n%s\nVMOVUPS Z2,(AX)\nPUSHFQ\nPOPQ R10\nMOVQ R10,64(AX)\nRET\n", row.instruction()) + kind := 0 + if strings.Contains(row.op, "MPS") { + kind = 1 + } else if row.op == "VDBPSADBW" { + kind = 2 + } + legacy := 0 + if !strings.HasPrefix(row.op, "V") { + legacy = 1 + } + fmt.Fprintf(&table, "{sad%d,%d,%d,%d,%d,%d},\n", index, kind, row.width, row.imm, row.masking, legacy) + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "sad", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for index := range cases { + name := fmt.Sprintf("sad%d", index) + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, Options{ + Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + var main strings.Builder + main.WriteString(`#include +#include +#include +typedef void (*kernel)(void*,const void*,const void*,const void*,uint64_t,uint64_t); +`) + for index := range cases { + fmt.Fprintf(&main, "extern void sad%d(void*,const void*,const void*,const void*,uint64_t,uint64_t);\n", index) + } + main.WriteString("static const struct {kernel run;int kind,width,imm,masking,legacy;} cases[]={\n" + table.String() + "};\n") + main.WriteString(` +static unsigned diff(unsigned a,unsigned b) {return a>b ? a-b : b-a;} +static void reference(uint8_t *out,int kind,int width,int imm,const uint8_t *a,const uint8_t *b) { + memset(out,0,width); + uint16_t sums[32]={0}; + if(kind==0) { + for(int i=0;i>(3*lane),block=lane*16+(select&3)*4,window=lane*16+((select>>2)&1)*4; + for(int word=0;word<8;++word) for(int i=0;i<4;++i) sums[lane*8+word]+=diff(a[block+i],b[window+word+i]); + } + } else { + for(int lane=0;lane>(2*d))&3)*4,4); + for(int word=0;word<8;++word) for(int i=0;i<4;++i) + sums[lane*8+word]+=diff(shuffled[(word/4)*8+word%4+i],b[lane*16+(word/4)*8+(word%4/2)*4+i]); + } + } + for(int word=0;word>8;} +} +int main(void) { + for(unsigned row=0;row>(i/2)&1)) value=cases[row].masking==2 ? 0 : old[i]; + want[1+i]=value; + } + uint64_t flags;memcpy(&flags,out+65,8);memcpy(want+65,&flags,8); + if((flags&0x8d5)!=(seededFlags&0x8d5) || memcmp(out,want,sizeof(out)) || memcmp(aa,beforeA,sizeof(aa)) || memcmp(bb,beforeB,sizeof(bb))) { + fprintf(stderr,"SAD row=%u kind=%d width=%d imm=%d masking=%d legacy=%d sample=%u flags=%llx\n",row,cases[row].kind,cases[row].width,cases[row].imm,cases[row].masking,cases[row].legacy,sample,(unsigned long long)flags); + for(int i=0;i<80;++i) if(out[i]!=want[i]) fprintf(stderr,"byte%d got=%u want=%u\n",i,out[i],want[i]); + return 1; + } + } + return 0; +} +`) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + ir := module.String() + if optimized { + ir = x86MulDivRoundOptimize(t, ir) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "sad-independent-whole-bytes", triple, ir, main.String(), prefix) + t.Logf("%d initialized kernels x 32 inputs; all 64 alias bytes, merge/zero masks, six status flags and canaries", len(cases)) + }) + } + // These are LLVM software-lowering numerical tests plus actual original + // SSE execution. Z source acceptance/objects do not prove native AVX512. +} diff --git a/amd64_packed_shift_aligned_count_test.go b/amd64_packed_shift_aligned_count_test.go new file mode 100644 index 000000000..11b850778 --- /dev/null +++ b/amd64_packed_shift_aligned_count_test.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86LegacyShiftMemoryOpcodes = []struct { + name string + code byte +}{ + {"PSLLW", 0xf1}, {"PSLLL", 0xf2}, {"PSLLQ", 0xf3}, + {"PSRLW", 0xd1}, {"PSRLL", 0xd2}, {"PSRLQ", 0xd3}, +} + +func TestX86PackedShiftLegacyMemoryActualGoEncoding(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, operation := range x86LegacyShiftMemoryOpcodes { + source := "TEXT shifts(SB),4,$0-0\n" + operation.name + " (CX),X0\nRET\n" + dir := t.TempDir() + input, object := filepath.Join(dir, "original.s"), filepath.Join(dir, "original.o") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", object, input) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch, "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual Go %s %s: %v\n%s", arch, operation.name, err, output) + } + var code []byte + for _, line := range regexp.MustCompile(`(?m)^\s*0x[[:xdigit:]]+\s+((?:[[:xdigit:]]{2} )+)`).FindAllSubmatch(output, -1) { + part, err := hex.DecodeString(strings.ReplaceAll(string(line[1]), " ", "")) + if err != nil { + t.Fatal(err) + } + code = append(code, part...) + } + want := []byte{0x66, 0x0f, operation.code, 0x01, 0xc3} + if !bytes.Equal(code, want) { + t.Fatalf("actual Go %s %s memory encoding=%x, want=%x\n%s", arch, operation.name, code, want, output) + } + } + } +} + +func TestX86PackedShiftNativeCountThreeAPIsFiveO2Objects(t *testing.T) { + llc, disassembler := findLLVM22Tool("llc"), findLLVM22Tool("llvm-objdump") + if llc == "" || disassembler == "" { + t.Fatal("LLVM22 llc and llvm-objdump required") + } + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, operation := range x86LegacyShiftMemoryOpcodes { + for _, memory := range []string{"(AX)", "8(AX)(FS)", "8(AX)(GS)"} { + fmt.Fprintf(&source, "%s %s,X0\n", operation.name, memory) + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + result, err := translateGoModuleInContext(context, x86PackedShiftGoPackage(t, target.arch), + []byte(strings.Replace(source.String(), "shifts(SB)", "·shifts(SB)", 1)), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + for api, body := range []string{ir, module.String(), result.Module.String()} { + for _, mnemonic := range []string{"psllw", "pslld", "psllq", "psrlw", "psrld", "psrlq"} { + for _, segment := range []string{"", "%fs:", "%gs:"} { + if strings.Count(body, `asm sideeffect "`+mnemonic+` `+segment+`($2), $0", "=x,0,r,~{memory}"`) != 1 { + t.Fatalf("%s API%d lost original %s %s m128 count operation", target.triple, api, mnemonic, segment) + } + } + } + if strings.Contains(body, `asm sideeffect "movdqa`) { + t.Fatal("count access was replaced by a stronger MOVDQA access") + } + dir := t.TempDir() + input, object := filepath.Join(dir, "native-count.ll"), filepath.Join(dir, "native-count.o") + if err := os.WriteFile(input, []byte(body), 0600); err != nil { + t.Fatal(err) + } + stage, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(stage, llc, "-O2", "-mtriple="+target.triple, "-filetype=obj", input, "-o", object) + output, err := runtimeOracleCombinedOutput(stage, command) + cancel() + if err != nil { + t.Fatalf("O2 %s API%d: %v\n%s", target.triple, api, err, output) + } + stage, cancel = runtimeOracleStageContext(t) + command = exec.CommandContext(stage, disassembler, "-d", object) + output, err = runtimeOracleCombinedOutput(stage, command) + cancel() + if err != nil { + t.Fatalf("O2 disassembly %s API%d: %v\n%s", target.triple, api, err, output) + } + for _, mnemonic := range []string{"psllw", "pslld", "psllq", "psrlw", "psrld", "psrlq"} { + pattern := regexp.MustCompile(`(?m)\b` + mnemonic + `\s+[^\n]*\([^\n]*\)`) + if len(pattern.FindAll(output, -1)) != 3 { + t.Fatalf("O2 actual original %s m128 count %s API%d:\n%s", mnemonic, target.triple, api, output) + } + } + for _, segment := range []string{"%fs:", "%gs:"} { + if strings.Count(string(output), segment) != 6 { + t.Fatalf("O2 lost six native %s count overrides %s API%d:\n%s", segment, target.triple, api, output) + } + } + } + }() + } +} + +func TestX86PackedShiftLegacyNativeBackingAndCallerPermissionsRemainContext(t *testing.T) { + for _, operation := range x86LegacyShiftMemoryOpcodes { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+operation.name+" count+0(FP),X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.Len() + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("unproved native FP backing emitted IR: %s %v", operation.name, err) + } + } + // Initialized vectors do not supply native backing or memory permission. + for _, operation := range x86LegacyShiftMemoryOpcodes { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+operation.name+" (CX),X0\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !recognized || err != nil { + t.Fatal(err) + } + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": ^uint64(0)}} + if x86CallerPackedShiftStep(&ins, form, &state, nil) { + t.Fatal("initialized vector granted legacy native-memory permission") + } + } +} diff --git a/amd64_packed_shift_alignment_linux_test.go b/amd64_packed_shift_alignment_linux_test.go new file mode 100644 index 000000000..3410b242d --- /dev/null +++ b/amd64_packed_shift_alignment_linux_test.go @@ -0,0 +1,317 @@ +//go:build linux && (amd64 || arm64) + +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +func x86PackedShiftLinuxExecution(t *testing.T) []string { + t.Helper() + if runtime.GOARCH == "amd64" { + return nil // Required native Linux/amd64 counterpart, not Darwin Rosetta. + } + qemu, err := exec.LookPath("qemu-x86_64") + if err != nil { + t.Fatal("required explicit QEMU x86 execution:", err) + } + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, qemu, "--version")) + cancel() + if err != nil || !strings.HasPrefix(string(output), "qemu-x86_64 version 10.2.3 ") { + t.Fatalf("required QEMU10.2.3: %v\n%s", err, output) + } + t.Logf("explicit executable transport: %s -L /usr/x86_64-linux-gnu", qemu) + return []string{qemu, "-L", "/usr/x86_64-linux-gnu"} +} + +const x86PackedShiftLegacyMemoryGoDriver = `package main +import ( + "fmt" + "os" + "strconv" + "unsafe" +) +func numeric(functions []func(unsafe.Pointer,unsafe.Pointer,unsafe.Pointer,uint64)) { + limits:=[]uint64{0,1,7,15,16,17,31,32,33,63,64,65,255,256,1<<32,^uint64(0)} + flags:=[3][5]byte{{1,0,1,0,1},{0,1,0,1,1},{0,0,0,0,0}} + for kernel,fn:=range functions { + for round,count:=range limits { + for seed:=0;seed<3;seed++ { + var data [18]byte + var storage [96]byte + var out [23]byte + for i:=range data { data[i]=byte(i*71+round*97+13) } + for i:=range storage { storage[i]=byte(i*31+7) } + for i:=range out { out[i]=0xc3 } + start:=uintptr(unsafe.Pointer(&storage[0])) + offset:=int((16-start%16)%16) + for i:=0;i<8;i++ { storage[offset+i]=byte(count>>(8*i)) } + beforeData,beforeStorage:=data,storage + fn(unsafe.Pointer(&out[1]),unsafe.Pointer(&data[1]),unsafe.Pointer(&storage[offset]),uint64(seed)) + if out[0]!=0xc3 || out[22]!=0xc3 || data!=beforeData || storage!=beforeStorage { + panic("canary/input mutation") + } + for i:=0;i<5;i++ { + if out[17+i]!=flags[seed][i] { panic("flags") } + } + bytes:=[]int{2,4,8,2,4,8}[kernel] + for i:=0;i<16;i+=bytes { + var value,want uint64 + for b:=0;b>count } + } + for b:=0;b>(8*b)) { panic("numeric") } + } + } + } + } + } +} +func main() { + functions:=[]func(unsafe.Pointer,unsafe.Pointer,unsafe.Pointer,uint64){legacycount0,legacycount1,legacycount2,legacycount3,legacycount4,legacycount5} + if os.Args[1]=="numeric" { numeric(functions); return } + index,err:=strconv.Atoi(os.Args[1]) + if err!=nil { panic(err) } + var data,count [64]byte + var out [23]byte + for i:=range data { + data[i]=byte(i*71+13) + count[i]=byte(i*31+7) + } + for i:=range out { out[i]=0xc3 } + start:=uintptr(unsafe.Pointer(&count[0])) + offset:=(16-start%16)%16+1 + beforeData,beforeCount:=data,count + fmt.Printf("original Go count mod16=%d\n",uintptr(unsafe.Pointer(&count[offset]))%16) + functions[index](unsafe.Pointer(&out[1]),unsafe.Pointer(&data[1]),unsafe.Pointer(&count[offset]),0) + if out[0]!=0xc3 || out[22]!=0xc3 || data!=beforeData || count!=beforeCount { panic("canary/input mutation") } + flags:=[5]byte{1,0,1,0,1} + for i:=0;i<5;i++ { if out[17+i]!=flags[i] { panic("flags") } } + /* The initialized low count qword exceeds every lane width. */ + for i:=1;i<=16;i++ { if out[i]!=0 { panic("numeric") } } + os.Exit(90) +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedShiftOriginalLegacyMemory(t *testing.T) { + prefix := x86PackedShiftLinuxExecution(t) + x86PackedShiftVerifyGuestArguments(t, prefix) + _, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + observer := x86PackedShiftBuildOriginalCountObserver(t, x86PackedShiftLinuxCCompiler(t, clang)) + assembly, _ := x86PackedShiftLegacyMemoryAssembly() + var declarations strings.Builder + for i := range x86LegacyShiftMemoryOpcodes { + fmt.Fprintf(&declarations, "func legacycount%d(out,data,count unsafe.Pointer,seed uint64)\n", i) + } + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module originalcount\n\ngo 1.20\n", + "main.go": x86PackedShiftLegacyMemoryGoDriver + declarations.String(), + "original_amd64.s": strings.ReplaceAll(assembly, "TEXT legacycount", "TEXT ·legacycount"), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + env := append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + executable := filepath.Join(dir, "original") + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "build", "-p=2", "-o", executable, ".") + command.Dir, command.Env = dir, env + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go body/link: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + argv := x86PackedShiftGuestExecution(prefix, executable, "numeric") + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go aligned numeric/flags: %v\n%s", err, output) + } + for i := range x86LegacyShiftMemoryOpcodes { + ctx, cancel = runtimeOracleStageContext(t) + argv = x86PackedShiftGuestExecution(prefix, observer, strconv.Itoa(i)) + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + reference, referenceErr := runtimeOracleCombinedOutput(ctx, command) + cancel() + status, observationErr := x86PackedShiftCountObservation(i, reference, referenceErr) + if observationErr != nil { + t.Fatalf("original native opcode observer %d: %v\n%s", i, observationErr, reference) + } + ctx, cancel = runtimeOracleStageContext(t) + argv := x86PackedShiftGuestExecution(prefix, executable, strconv.Itoa(i)) + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + exit, pureExit := err.(*exec.ExitError) + text := string(output) + if !strings.Contains(text, "original Go count mod16=1") || !pureExit { + t.Fatalf("original Go unaligned m128 count identity %d: %v\n%s", i, err, output) + } + if status == "normal" { + if exit.ExitCode() != 90 { + t.Fatalf("original Go differs from normal native opcode %d: %v\n%s", i, err, output) + } + } else if exit.ExitCode() != 2 || !strings.Contains(text, "main.legacycount") || + (!strings.Contains(text, "unexpected fault address") && !strings.Contains(text, "invalid memory address")) { + t.Fatalf("original Go differs from native opcode memory fault %d: %v\n%s", i, err, output) + } + t.Logf("original Go/native opcode %s mod16=1 status=%s", x86LegacyShiftMemoryOpcodes[i].name, status) + } + t.Log("6 original Go m128 kernels ×48 numeric/flags/canary inputs and 6 independent native opcode unaligned status comparisons") +} + +func x86PackedShiftVerifyGuestArguments(t *testing.T, prefix []string) { + t.Helper() + dir := t.TempDir() + const source = `package main +import ( + "fmt" + "os" +) +func main() { fmt.Printf("%q\n", os.Args) } +` + for name, contents := range map[string]string{ + "go.mod": "module argumenttransport\n\ngo 1.20\n", "main.go": source, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(dir, "argument-transport") + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "build", "-p=2", "-o", executable, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0", + "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("original Go argument witness body/link: %v\n%s", err, output) + } + ctx, cancel = runtimeOracleStageContext(t) + argv := append(append([]string(nil), prefix...), executable) + command = exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err = runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || string(output) != fmt.Sprintf("%q\n", []string{executable}) { + t.Fatalf("guest no-argument witness: %v\n%s", err, output) + } + t.Logf("guest no-argument witness: %s", strings.TrimSpace(string(output))) + arguments := []string{"first", "with space"} + for _, repaired := range []bool{false, true} { + argv := append(append([]string(nil), prefix...), executable) + argv = append(argv, arguments...) + want := append([]string{executable}, arguments...) + if len(prefix) != 0 && !repaired { + // The authenticated binfmt release treats the first argument as + // original argv[0] even for ordinary explicit execution. + want = arguments + } + if repaired { + argv = x86PackedShiftGuestExecution(prefix, executable, arguments...) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, argv[0], argv[1:]...) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || string(output) != fmt.Sprintf("%q\n", want) { + t.Fatalf("guest argument witness repaired=%t command=%q: %v\n%s; want %q", + repaired, argv, err, output, fmt.Sprintf("%q\n", want)) + } + t.Logf("guest argument witness repaired=%t: %s", repaired, strings.TrimSpace(string(output))) + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedShiftLLVMNativeCount(t *testing.T) { + prefix := x86PackedShiftLinuxExecution(t) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + assembly, sigs := x86PackedShiftLegacyMemoryAssembly() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + const triple = "x86_64-unknown-linux-gnu" + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + driver := x86PackedShiftLegacyCountCHeader + x86PackedShiftLegacyCountDeclarations() + x86PackedShiftOriginalCountC() + + x86PackedShiftCountCChecks + x86PackedShiftLegacyNativeCountCMain + compileAndRunRuntimeTestWithCompiler(t, llc, x86PackedShiftLinuxCCompiler(t, clang), "legacy_count_contract", triple, ir, driver, prefix) + t.Log("6 original opcode/LLVM m128 kernels ×48 numeric/flags/canary inputs, 90 unaligned status comparisons and 6 original/LLVM full128 guard-page faults") +} + +func x86PackedShiftLinuxCCompiler(t *testing.T, clang string) []string { + t.Helper() + if runtime.GOARCH == "amd64" { + return []string{clang} + } + gcc, err := exec.LookPath("x86_64-linux-gnu-gcc") + if err != nil { + t.Fatal("required AMD64 C/link driver:", err) + } + return []string{gcc, "-no-pie"} +} + +func TestCrossLinuxRuntimeMatrixX86PackedShiftCompleteNumerical(t *testing.T) { + prefix := x86PackedShiftLinuxExecution(t) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + cases := x86PackedShiftNumericalCases() + assembly, sigs := x86PackedShiftNumericalAssembly(cases) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + const triple = "x86_64-unknown-linux-gnu" + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + var driver strings.Builder + driver.WriteString(x86PackedShiftIndependentC) + for _, c := range cases { + fmt.Fprintf(&driver, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t);\n", c.name) + } + driver.WriteString("static const struct shift_case cases[]={\n") + for _, c := range cases { + fmt.Fprintf(&driver, "{%s,%q,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d},\n", c.name, c.row, + c.width, c.bits, c.countMode, c.immediate, c.memory, c.maskMode, c.destination, + x86ShiftBoolean(c.left), x86ShiftBoolean(c.legacy), x86ShiftBoolean(c.broadcast)) + } + driver.WriteString(`}; +int main(void) { + for(unsigned i=0;i= 6 || len(output) > 256 || !strings.HasSuffix(string(output), "\n") { + return "", fmt.Errorf("unbound original count observation") + } + lines := strings.Split(strings.TrimSuffix(string(output), "\n"), "\n") + diagnostic := false + if len(lines) == 2 && lines[0] == "qemu: uncaught target signal 11 (Segmentation fault) - core dumped" { + diagnostic = true + lines = lines[1:] + } + if len(lines) != 1 { + return "", fmt.Errorf("unexpected original count observer output: %q", output) + } + prefix := fmt.Sprintf("COUNT_NATIVE_STATUS kernel=%d mod16=1 ", kernel) + switch lines[0] { + case prefix + "wait=0 outcome=normal": + if !diagnostic { + return "normal", nil + } + case prefix + "wait=11 outcome=fault", prefix + "wait=139 outcome=fault": + return "fault", nil + } + return "", fmt.Errorf("invalid original count status/kernel/address: %q", output) +} diff --git a/amd64_packed_shift_dependency_test.go b/amd64_packed_shift_dependency_test.go new file mode 100644 index 000000000..876316bf3 --- /dev/null +++ b/amd64_packed_shift_dependency_test.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +// An independent bit-by-bit relation, not the production interval arithmetic. +// Every possible in-range dynamic count is enumerated; all other counts emit +// zero. Operand bytes are known only when every possible selected bit is known. +func x86ShiftIndependentBitInputs(bits, width, count int, left, byteShift, dynamic, perLane bool) [][2]uint64 { + if byteShift { + bits = 128 + } + masks := make([][2]uint64, width) + for output := 0; output < width*8; output++ { + first, end := int(uint8(count)), int(uint8(count))+1 + if byteShift { + first, end = first*8, first*8+1 + } + if dynamic { + first, end = 0, bits + } + for shift := first; shift < end; shift++ { + input := output%bits + shift + if left { + input = output%bits - shift + } + if input >= 0 && input < bits { + input += output / bits * bits + masks[output/8][0] |= uint64(1) << uint(input/8) + } + } + if dynamic { + start, end := 0, 8 + if perLane { + start, end = output/bits*(bits/8), (output/bits+1)*(bits/8) + } + for at := start; at < end; at++ { + masks[output/8][1] |= uint64(1) << uint(at) + } + } + } + return masks +} + +func TestX86PackedShiftResultEveryByteDependency(t *testing.T) { + for _, bank := range []struct { + name string + width int + }{{"X", 16}, {"Y", 32}, {"Z", 64}} { + for _, bits := range []int{16, 32, 64, 128} { + for _, left := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + byteShift, dynamic, perLane := bits == 128, mode != 0, mode == 2 + if byteShift && dynamic { + continue + } + name := "VPSRL" + if left { + name = "VPSLL" + } + if perLane { + name += "V" + } + name += map[int]string{16: "W", 32: "D", 64: "Q", 128: "DQ"}[bits] + last := 255 + if dynamic { + last = 0 + } + for immediate := 0; immediate <= last; immediate++ { + count := fmt.Sprintf("$%d", immediate) + if dynamic { + count = "X0" + if perLane { + count = bank.name + "0" + } + } + row := fmt.Sprintf("%s %s,%s1,%s2", name, count, bank.name, bank.name) + ins := x86ScalarLaneInstruction(t, row) + form, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !known || err != nil { + t.Fatal(row, err) + } + want := x86ShiftIndependentBitInputs(bits, bank.width, immediate, left, byteShift, dynamic, perLane) + for output, masks := range want { + data, counts := x86PackedShiftByteInputs(form, output) + if data != masks[0] || counts != masks[1] { + t.Fatalf("%s byte=%d data=%x/%x counts=%x/%x", row, output, data, masks[0], counts, masks[1]) + } + } + for missing := -1; missing < bank.width; missing++ { + for operand := 0; operand < 2; operand++ { + inputs := [2]uint64{^uint64(0), ^uint64(0)} + if missing >= 0 { + inputs[operand] &^= uint64(1) << uint(missing) + } + var expected uint64 + for output, masks := range want { + if inputs[0]&masks[0] == masks[0] && inputs[1]&masks[1] == masks[1] { + expected |= uint64(1) << uint(output) + } + } + if got := x86PackedShiftDefinedBytes(form, inputs[0], inputs[1], 0); got != expected { + t.Fatal(row, missing, operand, got, expected) + } + state := x86ResultState{flags: x86ByteFlagsAll} + state.writeVector(0, 64, inputs[1], false) + state.writeVector(1, 64, inputs[0], false) + physical := expected | ^x86CallerVectorByteMask(int64(bank.width)) + if !x86ResultTransfer(ins, &state, nil, 0, "amd64") || state.vectorBytes(2) != physical || state.flags != x86ByteFlagsAll { + t.Fatal("complete result byte state differs", row, missing, operand) + } + } + } + } + } + } + } + } +} diff --git a/amd64_packed_shift_exec_test.go b/amd64_packed_shift_exec_test.go new file mode 100644 index 000000000..e321197e1 --- /dev/null +++ b/amd64_packed_shift_exec_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "reflect" + "testing" +) + +func x86PackedShiftGuestExecution(prefix []string, executable string, arguments ...string) []string { + command := append([]string(nil), prefix...) + command = append(command, executable) + if len(prefix) != 0 { + // The pinned binfmt QEMU preserves a separate original argv[0]. + // Native execution must not duplicate it. The Linux runtime oracle + // independently verifies this pinned transport before any ASM runs. + command = append(command, executable) + } + return append(command, arguments...) +} + +func TestX86PackedShiftGuestArgumentTransport(t *testing.T) { + for _, c := range []struct { + name string + prefix []string + arguments []string + want []string + }{ + {"native", nil, []string{"numeric"}, []string{"/guest", "numeric"}}, + {"native_empty", nil, nil, []string{"/guest"}}, + {"pinned_empty", []string{"/qemu", "-L", "/sysroot"}, nil, + []string{"/qemu", "-L", "/sysroot", "/guest", "/guest"}}, + {"pinned_numeric", []string{"/qemu", "-L", "/sysroot"}, []string{"numeric"}, + []string{"/qemu", "-L", "/sysroot", "/guest", "/guest", "numeric"}}, + {"pinned_fault", []string{"/qemu", "-L", "/sysroot"}, []string{"3", "with space"}, + []string{"/qemu", "-L", "/sysroot", "/guest", "/guest", "3", "with space"}}, + } { + t.Run(c.name, func(t *testing.T) { + beforePrefix := append([]string(nil), c.prefix...) + beforeArguments := append([]string(nil), c.arguments...) + got := x86PackedShiftGuestExecution(c.prefix, "/guest", c.arguments...) + if !reflect.DeepEqual(got, c.want) { + t.Fatalf("guest command: got %q, want %q", got, c.want) + } + if !reflect.DeepEqual(c.prefix, beforePrefix) || !reflect.DeepEqual(c.arguments, beforeArguments) { + t.Fatal("argument construction mutated its inputs") + } + }) + } +} diff --git a/amd64_packed_shift_fault_test.go b/amd64_packed_shift_fault_test.go new file mode 100644 index 000000000..bca887ecb --- /dev/null +++ b/amd64_packed_shift_fault_test.go @@ -0,0 +1,100 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedShiftNativeReadFaults(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires Linux/amd64 memory-fault counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + for i, row := range []string{ + "VPSRLW.Z (CX),Y0,K1,Y1", "VPSRLD.Z (CX),Y0,K1,Y1", "VPSRLQ.Z (CX),Y0,K1,Y1", + "VPSRLW.Z $1,(CX),K1,Y1", "VPSRLDQ $1,(CX),Y1", + "VPSRLD.Z $1,(CX),K1,Y1", "VPSRLQ.BCST.Z $1,(CX),K1,Y1", + "VPSRLVW.Z (CX),Y0,K1,Y1", "VPSRLVD.BCST.Z (CX),Y0,K1,Y1", + "VPSRLD $255,(CX),Y1", "VPSRLVD (CX),Y0,Y1", "VPSRLVD.Z (CX),Y0,K1,Y1", + } { + name := fmt.Sprintf("faultshift%d", i) + mask := 0 + if i == 11 { + mask = 1 + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),AX\nMOVQ data+8(FP),BX\nMOVQ count+16(FP),CX\nVMOVUPS (BX),Y0\nVPXOR Y0,Y0,Y0\nMOVQ $%d,DX\nKMOVQ DX,K1\n%s\nVMOVUPS Y1,(AX)\nRET\n", name, mask, row) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*);\n", name) + accessible := 8 + if i == 11 { + accessible = 3 + } + fmt.Fprintf(&checks, "if(check_fault(%s,%d,%d,page)) return %d;\n", name, x86ShiftBoolean(i < 5 || i >= 9), accessible, i+10) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const helper = ` +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*fault_fn)(uint8_t*,const uint8_t*,const uint8_t*); +static int check_fault(fault_fn fn,int should_fault,int accessible,size_t page) { + uint8_t *area=mmap(0,page*2,PROT_READ|PROT_WRITE,MAP_ANON|MAP_PRIVATE,-1,0); + if(area==MAP_FAILED || mprotect(area+page,page,PROT_NONE)) return 1; + uint8_t data[32],out[34]; memset(data,0x71,32); memset(out,0xa7,34); + memset(area+page-8,0,8); + pid_t child=fork(); if(child<0) return 2; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + + const uint8_t *address=should_fault?area+page-accessible:area+page; + fn(out+1,data,address); + if(out[0]!=0xa7 || out[33]!=0xa7) _exit(3); + for(int i=1;i<=32;++i) if(out[i]) _exit(4); + _exit(0); + } + int status=0; if(waitpid(child,&status,0)!=child) return 5; + if(munmap(area,page*2)) return 6; + if(should_fault) { + int memory_signal=WIFSIGNALED(status) && WTERMSIG(status)==SIGSEGV; +#if defined(__APPLE__) + /* Rosetta's guarded-page diagnostic can report the host's SIGBUS. */ + memory_signal=memory_signal || (WIFSIGNALED(status) && WTERMSIG(status)==SIGBUS); +#endif + if(!memory_signal) { fprintf(stderr,"expected full-read memory fault, status=%x\n",status); return 7; } + } else if(!WIFEXITED(status) || WEXITSTATUS(status)!=0) { fprintf(stderr,"expected masked fault suppression, status=%x\n",status); return 8; } + return 0; +} +` + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + driver := helper + declarations.String() + "int main(void) { size_t page=(size_t)sysconf(_SC_PAGESIZE);\n" + checks.String() + "return 0; }\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_shift_memory_faults", triple, ir, driver, prefix) + t.Logf("8 full/active native reads and 4 lane-mask suppression cases; Rosetta=%t", crossRosetta) +} diff --git a/amd64_packed_shift_forms.go b/amd64_packed_shift_forms.go new file mode 100644 index 000000000..770aca3e2 --- /dev/null +++ b/amd64_packed_shift_forms.go @@ -0,0 +1,194 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedShiftCount uint8 + +const ( + x86PackedShiftUniform x86PackedShiftCount = iota + x86PackedShiftPerLane + x86PackedShiftBytes +) + +type x86PackedShiftForm struct { + countMode x86PackedShiftCount + laneBits, byteWidth int + count, data, destination, mask, memory int + memoryBytes int + left, legacy, mmx, broadcast, zeroing bool + faultSuppressed bool + countOperand Operand +} + +// Properties establish encoding and access shape only, never initialized +// data, pointer bounds, alignment or permission to return through raw bytes. +func x86PackedShiftProperties(op Op, ins Instr, goarch string) (x86PackedShiftForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + uniform, recognized := amd64PackedLogicalShiftSpecs[Op(base)] + perLane, variable := amd64PerLaneVariableShiftSpecs[base] + variable = variable && !perLane.arithmetic + byteShift := base == "PSLLO" || base == "PSLLDQ" || base == "PSRLO" || base == "PSRLDQ" || base == "VPSLLDQ" || base == "VPSRLDQ" + if !recognized && !variable && !byteShift { + return x86PackedShiftForm{}, false, nil + } + f := x86PackedShiftForm{count: 0, data: 1, destination: len(ins.Args) - 1, mask: -1, memory: -1} + fail := func() (x86PackedShiftForm, bool, error) { + return f, true, fmt.Errorf("%s %s outside Go packed logical-shift grammar: %q", goarch, op, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return fail() + } + local := ins + local.Op = op + if recognized && !uniform.vector { + var err error + local, err = x86LegacyMMXReadInstruction(local, goarch) + if err != nil { + return f, true, err + } + } + f.countMode, f.laneBits, f.left, f.legacy = x86PackedShiftUniform, uniform.laneBits, uniform.left, !uniform.vector + if variable { + f.countMode, f.laneBits, f.left, f.legacy = x86PackedShiftPerLane, perLane.laneBits, perLane.operation == "shl", false + } + if byteShift { + f.countMode, f.laneBits, f.left, f.legacy = x86PackedShiftBytes, 8, strings.Contains(base, "SLL"), !strings.HasPrefix(base, "V") + } + if f.legacy { + if dotted || len(ins.Args) != 2 || ins.Args[1].Kind != OpReg { + return fail() + } + f.data = f.destination + f.byteWidth = 16 + _, f.mmx = amd64ParseMReg(ins.Args[f.destination].Reg) + if f.mmx { + if byteShift || goarch == "386" { + return fail() + } + f.byteWidth = 8 + } + } else { + switch suffix { + case "": + if dotted { + return fail() + } + case "Z": + f.zeroing = true + case "BCST": + f.broadcast = true + case "BCST.Z": + f.broadcast, f.zeroing = true, true + default: + return fail() + } + if len(ins.Args) != 3 && len(ins.Args) != 4 || byteShift && dotted || f.broadcast && f.laneBits == 16 { + return fail() + } + if len(ins.Args) == 4 { + if byteShift || goarch == "386" && !ins.x86Encoded || !amd64NonzeroKOperand(ins.Args[2]) { + return fail() + } + f.mask = 2 + } else if f.zeroing { + return fail() + } + f.byteWidth = amd64VectorByteWidth(ins.Args[f.destination].Reg) + } + register := func(operand Operand, width int) bool { + if f.mmx { + _, valid := amd64ParseMReg(operand.Reg) + return operand.Kind == OpReg && valid + } + if f.legacy { + index, valid := amd64ParseXReg(operand.Reg) + return operand.Kind == OpReg && valid && (goarch == "amd64" && index < 16 || goarch == "386" && index < 8) + } + return amd64GoPackedVectorMoveRegister(operand, width, goarch) + } + if !register(ins.Args[f.destination], f.byteWidth) { + return fail() + } + count, data := local.Args[f.count], local.Args[f.data] + if count.Kind == OpImm { + var valid bool + count, valid = x86ImmediateShuffleConstant(count, goarch, ins.x86Encoded) + if !valid { + return fail() + } + if variable || count.Imm < -128 || count.Imm > 255 || f.legacy && count.Imm > 127 { + return fail() + } + if !f.legacy && count.Imm < 0 && amd64PackedByteShiftRequiresEVEX(data, ins.Args[f.destination], f.byteWidth) || count.Imm < 0 && f.mask >= 0 { + return fail() + } + if isAMD64MemoryOperand(data) { + f.memory, f.memoryBytes = f.data, f.byteWidth + if f.broadcast { + f.memoryBytes = f.laneBits / 8 + } + } else if f.broadcast || !register(data, f.byteWidth) { + return fail() + } + // EVEX W uniform shifts and byte shifts do not suppress faults. + // D/Q immediate rows use E4, independently of the count rows. + f.faultSuppressed = !f.legacy && !byteShift && f.laneBits != 16 + } else { + if byteShift || f.broadcast && !variable || !register(data, f.byteWidth) { + return fail() + } + countWidth := f.byteWidth + if !variable && !f.mmx { + countWidth = 16 + } + if isAMD64MemoryOperand(count) { + f.memory, f.memoryBytes = f.count, countWidth + if f.broadcast { + f.memoryBytes = f.laneBits / 8 + } + } else if f.broadcast || !register(count, countWidth) { + return fail() + } + f.faultSuppressed = variable + } + f.countOperand = count + if f.memory >= 0 && !x86PackedShiftMemoryValid(ins.Args[f.memory], goarch) { + return fail() + } + if f.memory >= 0 && ins.Args[f.memory].Kind == OpMem { + memory := ins.Args[f.memory].Mem + if err := x86SymbolIndexedMemoryError(memory, goarch); err != nil { + return f, true, err + } + if memory.OffRaw != "" || memory.IndexExt != "" { + return fail() + } + } + if goarch == "386" { + for _, operand := range ins.Args { + width := amd64VectorByteWidth(operand.Reg) + if index, valid := amd64VectorRegisterIndex(operand.Reg, width); operand.Kind == OpReg && valid && index >= 8 { + return f, true, fmt.Errorf("mode32 packed shift has no physical SIMD register %s: %w", operand.Reg, ErrProbeNeedsContext) + } + } + } + return f, true, nil +} + +func x86PackedShiftMemoryValid(operand Operand, goarch string) bool { + if operand.Kind == OpSym { + symbol := strings.TrimSpace(operand.Sym) + _, integer := parseInt(symbol) + return !strings.HasPrefix(symbol, "$") && (strings.HasSuffix(symbol, "(SB)") || integer == nil) + } + if operand.Kind != OpMem { + return operand.Kind == OpFP + } + memory := operand.Mem + return (memory.Off == int64(int32(memory.Off)) || goarch == "386" && memory.Off == int64(uint32(memory.Off))) && + x86MemoryRegistersValidForArch(memory, goarch) && (memory.Index == "" || memory.Index != SP && + (memory.Scale == 1 || memory.Scale == 2 || memory.Scale == 4 || memory.Scale == 8)) +} diff --git a/amd64_packed_shift_forms_test.go b/amd64_packed_shift_forms_test.go new file mode 100644 index 000000000..0112f98bd --- /dev/null +++ b/amd64_packed_shift_forms_test.go @@ -0,0 +1,365 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "sort" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86PackedShiftNames() []string { + var names []string + for op := range amd64PackedLogicalShiftSpecs { + names = append(names, string(op)) + } + for op, spec := range amd64PerLaneVariableShiftSpecs { + if !spec.arithmetic { + names = append(names, op) + } + } + names = append(names, "PSLLO", "PSLLDQ", "PSRLO", "PSRLDQ", "VPSLLDQ", "VPSRLDQ") + sort.Strings(names) + return names +} + +func x86PackedShiftRows(op, arch string) []string { + legacy := !strings.HasPrefix(op, "V") + bytes := strings.HasSuffix(op, "DQ") || op == "PSLLO" || op == "PSRLO" + perLane := strings.Contains(op, "LLV") || strings.Contains(op, "RLV") + if legacy { + rows := []string{op + " $-128,X0", op + " $127,X1"} + if !bytes { + rows = append(rows, op+" X0,X1", op+" (AX),X1") + if arch == "amd64" { + rows = append(rows, op+" $1,M0", op+" M0,M1", op+" (AX),M1") + } + } + return rows + } + var rows []string + for _, bank := range []string{"X", "Y", "Z"} { + if !perLane { + rows = append(rows, fmt.Sprintf("%s $0,%s0,%s2", op, bank, bank), fmt.Sprintf("%s $255,(AX),%s2", op, bank)) + if bank != "Z" { + rows = append(rows, fmt.Sprintf("%s $-128,%s0,%s2", op, bank, bank)) + } + } + if bytes { + continue + } + countBank := "X" + if perLane { + countBank = bank + } + rows = append(rows, fmt.Sprintf("%s %s0,%s1,%s2", op, countBank, bank, bank), fmt.Sprintf("%s (AX),%s1,%s2", op, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s0,%s1,K1,%s2", op, countBank, bank, bank), fmt.Sprintf("%s.Z (AX),%s1,K7,%s2", op, bank, bank)) + if !perLane { + rows = append(rows, fmt.Sprintf("%s.Z $3,(AX),K1,%s2", op, bank)) + } + } + if !strings.HasSuffix(op, "W") { + input := "$3,(AX)," + if perLane { + input = "(AX)," + bank + "1," + } + rows = append(rows, fmt.Sprintf("%s.BCST %s%s2", op, input, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s.BCST.Z %sK1,%s2", op, input, bank)) + } + } + } + return rows +} + +func TestX86PackedShiftEveryGoOpcodeAndAttribute(t *testing.T) { + if len(x86PackedShiftNames()) != 24 { + t.Fatal("incomplete logical-shift family") + } + dir := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + re := regexp.MustCompile(`\{as: A(VPS(?:LL|RL)(?:[WDQ]|V[WDQ]|DQ)), ytab: ([^,]+),`) + rows := re.FindAllStringSubmatch(string(avx), -1) + if len(rows) != 14 { + t.Fatalf("Go AVX family inventory changed: %d", len(rows)) + } + for _, row := range rows { + op, table := row[1], "_yvpslld" + if strings.HasSuffix(op, "DQ") { + table = "_yvpslldq" + } else if strings.Contains(op, "LLV") || strings.Contains(op, "RLV") { + table = "_yvandnpd" + if strings.HasSuffix(op, "W") { + table = "_yvblendmpd" + } + } + if row[2] != table { + t.Fatalf("%s Go table %s != %s", op, row[2], table) + } + start := strings.Index(string(avx), row[0]) + block := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + for _, line := range strings.Split(block, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + byteShift := strings.HasSuffix(op, "DQ") + uniformCount := !strings.Contains(op, "LLV") && !strings.Contains(op, "RLV") && !strings.Contains(line, "0x7") + broadcast := !byteShift && !strings.HasSuffix(op, "W") && !uniformCount + if strings.Contains(line, "evexZeroingEnabled") == byteShift || strings.Contains(line, "evexBcstN") != broadcast || + strings.Contains(line, "evexRounding") || strings.Contains(line, "evexSae") { + t.Fatalf("%s attribute disagreement: %s", op, line) + } + } + } + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + if !strings.Contains(string(legacy), "{A"+op+", yps,") { + t.Fatalf("Go yps member changed: %s", op) + } + } + // Bind the complete operand-class inventory, not just the table names. + // These four AVX tables are shared with unrelated instructions; checking + // their actual rows detects a changed immediate, mask or memory class. + tables := map[string][]string{ + "_yvpslld": { + "Yu8,Yxr,Yxr", "Yi8,Yxr,Yxr", "Yu8,Yyr,Yyr", "Yi8,Yyr,Yyr", + "Yxm,Yxr,Yxr", "Yxm,Yyr,Yyr", + "Yu8,YxmEvex,YxrEvex", "Yu8,YxmEvex,Yknot0,YxrEvex", + "Yu8,YymEvex,YyrEvex", "Yu8,YymEvex,Yknot0,YyrEvex", + "Yu8,Yzm,Yzr", "Yu8,Yzm,Yknot0,Yzr", + "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YxmEvex,YyrEvex,YyrEvex", "YxmEvex,YyrEvex,Yknot0,YyrEvex", + "YxmEvex,Yzr,Yzr", "YxmEvex,Yzr,Yknot0,Yzr", + }, + "_yvpslldq": { + "Yu8,Yxr,Yxr", "Yi8,Yxr,Yxr", "Yu8,Yyr,Yyr", "Yi8,Yyr,Yyr", + "Yu8,YxmEvex,YxrEvex", "Yu8,YymEvex,YyrEvex", "Yu8,Yzm,Yzr", + }, + "_yvandnpd": { + "Yxm,Yxr,Yxr", "Yym,Yyr,Yyr", + "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", + "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr", + }, + "_yvblendmpd": { + "YxmEvex,YxrEvex,YxrEvex", "YxmEvex,YxrEvex,Yknot0,YxrEvex", + "YymEvex,YyrEvex,YyrEvex", "YymEvex,YyrEvex,Yknot0,YyrEvex", + "Yzm,Yzr,Yzr", "Yzm,Yzr,Yknot0,Yzr", + }, + } + assertRows := func(source, name string, want []string) { + t.Helper() + block := regexp.MustCompile(`(?s)var ` + regexp.QuoteMeta(name) + ` = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(source) + if len(block) != 2 { + t.Fatalf("missing Go table %s", name) + } + var got []string + for _, row := range regexp.MustCompile(`argList\{([^}]+)\}`).FindAllStringSubmatch(block[1], -1) { + got = append(got, strings.ReplaceAll(row[1], " ", "")) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("%s operand classes changed:\ngot %q\nwant %q", name, got, want) + } + } + for name, want := range tables { + assertRows(string(avx), name, want) + } + assertRows(string(legacy), "yps", []string{"Ymm,Ymr", "Yi8,Ymr", "Yxm,Yxr", "Yi8,Yxr"}) + assertRows(string(legacy), "ypsdq", []string{"Yi8,Yxr"}) + aliases, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/asm/internal/arch/arch.go")) + if err != nil { + t.Fatal(err) + } + for alias, canonical := range map[string]string{"PSLLDQ": "APSLLO", "PSRLDQ": "APSRLO"} { + if !regexp.MustCompile(`instructions\["` + alias + `"\]\s*=\s*x86\.` + canonical).Match(aliases) { + t.Fatalf("Go byte-shift alias changed: %s", alias) + } + } +} + +func TestX86PackedShiftCompleteGoRowsThreeAPIsFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc missing") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, op := range x86PackedShiftNames() { + for _, row := range x86PackedShiftRows(op, target.arch) { + fmt.Fprintln(&source, row) + } + } + source.WriteString("RET\n") + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "shifts.ll", "shifts.o", ir) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "module.ll", "module.o", module.String()) + pkg := x86PackedShiftGoPackage(t, target.arch) + translated, err := translateGoModuleInContext(ctx, pkg, []byte(strings.Replace(source.String(), "shifts(SB)", "·shifts(SB)", 1)), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if translated != nil { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "go-module.ll", "go-module.o", translated.Module.String()) + }) + } +} + +func TestX86PackedShiftImmediateAnd386RejectedGoRows(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{ + "PSLLW $128,X0", "PSRLO $128,X0", "VPSRLW $-1,X20,X21", + "VPSRLW $-1,(AX),X1", "VPSRLQ.Z $-1,X0,K1,X1", "VPSLLDQ $-1,Z0,Z1", + "VPSRLD $256,X0,X1", "VPSRLVW $1,X0,X1", "VPSRLW.BCST $1,(AX),X1", + } { + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + assertX86PackedLogicalShiftRejected(t, arch, testTargetTriple("linux", arch), row) + } + } + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + for _, row := range []string{op + " $1,M0", op + " M0,M1", op + " (AX),M1"} { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + assertX86PackedLogicalShiftRejected(t, "386", "i386-unknown-linux-gnu", row) + } + } +} + +func TestX86PackedShiftIdentityAndInvalidFormsHaveNoIRSideEffects(t *testing.T) { + for _, op := range x86PackedShiftNames() { + row := x86PackedShiftRows(op, "amd64")[0] + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", Op(strings.ToLower(op)), original.Op} { + ins := original + ins.Op = identity + ctx, _ := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + if recognized, _, err := ctx.lowerX86PackedShift(Op(strings.ToLower(op)), ins); !recognized || err != nil { + t.Fatalf("matching identity %s/%s: %v", op, identity, err) + } + } + for _, dispatch := range []Op{"UNKNOWN_SHIFT", "PSRAD", "PSLLL"} { + if dispatch == original.Op { + continue + } + ins := original + before := append([]Operand(nil), ins.Args...) + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + length := output.Len() + recognized, _, err := ctx.lowerX86PackedShift(dispatch, ins) + if recognized && err == nil || output.Len() != length || !reflect.DeepEqual(before, ins.Args) || ins.Op != original.Op { + t.Fatalf("conflicting/unknown %s/%s mutated or emitted IR", op, dispatch) + } + } + } + for _, row := range []string{ + "VPSRLW $8,X0,Y1", "VPSLLDQ.Z $1,X0,K1,X1", "VPSLLW $1,X0,K0,X1", + "VPSLLD.BCST X0,X1,X2", "VPSRLD.RN_SAE $1,X0,X1", "VPSRLW $1,8(BAD),X1", + "VPSRLW $1,8(AX)(SP*2),X1", "VPSRLW $1,2147483648(AX),X1", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + length := output.Len() + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); err == nil || output.Len() != length { + t.Fatalf("invalid row %s emitted %d bytes: %v", row, output.Len()-length, err) + } + } +} + +func TestX86PackedShiftFullNativeMemoryReadAndSuppression(t *testing.T) { + for _, test := range []struct { + row string + bytes int + suppressed bool + }{ + {"VPSRLW (AX),Y0,K1,Y1", 16, false}, + {"VPSRLD (AX),Z0,K1,Z1", 16, false}, + {"VPSRLQ (AX),Z0,K1,Z1", 16, false}, + {"VPSRLW.Z $1,(AX),K1,Z1", 64, false}, + {"VPSRLD.Z $1,(AX),K1,Z1", 64, true}, + {"VPSRLQ.BCST.Z $1,(AX),K1,Z1", 8, true}, + {"VPSRLVW.Z (AX),Z0,K1,Z1", 64, true}, + {"VPSRLVD.BCST.Z (AX),Z0,K1,Z1", 4, true}, + {"VPSRLDQ $1,(AX),Z1", 64, false}, + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+test.row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, recognized, err := x86PackedShiftProperties(ins.Op, ins, "amd64") + if !recognized || err != nil || form.memoryBytes != test.bytes || form.faultSuppressed != test.suppressed { + t.Fatalf("native operand mismatch %s: %+v/%v", test.row, form, err) + } + if !test.suppressed { + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), fmt.Sprintf("load volatile <%d x i8>", test.bytes)) { + t.Fatalf("lost full native read %s:\n%s", test.row, output.String()) + } + } + } +} + +func x86PackedShiftGoPackage(t *testing.T, arch string) GoPackage { + t.Helper() + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "shifts.go", "package shifts\nfunc shifts()\n", 0) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("shifts", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + return GoPackage{Path: "shifts", Types: typed, Syntax: []*ast.File{file}} +} diff --git a/amd64_packed_shift_fp.go b/amd64_packed_shift_fp.go new file mode 100644 index 000000000..167f8ae01 --- /dev/null +++ b/amd64_packed_shift_fp.go @@ -0,0 +1,25 @@ +package plan9asm + +import ( + "fmt" + "math" +) + +// A logical FP span is usable only when every byte belongs to a validated +// scalar ABI0 slot. Padding and unknown native caller-frame bytes are not +// synthesized. This is a value/span proof, not a physical alignment proof. +func (c *amd64Ctx) validateX86PackedShiftFPSpan(off int64, width int) error { + if width <= 0 || off < 0 || off > math.MaxInt64-int64(width) { + return fmt.Errorf("packed shift FP span is not bounded: %w", ErrProbeNeedsContext) + } + if err := c.validateFPVectorSpan(off, width); err != nil { + return fmt.Errorf("packed shift needs complete FP bytes: %v: %w", err, ErrProbeNeedsContext) + } + for byteOffset := off; byteOffset < off+int64(width); byteOffset++ { + slot, _, _ := c.fpVectorSlot(byteOffset) + if x86FrameScalarBytes(slot.Type, c.goarch) == 0 { + return fmt.Errorf("packed shift FP type %s lacks scalar byte storage: %w", slot.Type, ErrProbeNeedsContext) + } + } + return nil +} diff --git a/amd64_packed_shift_fp_context_test.go b/amd64_packed_shift_fp_context_test.go new file mode 100644 index 000000000..36625ba93 --- /dev/null +++ b/amd64_packed_shift_fp_context_test.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedShiftCompleteFPSpanThreeAPIsFiveObjects(t *testing.T) { + // Even a fully declared logical ABI0 span comprises independent LLVM + // allocas. Read every bound byte without using a pointer to just the first + // alloca, and retain mask-controlled reads. BCST at +4 crosses two slots. + rows := []struct { + instruction string + masked bool + }{ + {"VPSRLW a+0(FP),Y1,Y2", false}, + {"VPSRLQ a+0(FP),Z1,Z2", false}, + {"VPSRLVW a+0(FP),Z1,Z2", false}, + {"VPSRLVD.BCST a+4(FP),Z1,Z2", false}, + {"VPSRLVQ.BCST a+4(FP),Z1,Z2", false}, + {"VPSRLD $1,a+0(FP),Z2", false}, + {"VPSRLQ.BCST $1,a+4(FP),Z2", false}, + {"VPSRLDQ $1,a+0(FP),Z2", false}, + {"VPSRLVW.Z a+0(FP),Z1,K1,Z2", true}, + {"VPSRLVD.BCST.Z a+4(FP),Z1,K1,Z2", true}, + {"VPSRLVQ.BCST.Z a+4(FP),Z1,K1,Z2", true}, + {"VPSRLQ.BCST.Z $1,a+4(FP),K1,Z2", true}, + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "shifts.go", "package shifts\nfunc shifts(a,b,c,d,e,f,g,h uint64)\n", 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", target.arch) + typed, err := (&types.Config{Sizes: sizes}).Check("shifts", fset, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + pkg := GoPackage{Path: "shifts", Types: typed, Syntax: []*ast.File{file}} + sig, err := DeriveGoFuncABI0Signature("shifts", typed.Scope().Lookup("shifts").(*types.Func), target.arch, sizes) + if err != nil { + t.Fatal(err) + } + for _, row := range rows { + if row.masked && target.arch == "386" { + continue // Go's 386 source frontend rejects this explicit mask row. + } + source := "TEXT shifts(SB),4,$0-64\n" + row.instruction + "\nRET\n" + goAssemblerOracleOutput(t, target.arch, source, "", true) + parsed, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"shifts": sig}} + ir, err := Translate(parsed, options) + if err != nil { + t.Fatalf("%s original FP read %s: Translate=%v", target.triple, row.instruction, err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "fp-shift.ll", "fp-shift.o", ir) + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, parsed, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatalf("%s original FP read %s: Module=%v", target.triple, row.instruction, err) + } + compileLLVMToObject(t, llc, target.triple, "fp-module.ll", "fp-module.o", module.String()) + result, err := translateGoModuleInContext(context, pkg, []byte(strings.Replace(source, "shifts(SB)", "·shifts(SB)", 1)), + GoModuleOptions{GOARCH: target.arch, Sizes: sizes, TargetTriple: target.triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if err != nil { + t.Fatalf("%s original FP read %s: GoModule=%v", target.triple, row.instruction, err) + } + compileLLVMToObject(t, llc, target.triple, "fp-go-module.ll", "fp-go-module.o", result.Module.String()) + }() + instruction := parsed.Funcs[0].Instrs[1] + ctx, output := newAMD64CtxWithFuncForTest(t, parsed.Funcs[0], sig, nil) + if _, _, err := ctx.lowerX86PackedShift(instruction.Op, instruction); err != nil { + t.Fatalf("%s complete FP span: %v", row.instruction, err) + } + if row.masked && !strings.Contains(output.String(), "br i1") { + t.Fatalf("%s lost conditional FP reads", row.instruction) + } + } + } +} + +func TestX86PackedShiftIncompleteFPSpansFailBeforeIR(t *testing.T) { + for _, instruction := range []string{ + "VPSRLW a+8(FP),Y1,Y2", "VPSRLQ.BCST $1,a+12(FP),Z2", + "VPSRLVD.BCST.Z a+14(FP),Z1,K1,Z2", "VPSRLDQ $1,a+0(FP),Z2", + "VPSRLVW.Z a+0(FP),Z1,K1,Z2", "VPSRLQ.BCST $1,a-1(FP),Z2", + } { + source := "TEXT shifts(SB),4,$0-16\n" + instruction + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, frame := range []FrameLayout{ + {Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}}, + {Params: []FrameSlot{{Offset: 0, Type: I8, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}}, + } { + sig := FuncSig{Ret: Void, Args: []LLVMType{frame.Params[0].Type, frame.Params[1].Type}, Frame: frame} + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.Len() + ins := file.Funcs[0].Instrs[1] + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || before != output.Len() { + t.Fatalf("%s incomplete/padded FP span emitted IR: %v", instruction, err) + } + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, {"386", "i686-pc-windows-msvc"}, + } { + if target.arch == "386" && strings.Contains(instruction, "K1") { + continue + } + fset := token.NewFileSet() + declaration, err := parser.ParseFile(fset, "shifts.go", "package shifts\nfunc shifts(a,b uint64)\n", 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", target.arch) + typed, err := (&types.Config{Sizes: sizes}).Check("shifts", fset, []*ast.File{declaration}, nil) + if err != nil { + t.Fatal(err) + } + sig, err := DeriveGoFuncABI0Signature("shifts", typed.Scope().Lookup("shifts").(*types.Func), target.arch, sizes) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"shifts": sig}} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s original unbound FP span %s: Translate=%v", target.triple, instruction, err) + } + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s original unbound FP span %s: Module=%v", target.triple, instruction, err) + } + pkg := GoPackage{Path: "shifts", Types: typed, Syntax: []*ast.File{declaration}} + result, err := translateGoModuleInContext(context, pkg, []byte(strings.Replace(source, "shifts(SB)", "·shifts(SB)", 1)), + GoModuleOptions{GOARCH: target.arch, Sizes: sizes, TargetTriple: target.triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s original unbound FP span %s: GoModule=%v", target.triple, instruction, err) + } + }() + } + } +} diff --git a/amd64_packed_shift_fp_runtime_test.go b/amd64_packed_shift_fp_runtime_test.go new file mode 100644 index 000000000..b38497dbd --- /dev/null +++ b/amd64_packed_shift_fp_runtime_test.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "os/exec" + "runtime" + "strings" + "testing" +) + +func TestX86PackedShiftFPSpanNumerical(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + var prefix []string + flags := []string{"-no-pie"} + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + if !rosettaAvailable() { + t.Fatal("Darwin x86 diagnostic transport required") + } + prefix = []string{"/usr/bin/arch", "-x86_64"} + flags = []string{"-arch", "x86_64"} + } else if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + qemu, err := exec.LookPath("qemu-x86_64") + if err != nil { + t.Fatal("explicit Linux x86 runtime transport required:", err) + } + clang, err = exec.LookPath("x86_64-linux-gnu-gcc") + if err != nil { + t.Fatal("x86 cross C compiler required:", err) + } + prefix = []string{qemu, "-L", "/usr/x86_64-linux-gnu"} + } else if runtime.GOARCH != "amd64" { + t.Skip("required Linux/amd64 counterpart supplies this numerical axis") + } + const source = `TEXT fpbytes(SB),4,$0-16 +VPSRLD.BCST $0,a+0(FP),X0 +MOVQ X0,AX +MOVQ AX,r+8(FP) +RET +TEXT fpcross(SB),4,$0-24 +VPSRLQ.BCST $0,a+4(FP),X0 +MOVQ X0,AX +MOVQ AX,r+16(FP) +RET +TEXT fpcount(SB),4,$0-24 +MOVQ $0x89abcdef76543210,AX +MOVQ AX,X0 +VPSRLD a+0(FP),X0,X1 +MOVQ X1,AX +MOVQ AX,r+16(FP) +RET +` + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + bytesSig := FuncSig{Name: "fpbytes", Args: []LLVMType{I8, I8, I8, I8}, Ret: I64, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I8, Index: 0, Field: -1}, {Offset: 1, Type: I8, Index: 1, Field: -1}, + {Offset: 2, Type: I8, Index: 2, Field: -1}, {Offset: 3, Type: I8, Index: 3, Field: -1}, + }, Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}}} + wideSig := FuncSig{Args: []LLVMType{I64, I64}, Ret: I64, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: I64, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}, + }, Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0, Field: -1}}}} + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"fpbytes": bytesSig, "fpcross": wideSig, "fpcount": wideSig}}) + if err != nil { + t.Fatal(err) + } + const driver = ` +#include +#include +extern uint64_t fpbytes(uint8_t,uint8_t,uint8_t,uint8_t); +extern uint64_t fpcross(uint64_t,uint64_t); +extern uint64_t fpcount(uint64_t,uint64_t); +int main(void) { + for(unsigned i=0;i<96;++i) { + uint8_t a=(uint8_t)(i*71+13),b=(uint8_t)(i*97+11); + uint8_t c=(uint8_t)(i*53+17),d=(uint8_t)(i*19+7); + uint64_t word=(uint32_t)a|((uint32_t)b<<8)|((uint32_t)c<<16)|((uint32_t)d<<24); + if(fpbytes(a,b,c,d)!=(word|(word<<32))) { fprintf(stderr,"four byte FP BCST mismatch %u\n",i); return 1; } + uint64_t first=UINT64_C(0xabcdef1276543210)^(UINT64_C(0x12345678)*i); + uint64_t second=UINT64_C(0x179bdeff23056789)^(UINT64_C(0x87654321)*i); + if(fpcross(first,second)!=((first>>32)|(second<<32))) { fprintf(stderr,"cross slot FP BCST mismatch %u\n",i); return 2; } + uint64_t count=i<40?i:UINT64_C(0x100000000)+i; + uint64_t expected=count<32?((uint64_t)(UINT32_C(0x89abcdef)>>count)<<32)|(UINT32_C(0x76543210)>>count):0; + if(fpcount(count,second)!=expected) { fprintf(stderr,"full m128 FP count mismatch %u\n",i); return 3; } + } + return 0; +} +` + compileAndRunRuntimeTestWithCompiler(t, llc, append([]string{clang}, flags...), "packed_shift_fp_span", triple, ir, driver, prefix) + if strings.Contains(ir, "inttoptr i64 0") { + t.Fatal("FP bytes were replaced by an unbound zero pointer") + } + t.Logf("three initialized FP span programs ×96 numerical inputs; original Go encoding accepted, LLVM execution transport=%v", prefix) +} diff --git a/amd64_packed_shift_mode32_test.go b/amd64_packed_shift_mode32_test.go new file mode 100644 index 000000000..076e5343f --- /dev/null +++ b/amd64_packed_shift_mode32_test.go @@ -0,0 +1,97 @@ +package plan9asm + +import ( + "encoding/hex" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86PackedShiftActualGoBytes(t *testing.T, arch, source string) []byte { + t.Helper() + dir := t.TempDir() + input := filepath.Join(dir, "original.s") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-o", filepath.Join(dir, "original.o"), input) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch, "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil { + t.Fatalf("actual Go %s original source: %v\n%s", arch, err, output) + } + var code []byte + for _, line := range regexp.MustCompile(`(?m)^\s*0x[[:xdigit:]]+\s+((?:[[:xdigit:]]{2} )+)`).FindAllSubmatch(output, -1) { + part, err := hex.DecodeString(strings.ReplaceAll(string(line[1]), " ", "")) + if err != nil { + t.Fatal(err) + } + code = append(code, part...) + } + return code +} + +func TestX86PackedShiftMode32NamedPhysicalBanks(t *testing.T) { + for _, row := range []string{ + "VPSRLW $1,X20,X21", "VPSRLD X0,Y20,Y21", "VPSRLQ (AX),X20,X21", + "VPSRLVW X1,X20,X21", "VPSRLVD.BCST (AX),Y20,Y21", "VPSRLVQ X0,X20,X21", + "VPSLLDQ $1,X16,X31", "VPSRLDQ $1,Y20,Y21", + } { + source := "TEXT shifts(SB),4,$0-0\n" + row + "\nRET\n" + code := x86PackedShiftActualGoBytes(t, "386", source) + _, _, recognized, err := decodedX86PackedUniformShiftInstruction(code, 32) + if !recognized { + _, _, recognized, err = decodedX86PerLaneVariableShiftInstruction(code, 32) + } + if !recognized { + _, _, recognized, err = decodedX86RawVectorByteShiftInstruction(code, 32) + } + if !recognized || err == nil { + t.Fatalf("mode32 raw bytes %x for %s did not reject unavailable physical bank: %v", code, row, err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + options := Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"shifts": {Ret: Void}}} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s named unavailable SIMD bank %s: %v", triple, row, err) + } + context := llvm.NewContext() + func() { + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s module unavailable SIMD bank %s: %v", triple, row, err) + } + result, err := translateGoModuleInContext(context, x86PackedShiftGoPackage(t, "386"), + []byte(strings.Replace(source, "shifts(SB)", "·shifts(SB)", 1)), GoModuleOptions{GOARCH: "386", TargetTriple: triple}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s Go module unavailable SIMD bank %s: %v", triple, row, err) + } + }() + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := output.Len() + ins := file.Funcs[0].Instrs[1] + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || output.Len() != before { + t.Fatalf("unavailable mode32 SIMD bank emitted IR: %s %v", row, err) + } + } + } +} diff --git a/amd64_packed_shift_native_count.go b/amd64_packed_shift_native_count.go new file mode 100644 index 000000000..aaa5eb6a6 --- /dev/null +++ b/amd64_packed_shift_native_count.go @@ -0,0 +1,48 @@ +package plan9asm + +import "fmt" + +func x86PackedShiftHasNativeCount(form x86PackedShiftForm) bool { + return form.legacy && !form.mmx && form.memory == form.count +} + +// Preserve the original SSE2 m128 operation, including its complete native +// read and fault behavior. Substituting MOVDQA would impose its alignment +// contract on a different instruction. The tied data input, vector output +// and memory clobber keep the original operation observable without granting +// the source address any permission or claiming an alignment guarantee. +func (c *amd64Ctx) emitX86PackedShiftNativeCount(form x86PackedShiftForm, source Operand, data string) (string, error) { + if !x86PackedShiftHasNativeCount(form) || form.byteWidth != 16 || form.countMode != x86PackedShiftUniform { + return "", fmt.Errorf("expected legacy SSE m128 packed count form") + } + name := "psrl" + if form.left { + name = "psll" + } + switch form.laneBits { + case 16: + name += "w" + case 32: + name += "d" + case 64: + name += "q" + default: + return "", fmt.Errorf("unsupported legacy packed count lane width %d", form.laneBits) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(source) + if err != nil { + return "", err + } + segment := "" + switch pointerType { + case "ptr addrspace(256)": + segment = "%gs:" + case "ptr addrspace(257)": + segment = "%fs:" + } + value := c.newTmp() + typeName := fmt.Sprintf("<%d x i%d>", 128/form.laneBits, form.laneBits) + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s %s, %s %s)\n", + value, typeName, name+" "+segment+"($2), $0", "=x,0,r,~{memory}", typeName, data, pointerType, pointer) + return "%" + value, nil +} diff --git a/amd64_packed_shift_native_count_linux_test.go b/amd64_packed_shift_native_count_linux_test.go new file mode 100644 index 000000000..6eef222c6 --- /dev/null +++ b/amd64_packed_shift_native_count_linux_test.go @@ -0,0 +1,46 @@ +//go:build linux && (amd64 || arm64) + +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func x86PackedShiftBuildOriginalCountObserver(t *testing.T, compiler []string) string { + t.Helper() + const main = ` +#include +int main(int argc,char **argv) { + if(argc!=2) return 97; + int kernel=atoi(argv[1]); + if(kernel<0 || kernel>=6) return 98; + _Alignas(16) uint8_t storage[96]; + for(int i=0;i<96;i++) storage[i]=(uint8_t)(i*31+7); + const uint8_t *count=storage+1; + if((uintptr_t)count%16!=1) return 99; + int status=observe_count(originals[kernel],kernel,0,0,count,0); + if(WIFEXITED(status) && WEXITSTATUS(status)==0) + printf("COUNT_NATIVE_STATUS kernel=%d mod16=%llu wait=%d outcome=normal\n",kernel,(unsigned long long)((uintptr_t)count%16),status); + else if(WIFSIGNALED(status) && WTERMSIG(status)==SIGSEGV) + printf("COUNT_NATIVE_STATUS kernel=%d mod16=%llu wait=%d outcome=fault\n",kernel,(unsigned long long)((uintptr_t)count%16),status); + else { fprintf(stderr,"original count status=%x\n",status); return 100; } + return 0; +} +` + dir := t.TempDir() + input, executable := filepath.Join(dir, "original.c"), filepath.Join(dir, "original-count-observer") + if err := os.WriteFile(input, []byte(x86PackedShiftLegacyCountCHeader+x86PackedShiftOriginalCountC()+x86PackedShiftCountCChecks+main), 0600); err != nil { + t.Fatal(err) + } + arguments := append(append([]string(nil), compiler[1:]...), "-O2", input, "-o", executable) + ctx, cancel := runtimeOracleStageContext(t) + output, err := runtimeOracleCombinedOutput(ctx, exec.CommandContext(ctx, compiler[0], arguments...)) + cancel() + if err != nil { + t.Fatalf("original opcode status observer: %v\n%s", err, output) + } + return executable +} diff --git a/amd64_packed_shift_native_count_runtime_test.go b/amd64_packed_shift_native_count_runtime_test.go new file mode 100644 index 000000000..900588393 --- /dev/null +++ b/amd64_packed_shift_native_count_runtime_test.go @@ -0,0 +1,232 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86PackedShiftLegacyNativeStatusSemantics(t *testing.T) { + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { + if !rosettaAvailable() { + t.Fatal("Darwin x86 diagnostic transport required") + } + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" || runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("required Linux original/LLVM native-count counterpart supplies this execution axis") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + assembly, sigs := x86PackedShiftLegacyMemoryAssembly() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + driver := x86PackedShiftLegacyCountCHeader + x86PackedShiftLegacyCountDeclarations() + + x86PackedShiftOriginalCountC() + x86PackedShiftCountCChecks + x86PackedShiftLegacyNativeCountCMain + compileAndRunRuntimeTestForTarget(t, llc, clang, "legacy_count_status", triple, ir, driver, prefix) + t.Logf("6 original-opcode/LLVM kernels ×48 numerical inputs, 90 unaligned status comparisons, 6 full128 guard-page fault pairs; transport=%v", prefix) +} + +func x86PackedShiftLegacyCountDeclarations() string { + var source strings.Builder + for i := range x86LegacyShiftMemoryOpcodes { + fmt.Fprintf(&source, "extern void legacycount%d(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", i) + } + source.WriteString("static legacy_fn kernels[]={legacycount0,legacycount1,legacycount2,legacycount3,legacycount4,legacycount5};\n") + return source.String() +} + +func x86PackedShiftLegacyMemoryAssembly() (string, map[string]FuncSig) { + var source strings.Builder + sigs := map[string]FuncSig{} + for i, operation := range x86LegacyShiftMemoryOpcodes { + name := fmt.Sprintf("legacycount%d", i) + fmt.Fprintf(&source, `TEXT %s(SB),4,$0-32 +MOVQ out+0(FP),AX +MOVQ data+8(FP),BX +MOVQ count+16(FP),CX +MOVQ seed+24(FP),DX +MOVOU (BX),X0 +CMPQ DX,$0 +JE carry +CMPQ DX,$1 +JE overflow +MOVQ $1,DX +CMPQ DX,$0 +JMP shifted +carry: +MOVQ $-1,DX +ADDQ $1,DX +JMP shifted +overflow: +MOVQ $0x7fffffffffffffff,DX +ADDQ $1,DX +shifted: +%s (CX),X0 +MOVOU X0,(AX) +SETCS 16(AX) +SETOS 17(AX) +SETEQ 18(AX) +SETMI 19(AX) +SETPS 20(AX) +RET +`, name, operation.name) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + } + return source.String(), sigs +} + +const x86PackedShiftLegacyCountCHeader = ` +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*legacy_fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t); +` + +// This independent reference executes the original six physical opcodes. The +// Go object oracle binds each corresponding mnemonic to its original bytes; +// neither this reference nor the math oracle substitutes a different load. +func x86PackedShiftOriginalCountC() string { + var source strings.Builder + for i, mnemonic := range []string{"psllw", "pslld", "psllq", "psrlw", "psrld", "psrlq"} { + fmt.Fprintf(&source, ` +static void original%d(uint8_t *out,const uint8_t *data,const uint8_t *count,uint64_t seed) { + __asm__ volatile( + "movdqu (%%[data]),%%%%xmm0\n\t" + "cmpq $0,%%[seed]\n\tje 1f\n\t" + "cmpq $1,%%[seed]\n\tje 2f\n\t" + "movq $1,%%%%rax\n\tcmpq $0,%%%%rax\n\tjmp 3f\n\t" + "1: movq $-1,%%%%rax\n\taddq $1,%%%%rax\n\tjmp 3f\n\t" + "2: movq $0x7fffffffffffffff,%%%%rax\n\taddq $1,%%%%rax\n\t" + "3: %s (%%[count]),%%%%xmm0\n\t" + "movdqu %%%%xmm0,(%%[out])\n\t" + "setb 16(%%[out])\n\tseto 17(%%[out])\n\tsete 18(%%[out])\n\t" + "sets 19(%%[out])\n\tsetp 20(%%[out])\n\t" + : : [out] "r"(out),[data] "r"(data),[count] "r"(count),[seed] "r"(seed) + : "rax","xmm0","cc","memory"); +} +`, i, mnemonic) + } + source.WriteString("static legacy_fn originals[]={original0,original1,original2,original3,original4,original5};\n") + return source.String() +} + +// Status observation is independent of the translated implementation. A +// successful child must still satisfy all numerical, flag and canary checks. +const x86PackedShiftCountCChecks = ` +static uint64_t read_le(const uint8_t *p,int bytes) { + uint64_t value=0; + for(int i=0;i>shift; + for(int b=0;b>(8*b))) return 20+kernel; + } + return 0; +} +static int observe_count(legacy_fn fn,int kernel,int round,int seed,const uint8_t *count,int guard) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + + if(guard) { + uint8_t data[18],out[23]; + memset(data,13,18); memset(out,0xc3,23); + fn(out+1,data+1,count,0); + _exit(90); + } + _exit(check_count(fn,kernel,round,seed,count)); + } + int status=0; + if(waitpid(child,&status,0)!=child) return -1; + return status; +} +` + +const x86PackedShiftLegacyNativeCountCMain = ` +int main(void) { + const uint64_t counts[]={0,1,7,15,16,17,31,32,33,63,64,65,255,256,UINT64_C(1)<<32,UINT64_MAX}; + for(int kernel=0;kernel<6;kernel++) { + for(int round=0;round<16;round++) for(int seed=0;seed<3;seed++) { + _Alignas(16) uint8_t count[32]; + for(int i=0;i<32;i++) count[i]=(uint8_t)(i*31+7); + for(int i=0;i<8;i++) count[i]=(uint8_t)(counts[round]>>(8*i)); + int original=check_count(originals[kernel],kernel,round,seed,count); + int translated=check_count(kernels[kernel],kernel,round,seed,count); + if(original || translated) return 10+kernel; + } + for(int offset=1;offset<16;offset++) { + _Alignas(16) uint8_t count[48]; + memset(count,0x79,sizeof(count)); + memset(count+offset,0,8); count[offset]=1; + int original=observe_count(originals[kernel],kernel,offset,offset%3,count+offset,0); + int translated=observe_count(kernels[kernel],kernel,offset,offset%3,count+offset,0); + int normal=WIFEXITED(original) && WEXITSTATUS(original)==0; + int fault=count_memory_fault(original); + if(original<0 || translated<0 || (!normal && !fault) || original!=translated) { + fprintf(stderr,"legacy kernel=%d mod16=%d original=%x translated=%x\n",kernel,offset,original,translated); + return 30+kernel; + } + } + } + size_t page=(size_t)sysconf(_SC_PAGESIZE); + uint8_t *area=mmap(0,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(area==MAP_FAILED || mprotect(area+page,page,PROT_NONE)) return 40; + memset(area,7,page); + for(int kernel=0;kernel<6;kernel++) { + int original=observe_count(originals[kernel],kernel,0,0,area+page-8,1); + int translated=observe_count(kernels[kernel],kernel,0,0,area+page-8,1); + if(original<0 || translated<0 || !count_memory_fault(original) || original!=translated) { + fprintf(stderr,"legacy full128 kernel=%d original=%x translated=%x\n",kernel,original,translated); + return 50+kernel; + } + } + if(munmap(area,page*2)) return 60; + return 0; +} +` diff --git a/amd64_packed_shift_result.go b/amd64_packed_shift_result.go new file mode 100644 index 000000000..353e23234 --- /dev/null +++ b/amd64_packed_shift_result.go @@ -0,0 +1,106 @@ +package plan9asm + +// One output byte can span two input bytes for an immediate bit shift. With an +// initialized but unknown count, it instead requires the union of every input +// bit that could reach it. Oversized logical counts contribute only zeros. +// The masks describe numerical dependence, never the native memory envelope. +func x86PackedShiftByteInputs(form x86PackedShiftForm, output int) (data, count uint64) { + laneBytes := form.laneBits / 8 + if form.countMode == x86PackedShiftBytes { + laneBytes = 16 + } + lane, at := output/laneBytes*laneBytes, output%laneBytes + first, last := at*8, at*8+7 + if form.countOperand.Kind == OpImm { + shift := int(uint8(form.countOperand.Imm)) + if form.countMode == x86PackedShiftBytes { + shift *= 8 + } + if form.left { + first, last = first-shift, last-shift + } else { + first, last = first+shift, last+shift + } + } else { + if form.left { + first = 0 + } else { + last = laneBytes*8 - 1 + } + count = 255 // The upper qword of a uniform register count is ignored. + if form.countMode == x86PackedShiftPerLane { + count = x86CallerVectorByteMask(int64(laneBytes)) << uint(lane) + } + } + if first < 0 { + first = 0 + } + if last >= laneBytes*8 { + last = laneBytes*8 - 1 + } + if first <= last { + data = x86CallerVectorByteMask(int64(last/8-first/8+1)) << uint(lane+first/8) + } + return data, count +} + +func x86PackedShiftDefinedBytes(form x86PackedShiftForm, data, counts, old uint64) uint64 { + var known uint64 + for output := 0; output < form.byteWidth; output++ { + dataMask, countMask := x86PackedShiftByteInputs(form, output) + if data&dataMask == dataMask && counts&countMask == countMask { + known |= uint64(1) << uint(output) + } + } + // A defined mask is not a known active mask. Every inactive merging byte + // still needs its old destination; inactive zeroing bytes are constants. + if form.mask >= 0 && !form.zeroing { + known &= old + } + return known +} + +func x86ResultPackedShiftStep(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + form, known, err := x86PackedShiftProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil || form.mmx { + return false, true // Physical MMX remains a separate register domain. + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false, true + } + } + if form.memory >= 0 { + // Even a saturating count or zero mask cannot create memory authority. + // Legacy m128 counts additionally need original native backing. Only + // lowering's ephemeral same-file DATA directory can provide an SB + // input here; logical FP/AUTO storage still cannot stand in for it. + memory := ins.Args[form.memory] + if _, static := x86StaticSBReadAddress(memory, arch); form.legacy && !static { + return false, true + } + if !x86ResultRead(memory, int64(form.memoryBytes), state, owned, arch) { + return false, true + } + } + bytes := func(index int) uint64 { + if index == form.memory { + return x86CallerVectorByteMask(int64(form.byteWidth)) + } + operand := ins.Args[index] + register, _ := amd64VectorRegisterIndex(operand.Reg, amd64VectorByteWidth(operand.Reg)) + return state.vectorBytes(register) + } + data, counts := bytes(form.data), ^uint64(0) + if form.countOperand.Kind != OpImm { + counts = bytes(form.count) + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + state.writeVector(destination, form.byteWidth, + x86PackedShiftDefinedBytes(form, data, counts, state.vectorBytes(destination)), form.legacy) + return true, true +} diff --git a/amd64_packed_shift_result_grammar_test.go b/amd64_packed_shift_result_grammar_test.go new file mode 100644 index 000000000..b5e5fae9a --- /dev/null +++ b/amd64_packed_shift_result_grammar_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "sort" + "strings" + "testing" +) + +func TestX86PackedShiftResultIndependentGoGrammarAxes(t *testing.T) { + inventory := make(map[string]bool) + for _, row := range x86PackedShiftResultRows() { + inventory[strings.Fields(row.row)[0]] = true + } + var names []string + for name := range inventory { + names = append(names, name) + } + sort.Strings(names) + if !reflect.DeepEqual(names, x86PackedShiftNames()) { + t.Fatal("independent whole-family denominator changed", names) + } + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var rows []string + for _, name := range names { + vector := strings.HasPrefix(name, "V") + perLane := strings.Contains(name, "LLV") || strings.Contains(name, "RLV") + byteShift := strings.HasSuffix(name, "DQ") || strings.HasSuffix(name, "O") + banks := []string{"X"} + if vector { + banks = []string{"X", "Y", "Z"} + } + for _, bank := range banks { + if !perLane { + first, last := -128, 127 + if vector { + last = 255 + } + if bank == "Z" { + first = 0 + } + for immediate := first; immediate <= last; immediate++ { + row := fmt.Sprintf("%s $%d,%s1", name, immediate, bank) + if vector { + row += "," + bank + "2" + } + rows = append(rows, row) + } + } + limit := 32 + if arch == "386" && bank == "Z" { + // cmd/asm accepts high named X/Y banks in mode32, but + // rejects high Z. Neither supplies a physical register. + limit = 8 + } + if !vector { + limit = 16 + if arch == "386" { + limit = 8 + } + } + for index := 0; index < limit; index++ { + data := fmt.Sprintf("%s%d", bank, index) + if !vector { + rows = append(rows, name+" $7,"+data) + if !byteShift { + rows = append(rows, name+" "+data+",X1") + } + continue + } + if !perLane { + rows = append(rows, name+" $7,"+data+","+bank+"2", name+" $7,"+bank+"1,"+data) + } + if !byteShift { + countBank := "X" + if perLane { + countBank = bank + } + rows = append(rows, fmt.Sprintf("%s %s%d,%s1,%s2", name, countBank, index, bank, bank), + name+" "+countBank+"0,"+data+","+bank+"2", name+" "+countBank+"0,"+bank+"1,"+data) + } + } + } + } + if arch == "386" { + rows = append(rows, "PSRLQ $4294967297,X1", "PSRLDQ $-4294967297,X1", "VPSRLQ $4294967551,X1,X2", "VPSRLDQ $4294967551,X1,X2") + } + source := "TEXT f(SB),4,$0-0\n" + strings.Join(rows, "\n") + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for index, row := range rows { + ins := file.Funcs[0].Instrs[index+1] + originalCount := ins.Args[0] + context := false + if arch == "386" { + for _, arg := range ins.Args { + reg, valid := amd64VectorRegisterIndex(arg.Reg, amd64VectorByteWidth(arg.Reg)) + context = context || valid && reg >= 8 + } + } + form, known, err := x86PackedShiftProperties(ins.Op, ins, arch) + if !known || context && !errors.Is(err, ErrProbeNeedsContext) || !context && err != nil { + t.Fatal(row, err) + } + if !context && ins.Args[0].Kind == OpImm && uint8(form.countOperand.Imm) != uint8(ins.Args[0].Imm) { + t.Fatal("classified immediate changed encoded count", row) + } + if !reflect.DeepEqual(originalCount, ins.Args[0]) { + t.Fatal("classification rewrote the retained source", row) + } + } + t.Logf("%d actual Go forms, with physical mode32 context kept separate", len(rows)) + }) + } +} + +func TestX86PackedShiftResultMode32ClassifiedImmediateObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + source := "TEXT ·view(SB),4,$0-4\nMOVL $7,AX\n" + for dword := 0; dword < 4; dword++ { + source += fmt.Sprintf("PINSRD $%d,AX,X1\n", dword) + } + source += "PSRLQ $4294967297,X1\nPSRLDQ $-4294967297,X1\nVPSRLQ $4294967551,X1,X2\nVPSRLDQ $4294967551,X2,X1\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n" + for _, target := range x86RawReturnTargets { + if target.arch != "386" { + continue + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32\n", target.arch, target.triple, api) + if err != nil { + t.Fatal(target.triple, api, err) + } + compileLLVMToObject(t, llc, target.triple, "classified-shift.ll", "classified-shift.o", ir) + } + } +} + +func TestX86PackedShiftResultMode32HighZGoRejections(t *testing.T) { + seen := make(map[string]bool) + for _, candidate := range x86PackedShiftResultRows() { + name := strings.Fields(candidate.row)[0] + if !strings.HasPrefix(name, "V") || seen[name] { + continue + } + seen[name] = true + for index := 8; index < 32; index++ { + count := "$7" + if strings.Contains(name, "LLV") || strings.Contains(name, "RLV") { + count = "Z0" + } + row := fmt.Sprintf("%s %s,Z%d,Z2", name, count, index) + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + ins := x86ScalarLaneInstruction(t, row) + if _, known, err := x86PackedShiftProperties(ins.Op, ins, "386"); !known || err == nil { + t.Fatal("mode32 physical Z register admitted", row) + } + } + } +} + +func TestX86PackedShiftResultUnresolvedFormsCannotEmit(t *testing.T) { + for _, row := range []string{"PSLLW $0,X1", "PSRLDQ $0,X1", "VPSRLQ $0,X1,X2", "VPSRLDQ $0,X1,X2"} { + for _, mutation := range []string{"floating", "unresolved"} { + ins := x86ScalarLaneInstruction(t, row) + if mutation == "floating" { + ins.Args[0].ImmIsFloat = true + } else { + ins.Args[0].ImmRaw = "missing" + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + before := output.Len() + if _, _, err := ctx.lowerX86PackedShift(ins.Op, ins); err == nil || output.Len() != before { + t.Fatal("noninteger immediate emitted IR", row, mutation, err) + } + } + } + for _, mutation := range []string{"displacement", "index"} { + ins := x86ScalarLaneInstruction(t, "VPSRLQ (AX),X1,X2") + if mutation == "displacement" { + ins.Args[0].Mem.OffRaw = "missing" + } else { + ins.Args[0].Mem.IndexExt = "unknown" + } + if _, known, err := x86PackedShiftProperties(ins.Op, ins, "amd64"); !known || err == nil { + t.Fatal("unresolved memory accepted", mutation) + } + } +} diff --git a/amd64_packed_shift_result_runtime_test.go b/amd64_packed_shift_result_runtime_test.go new file mode 100644 index 000000000..1fe4f3119 --- /dev/null +++ b/amd64_packed_shift_result_runtime_test.go @@ -0,0 +1,152 @@ +package plan9asm + +import ( + "fmt" + "strconv" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedShiftPartialResults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, native, cases strings.Builder + for index, row := range x86PackedShiftResultRows() { + name := fmt.Sprint("partialShift", index) + fmt.Fprintf(&declarations, "func %s(data,count uint32) uint32\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-12\nMOVL data+0(FP),AX\nMOVL count+4(FP),BX\nMOVL $0,CX\n", name) + for dword := 0; dword < 4; dword++ { + if row.data&(15< +#include +#include +` + native.String() + ` +struct Case { + uint32_t (*translated)(uint32_t,uint32_t); + void (*original)(const uint8_t*,const uint8_t*,uint8_t*); + const char *row; + unsigned bits; + int immediate,left,perlane; + uint16_t dataKnown,countKnown; + unsigned lane; +}; +static const struct Case cases[]={ +` + cases.String() + `}; +static uint64_t read_shift_bytes(const uint8_t *bytes,unsigned width) { + uint64_t value=0; + for(unsigned i=0;idataKnown&(1u<>(i%4*8)); + if(test->countKnown&(1u<>(i*8)):0; + } + for(unsigned output=0;output<128;output++) { + unsigned base=output/test->bits*test->bits; + uint64_t shift=test->immediate>=0?(unsigned)test->immediate: + test->perlane?read_shift_bytes(count+base/8,test->bits/8):read_shift_bytes(count,8); + if(shift>=test->bits) continue; + int from=(int)(output%test->bits)+(test->left?-(int)shift:(int)shift); + if(from<0 || from>=(int)test->bits) continue; + unsigned input=base+(unsigned)from; + want[output/8]|=((data[input/8]>>(input%8))&1)<<(output%8); + } + uint32_t expected=(uint32_t)read_shift_bytes(want+test->lane*4,4); + uint32_t got=test->translated(inputs[sample],counts[n]); + if(got!=expected) { + fprintf(stderr,"%s sample=%u count=%u LLVM=%x want=%x\n",test->row,sample,counts[n],got,expected); + return 1; + } + if(test->original) { + test->original(count,data,original); + if(read_shift_bytes(original+test->lane*4,4)!=expected) { + fprintf(stderr,"%s original Go selected output differs\n",test->row); + return 2; + } + } + } + return 0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "shift-partial-result", triple, text, oracle, prefix) + }) + } + t.Log("44 partial-state scalar results x 96 inputs; original Go legacy opcodes plus independent bit oracle; no native AVX512 claim") +} diff --git a/amd64_packed_shift_result_test.go b/amd64_packed_shift_result_test.go new file mode 100644 index 000000000..24f65d243 --- /dev/null +++ b/amd64_packed_shift_result_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// This inventory is independent of the production opcode maps. Initialized +// dwords are deliberately narrower than X wherever the selected output permits. +type x86PackedShiftResultCase struct { + row string + data uint16 + count uint16 + lane int +} + +func x86PackedShiftResultRows() []x86PackedShiftResultCase { + var rows []x86PackedShiftResultCase + add := func(row string, data, count uint16, lane int) { + rows = append(rows, x86PackedShiftResultCase{row, data, count, lane}) + } + for _, op := range []string{"PSLLW", "PSLLL", "PSLLQ", "PSRLW", "PSRLL", "PSRLQ"} { + add(op+" $7,X1", 0xffff, 0, 0) + add(op+" X0,X1", 0xffff, 0x00ff, 0) + } + for _, op := range []string{"PSLLO", "PSLLDQ"} { + add(op+" $12,X1", 0x000f, 0, 3) + } + for _, op := range []string{"PSRLO", "PSRLDQ"} { + add(op+" $4,X1", 0x00f0, 0, 0) + } + for _, op := range []string{"VPSLLW", "VPSLLD", "VPSLLQ", "VPSRLW", "VPSRLD", "VPSRLQ"} { + add(op+" $7,X1,X2", 0xffff, 0, 0) + add(op+" X0,X1,X2", 0xffff, 0x00ff, 0) + } + for _, op := range []string{"VPSLLVW", "VPSLLVD", "VPSLLVQ", "VPSRLVW", "VPSRLVD", "VPSRLVQ"} { + add(op+" X0,X1,X2", 0xffff, 0xffff, 0) + } + add("VPSLLDQ $12,X1,X2", 0x000f, 0, 3) + add("VPSRLDQ $4,X1,X2", 0x00f0, 0, 0) + add("PSRLQ $32,X1", 0x00f0, 0, 0) + add("PSLLQ $32,X1", 0x000f, 0, 1) + add("PSLLQ X0,X1", 0x000f, 0x00ff, 0) + add("PSRLQ X0,X1", 0x00f0, 0x00ff, 1) + add("VPSLLVQ X0,X1,X2", 0x000f, 0x00ff, 0) + add("VPSRLVQ X0,X1,X2", 0x00f0, 0x00ff, 1) + add("PSRLDQ $16,X1", 0, 0, 0) + add("PSLLQ $64,X1", 0, 0, 0) + return rows +} + +func TestX86PackedShiftResultThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + var source, declarations strings.Builder + for index, row := range x86PackedShiftResultRows() { + name := fmt.Sprint("shiftResult", index) + fmt.Fprintf(&declarations, "func %s() uint32\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-4\n%s\n", name, + x86PackedShiftResultBody(row.row, row.data, row.count, row.lane, "ret+0(FP)")) + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + compileLLVMToObject(t, llc, target.triple, "shift-result.ll", "shift-result.o", ir) + } + }) + } +} + +func TestX86PackedShiftResultMemoryAndMaskBoundaries(t *testing.T) { + for _, row := range []string{ + "PSRLQ (AX),X1", "PSRLQ a+0(FP),X1", "PSRLQ local-16(SP),X1", + "VPSRLQ $255,(AX),X2", "VPSRLDQ $16,·data(SB),X2", + "VPSRLVQ (AX),X1,X2", "VPSRLQ.BCST $64,(AX),X2", + "PSRLQ $7,M1", + } { + ins := x86ScalarLaneInstruction(t, row) + state := x86ResultState{fp: make([]bool, 16), auto: make([]bool, 16)} + for i := range state.fp { + state.fp[i], state.auto[i] = true, true + } + for i := range state.xmm { + state.xmm[i] = 0xffff + } + owned := make([]bool, 16) + for i := range owned { + owned[i] = true + } + if x86ResultTransfer(ins, &state, owned, 16, "amd64") || len(ins.x86CallerMemory) != 0 { + t.Fatal("numerical effect borrowed native/MMX permission", row) + } + } + for _, row := range []string{"VPSRLQ $7,X1,K3,X2", "VPSRLQ.Z $7,X1,K3,X2"} { + ins := x86ScalarLaneInstruction(t, row) + for _, initialized := range []bool{false, true} { + state := x86ResultState{} + state.xmm[1], state.xmm[2], state.masks[3] = 0xffff, 0x5a5a, initialized + accepted := x86ResultTransfer(ins, &state, nil, 0, "amd64") + if accepted != initialized { + t.Fatal("uninitialized mask was ignored", row) + } + if initialized { + want := uint16(0x5a5a) + if strings.Contains(row, ".Z") { + want = 0xffff + } + if state.xmm[2] != want { + t.Fatal("merge/zero effect", row, state.xmm[2], want) + } + } + } + } + for _, body := range []string{ + "MOVL $7,AX\nPINSRD $0,AX,X1\nPSRLQ $1,X1\nPEXTRD $0,X1,AX\n", + "MOVL $7,AX\nPINSRD $1,AX,X1\nPSRLDQ $3,X1\nPEXTRD $0,X1,AX\n", + "MOVL $7,AX\nPINSRD $0,AX,X0\nPINSRD $0,AX,X1\nPSLLQ X0,X1\nPEXTRD $0,X1,AX\n", + } { + _, file, sig, source := x86ResultInitializationFixture(t, "amd64", "func Value() uint32", body+"MOVL AX,ret0+0(FP)\nRET") + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("one missing dependency acquired a result", err) + } + } + } +} + +func TestX86PackedShiftResultOwnedMemoryStillNeedsFullNativeTuple(t *testing.T) { + for _, fixture := range []struct { + row string + width int + }{ + {"VPSRLQ a+0(FP),X1,X2", 16}, + {"VPSRLVQ a+0(FP),X1,X2", 16}, + {"VPSRLVQ.BCST a+0(FP),X1,X2", 8}, + {"VPSRLQ $64,a+0(FP),X2", 16}, + {"VPSRLQ.BCST $64,a+0(FP),X2", 8}, + {"VPSRLDQ $16,a+0(FP),X2", 16}, + } { + for _, auto := range []bool{false, true} { + row := fixture.row + if auto { + row = strings.ReplaceAll(row, "a+0(FP)", "local-16(SP)") + } + goAssemblerOracleOutput(t, "amd64", "TEXT f(SB),4,$16-16\n"+row+"\nRET\n", "", true) + ins := x86ScalarLaneInstruction(t, row) + for missing := -1; missing < 16; missing++ { + state := x86ResultState{fp: make([]bool, 16), auto: make([]bool, 16)} + owned := make([]bool, 16) + for i := range state.fp { + state.fp[i], state.auto[i], owned[i] = i != missing, i != missing, true + } + state.xmm[1] = 0xffff + accepted := x86ResultTransfer(ins, &state, owned, 16, "amd64") + want := missing < 0 || missing >= fixture.width + if accepted != want || accepted && state.xmm[2] != 0xffff || len(ins.x86CallerMemory) != 0 { + t.Fatal("logical tuple/memory authority mismatch", row, missing, accepted, want) + } + } + } + } +} + +func x86PackedShiftResultBody(row string, data, count uint16, lane int, result string) string { + var body strings.Builder + body.WriteString("MOVL $7,AX\n") + for register, bytes := range []uint16{count, data} { + for dword := 0; dword < 4; dword++ { + if bytes&(15<= 0 { + continue + } + if legacy && immediate == 255 { + continue + } + if immediate < 0 && memory == 2 && !perLane { + continue + } + for _, destination := range []int{0, 1, 2} { + if legacy && destination != 0 { + continue + } + // Boundary immediates need not multiply every mask/alias + // combination; dynamic counts cover all boundaries below. + if !legacy && !perLane && immediate != 1 && immediate != -1 && (mask != 0 || destination != 2) { + continue + } + name := fmt.Sprintf("shift%d", len(cases)) + operation := op + if memory == 2 { + operation += ".BCST" + } + if mask == 2 { + operation += ".Z" + } + count := "$" + fmt.Sprint(immediate) + data := bank + "0" + mode := 0 + if immediate < 0 { + count = "X1" + if perLane { + count, mode = bank+"1", 1 + } + if memory != 0 { + count = "(CX)" + } + } else if memory != 0 { + data = "(BX)" + } + if byteShift { + mode = 2 + } + row := operation + " " + count + "," + data + if !legacy { + if mask != 0 { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + cases = append(cases, x86PackedShiftNumericalCase{ + name: name, row: row, width: width, bits: bits, countMode: mode, + immediate: immediate, memory: memory, maskMode: mask, destination: destination, + left: strings.Contains(op, "SLL"), legacy: legacy, broadcast: memory == 2, + }) + } + } + } + } + } + } + return cases +} + +func x86PackedShiftNumericalAssembly(cases []x86PackedShiftNumericalCase) (string, map[string]FuncSig) { + var assembly strings.Builder + sigs := map[string]FuncSig{} + for _, test := range cases { + fmt.Fprintf(&assembly, `TEXT %s(SB),4,$0-40 +MOVQ out+0(FP),AX +MOVQ data+8(FP),BX +MOVQ count+16(FP),CX +MOVQ mask+24(FP),DX +MOVQ seed+32(FP),SI +VMOVUPS (BX),Z0 +VMOVUPS (CX),Z1 +VMOVUPS (CX),Z2 +KMOVQ DX,K1 +CMPQ SI,$0 +JE carry +CMPQ SI,$1 +JE overflow +MOVQ $1,SI +CMPQ SI,$0 +JMP shifted +carry: +MOVQ $-1,SI +ADDQ $1,SI +JMP shifted +overflow: +MOVQ $0x7fffffffffffffff,SI +ADDQ $1,SI +shifted: +%s +VMOVUPS Z%d,(AX) +SETCS 64(AX) +SETOS 65(AX) +SETEQ 66(AX) +SETMI 67(AX) +SETPS 68(AX) +RET +`, test.name, test.row, test.destination) + sigs[test.name] = FuncSig{Name: test.name, Args: []LLVMType{Ptr, Ptr, Ptr, I64, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + {Offset: 32, Type: I64, Index: 4, Field: -1}, + }}} + } + return assembly.String(), sigs +} + +const x86PackedShiftIndependentC = ` +#include +#include +#include +typedef void (*shift_fn)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t); +struct shift_case { shift_fn fn; const char *row; int width,bits,mode,immediate,memory,mask,destination,left,legacy,broadcast; }; +static uint64_t read_le(const uint8_t *p,int bytes) { + uint64_t value=0; + for(int i=0;idestination==0?a:counts; + memcpy(out,old,64); + if(!c->legacy) memset(out+c->width,0,64-c->width); + if(c->mode==2) { + for(int i=0;iwidth;++i) { + int offset=i%16, from=c->left ? offset-c->immediate : offset+c->immediate; + out[i]=from>=0 && from<16 ? a[i-offset+from] : 0; + } + return; + } + int bytes=c->bits/8; + for(int i=0;iwidth;i+=bytes) { + if(c->mask && !((mask>>(i/bytes))&1)) { + if(c->mask==2) memset(out+i,0,bytes); + continue; + } + uint64_t count=c->immediate>=0?(uint64_t)c->immediate:read_le(counts+(c->mode==1 && !c->broadcast?i:0),c->mode==1?bytes:8); + uint64_t value=read_le(a+(c->broadcast && c->immediate>=0?0:i),bytes); + uint64_t result=0; + if(count<(uint64_t)c->bits) result=c->left?value<>count; + for(int b=0;b>(8*b)); + } +} +static int check_case(const struct shift_case *c,int index) { + const uint64_t limits[]={0,1,7,15,16,17,31,32,33,63,64,65,255,256,UINT64_C(1)<<32,UINT64_MAX}; + const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; + const uint8_t flags[3][5]={{1,0,1,0,1},{0,1,0,1,1},{0,0,0,0,0}}; + for(unsigned round=0;round<16;++round) for(unsigned m=0;m<4;++m) for(unsigned seed=0;seed<3;++seed) { + uint8_t aStorage[66],storage[71],beforeA[66],want[64]; + _Alignas(16) uint8_t countStorage[96],beforeCount[96]; + uint8_t *a=aStorage+1,*counts=countStorage+(c->legacy?16:3),*out=storage+1; + memset(aStorage,0xa7,sizeof(aStorage)); memset(countStorage,0xb9,sizeof(countStorage)); memset(storage,0xc3,sizeof(storage)); + for(int i=0;i<64;++i) { a[i]=(uint8_t)(i*71+round*97+13); counts[i]=(uint8_t)(i*43+19); } + int bytes=c->bits/8; + for(int i=0;i<64;i+=bytes) { + uint64_t count=limits[(round+i/bytes)%16]; + if(c->mode!=1) count=limits[round]; + for(int b=0;b>(8*b)); + } + if(c->mode==0) for(int b=0;b<8;++b) counts[b]=(uint8_t)(limits[round]>>(8*b)); + memcpy(beforeA,aStorage,sizeof(beforeA)); memcpy(beforeCount,countStorage,sizeof(beforeCount)); + expected(want,a,counts,masks[m],c); + c->fn(out,a,counts,masks[m],seed); + if(memcmp(want,out,64) || memcmp(flags[seed],out+64,5) || storage[0]!=0xc3 || storage[70]!=0xc3 || + memcmp(aStorage,beforeA,sizeof(beforeA)) || memcmp(countStorage,beforeCount,sizeof(beforeCount))) { + for(int i=0;i<69;++i) if(out[i]!=(i<64?want[i]:flags[seed][i-64])) { + fprintf(stderr,"case=%d %s round=%u mask=%u seed=%u byte=%d actual=%02x expected=%02x\n",index,c->row,round,m,seed,i,out[i],i<64?want[i]:flags[seed][i-64]); break; + } + return 1; + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedLogicalShiftCompleteSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("requires Linux/amd64 runtime counterpart") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM22 tools required") + } + cases := x86PackedShiftNumericalCases() + assembly, sigs := x86PackedShiftNumericalAssembly(cases) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + var driver strings.Builder + driver.WriteString(x86PackedShiftIndependentC) + for _, c := range cases { + fmt.Fprintf(&driver, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t,uint64_t);\n", c.name) + } + driver.WriteString("static const struct shift_case cases[]={\n") + for _, c := range cases { + fmt.Fprintf(&driver, "{%s,%q,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d},\n", c.name, c.row, c.width, c.bits, c.countMode, c.immediate, c.memory, c.maskMode, c.destination, x86ShiftBoolean(c.left), x86ShiftBoolean(c.legacy), x86ShiftBoolean(c.broadcast)) + } + driver.WriteString("};\nint main(void) { for(unsigned i=0;i>(8*i)) } + %s(unsafe.Pointer(&out[1]),unsafe.Pointer(&data[1]),unsafe.Pointer(&countData[1]),uint64(seed)) + if out[0]!=0xc3 || out[22]!=0xc3 { t.Fatal("canary") } + for i:=0;i<5;i++ { if out[17+i]!=flags[seed][i] { t.Fatalf("flags seed=%%d bit=%%d got=%%d",seed,i,out[17+i]) } } + for i:=0;i<16;i++ { + var want byte +`, name) + if byteShift { + from := "i-7" + if strings.Contains(op, "SRL") { + from = "i+7" + } + fmt.Fprintf(&tests, "from:=%s\nif from>=0 && from<16 { want=data[from+1] }\n", from) + } else { + operator := "<<" + if strings.Contains(op, "SRL") { + operator = ">>" + } + fmt.Fprintf(&tests, "offset:=(i/%d)*%d\nvar value uint64\nfor b:=0;b<%d;b++ { value|=uint64(data[offset+b+1])<<(8*b) }\nif count<%d { want=byte((value %s count)>>(8*(i%%%d))) }\n", bits/8, bits/8, bits/8, bits, operator, bits/8) + } + tests.WriteString("if out[i+1]!=want { t.Fatalf(\"byte %d got=%x want=%x\",i,out[i+1],want) }\n}\n}\n}\n") + } + tests.WriteString("}\n") + dir := t.TempDir() + for name, bytes := range map[string]string{"go.mod": "module original\n\ngo 1.20\n", "original.go": source.String(), "original_amd64.s": assembly.String(), "original_test.go": tests.String()} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(bytes), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOARCH=amd64", "CGO_ENABLED=0", "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go legacy shift: %v\n%s", err, output) + } + t.Logf("10 original Go SSE2 mnemonic/alias programs ×48 initialized/flags/canary inputs; Rosetta=%t", crossRosetta) +} diff --git a/amd64_packed_string_vector_linux_amd64_test.go b/amd64_packed_string_vector_linux_amd64_test.go new file mode 100644 index 000000000..8b60d50a0 --- /dev/null +++ b/amd64_packed_string_vector_linux_amd64_test.go @@ -0,0 +1,41 @@ +//go:build linux && amd64 + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// The portable byte oracle does not execute a VEX opcode. This required native +// counterpart executes actual Go ABI0 AVX sources and both LLVM levels, with +// complete Y aliases, six flags/LAHF/index and independent input canaries. +func TestCrossLinuxRuntimeMatrixX86PackedStringImplicitVectorNativeAVX(t *testing.T) { + x87IA32RequireNativeHost(t) + execution := x86FloatCompareExecutionTools(t) + execution.compiler = append(execution.compiler, "-mno-red-zone") + execution.requireAVX(t) + cases, declarations, source := x86PackedStringNativeYFixture() + original := x86DefinedStatusBuildOriginal(t, execution, declarations, source, x86PackedStringNativeGoDriver(cases)) + output := execution.run(t, original) + want := fmt.Sprintf("original native AVX packed-string observations %d failures 0\n", len(cases)*4) + if string(output) != want { + t.Fatalf("actual original Go AVX byte oracle: %s", output) + } + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "amd64", execution.triple, "Go") + main := strings.Replace(x86PackedStringVectorCOracle(cases), "view<3", "view<2", 1) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "native-packed-string-vector", input, main, level) + want := fmt.Sprintf("portable packed-string byte observations %d failures 0\n", len(cases)*4) + if string(output) != want { + t.Fatalf("native LLVM AVX byte oracle: %s", output) + } + }) + } +} diff --git a/amd64_packed_string_vector_runtime_test.go b/amd64_packed_string_vector_runtime_test.go new file mode 100644 index 000000000..9a2406d93 --- /dev/null +++ b/amd64_packed_string_vector_runtime_test.go @@ -0,0 +1,319 @@ +//go:build unix + +package plan9asm + +import ( + "crypto/sha256" + "debug/elf" + "fmt" + "os" + "regexp" + "strings" + "testing" +) + +func x86PackedStringVectorCOracle(cases []x86PackedStringVectorCase) string { + var functions, registrations strings.Builder + for index, row := range cases { + name := fmt.Sprintf("stringvector%d", index) + fmt.Fprintf(&functions, "extern void %s(uint8_t *,const uint8_t *);\n", name) + // Native legacy SSE supplies an independent low-mask/index and flags + // oracle. VEX upper-zeroing is checked separately against the ISA rule; + // this portable test never claims execution of a native VEX instruction. + fmt.Fprintf(&functions, `static void reference%d(const uint8_t *left,const uint8_t *right,uint8_t *mask,uint64_t *flags,uint64_t *index) { + uint64_t a=5,d=9,c=73,f; + __asm__ volatile( + "movdqu (%%[left]),%%%%xmm1\n\t" + "movdqu (%%[right]),%%%%xmm2\n\t" + "%s $%d,%%%%xmm2,%%%%xmm1\n\t" + "pushfq\n\t" + "popq %%[flags]\n\t" + "movdqu %%%%xmm0,(%%[mask])\n\t" + : "+a"(a),"+d"(d),"+c"(c),[flags]"=&r"(f) + : [left]"r"(left),[right]"r"(right),[mask]"r"(mask) + : "xmm0","xmm1","xmm2","cc","memory"); + *flags=f&UINT64_C(0x8d5); + *index=c; +} + +`, index, strings.ToLower(strings.TrimPrefix(row.op, "V")), row.imm) + fmt.Fprintf(®istrations, "{%s,reference%d,%d,%d,%d,%d,%q},\n", name, index, + row.source, row.dst, x86FloatAddSubBoolInt(strings.HasSuffix(row.op, "M")), + x86FloatAddSubBoolInt(strings.HasPrefix(row.op, "V")), row.op) + } + return `#include +#include +#include +` + functions.String() + ` +struct program { + void (*run)(uint8_t *,const uint8_t *); + void (*reference)(const uint8_t *,const uint8_t *,uint8_t *,uint64_t *,uint64_t *); + int source,destination,mask,vector; + const char *name; +}; +static const struct program programs[]={ +` + registrations.String() + `}; + +int main(void) { + unsigned observations=0; + for (unsigned program=0;programreference(input+1+p->destination*64,input+1+p->source*64,mask,&flags,&index); + p->run(output+1,input+1); + __asm__ volatile("stmxcsr %0":"=m"(after)); + __asm__ volatile("ldmxcsr %0"::"m"(saved)); + memcpy(&gotFlags,output+1+336,8); + memcpy(&gotIndex,output+1+352,8); + uint8_t lahf=(uint8_t)((flags&UINT64_C(0xd5))|2); + if ((gotFlags&UINT64_C(0x8d5))!=flags || output[1+344]!=lahf || gotIndex!=index || + after!=before || output[0]!=0xc3 || output[361]!=0xc3 || memcmp(input,original,sizeof(input))) { + fprintf(stderr,"%s/%u flags/index/input/canary mismatch\n",p->name,program); + return 1; + } + const unsigned widths[]={16,32,64},offsets[]={0,16,48}; + for (unsigned reg=0;reg<3;++reg) { + for (unsigned view=0;view<3;++view) { + for (unsigned byte=0;bytemask && reg==0) { + if (byte<16) { + want=mask[byte]; + } else if (p->vector) { + want=0; + } + } + uint8_t got=output[1+reg*112+offsets[view]+byte]; + if (got!=want) { + fprintf(stderr,"%s/%u round=%u bank=%u width=%u byte=%u got=%u want=%u\n", + p->name,program,round,reg,widths[view],byte,got,want); + return 1; + } + } + } + } + ++observations; + } + } + printf("portable packed-string byte observations %u failures 0\n",observations); + return 0; +} +` +} + +func TestX86PackedStringImplicitVectorByteOracle(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.compiler = append(execution.compiler, "-mno-red-zone") + cases := x86PackedStringVectorCases() + declarations, source := x86PackedStringVectorFixture(cases, "amd64") + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, "amd64", execution.triple, "Go") + // Only the execution target-feature request changes: all source operations + // and LLVM byte arithmetic stay intact. This makes the wide virtual bank + // oracle executable using SSE4.2, without asserting native AVX/AVX512. + features := regexp.MustCompile(`"target-features"="([^"]*)"`) + ir = features.ReplaceAllStringFunc(ir, func(attribute string) string { + values := features.FindStringSubmatch(attribute)[1] + for _, value := range strings.Split(values, ",") { + switch value { + case "+avx", "+avx512f", "+avx512vl", "+sse4.2": + default: + t.Fatalf("unreviewed portable feature request: %q", value) + } + } + return `"target-features"="+sse4.2"` + }) + if strings.Contains(ir, `"target-features"="+avx`) { + t.Fatal("portable oracle retained an unreviewed native AVX feature request") + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + output := execution.compileAndRunLLVM(t, "packed-string-vector-bytes", input, x86PackedStringVectorCOracle(cases), level) + want := fmt.Sprintf("portable packed-string byte observations %d failures 0\n", len(cases)*4) + if string(output) != want { + t.Fatalf("incomplete byte oracle: %s", output) + } + }) + } +} + +func x86PackedStringNativeYFixture() ([]x86PackedStringVectorCase, string, string) { + var cases []x86PackedStringVectorCase + for _, row := range x86PackedStringVectorCases() { + if row.imm == 8 { + cases = append(cases, row) + } + } + declarations, source := x86PackedStringVectorFixture(cases, "amd64") + // Actual native AVX has 32-byte banks. Z-bank execution is a distinct + // AVX512 requirement and is not silently inferred from this counterpart. + for reg := 0; reg < 3; reg++ { + source = strings.ReplaceAll(source, fmt.Sprintf("VMOVUPS %d(SI),Z%d\n", reg*64, reg), + fmt.Sprintf("VMOVUPS %d(SI),Y%d\n", reg*64, reg)) + source = strings.ReplaceAll(source, fmt.Sprintf("VMOVUPS Z%d,%d(DI)\n", reg, reg*112+48), "") + } + return cases, declarations, source +} + +func x86PackedStringNativeGoDriver(cases []x86PackedStringVectorCase) string { + var registrations strings.Builder + for index, row := range cases { + fmt.Fprintf(®istrations, "{stringvector%d,%d,%d,%t,%t,%t,%q},\n", index, row.source, row.dst, + strings.HasSuffix(row.op, "M"), strings.HasPrefix(row.op, "V"), + strings.Contains(row.op, "ESTR"), row.op) + } + return `package main + +import ( + "encoding/binary" + "fmt" + "os" + "unsafe" +) + +type program struct { + run func(*[360]byte,*[192]byte) + source,destination int + mask,vector,explicit bool + name string +} + +var programs=[]program{ +` + registrations.String() + `} + +func fail(name string,round int,detail string) { + fmt.Fprintf(os.Stderr,"%s round=%d %s\n",name,round,detail) + os.Exit(1) +} + +func main() { + observations:=0 + for _,p:=range programs { + for round:=0;round<4;round++ { + input:=[194]byte{} + output:=[362]byte{} + for i:=range input { + input[i]=byte(i*73+round*19+1) + } + input[1+3]=0 + input[1+64+7]=0 + input[1+128+11]=0 + original:=input + for i:=range output { + output[i]=0xc3 + } + p.run((*[360]byte)(unsafe.Pointer(&output[1])),(*[192]byte)(unsafe.Pointer(&input[1]))) + if input!=original || output[0]!=0xc3 || output[361]!=0xc3 { + fail(p.name,round,"input/canary mismatch") + } + left:=input[1+p.destination*64:1+p.destination*64+16] + right:=input[1+p.source*64:1+p.source*64+16] + la,lb:=5,9 + if !p.explicit { + la,lb=16,16 + for i:=0;i<16;i++ { + if left[i]==0 && la==16 { + la=i + } + if right[i]==0 && lb==16 { + lb=i + } + } + } + var result uint16 + for i:=0;i<16;i++ { + if i>=la && i>=lb || i>uint(b*8)) + } + } else if p.vector { + want=0 + } + } + if got:=output[1+reg*112+offsets[view]+b];got!=want { + fail(p.name,round,fmt.Sprintf("bank=%d width=%d byte=%d got=%d want=%d",reg,width,b,got,want)) + } + } + } + } + observations++ + } + } + fmt.Printf("original native AVX packed-string observations %d failures 0\n",observations) +} +` +} + +func TestX86PackedStringNativeAVXCounterpartOriginalObject(t *testing.T) { + cases, declarations, source := x86PackedStringNativeYFixture() + execution := x86FloatCompareExecution{triple: "x86_64-unknown-linux-gnu"} + executable := x86DefinedStatusBuildOriginal(t, execution, declarations, source, x86PackedStringNativeGoDriver(cases)) + object, err := elf.Open(executable) + if err != nil { + t.Fatal(err) + } + defer object.Close() + if object.Class != elf.ELFCLASS64 || object.Machine != elf.EM_X86_64 || object.Data != elf.ELFDATA2LSB { + t.Fatal("native AVX counterpart is not an actual Linux/amd64 ELF") + } + bytes, err := os.ReadFile(executable) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go Linux/amd64 ELF bytes=%d sha=%x; compile-only, native execution remains required", len(bytes), sha256.Sum256(bytes)) +} diff --git a/amd64_packed_string_vector_test.go b/amd64_packed_string_vector_test.go new file mode 100644 index 000000000..de3dd255e --- /dev/null +++ b/amd64_packed_string_vector_test.go @@ -0,0 +1,352 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86PackedStringCompleteActualGoTableAttributes(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + read := func(name string) string { + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + return strings.Join(strings.Fields(string(data)), "") + } + legacy, vector := read("asm6.go"), read("avx_optabs.go") + for _, table := range []struct{ source, definition string }{ + {legacy, "var yxshuf = []ytab{ {Zibm_r, 2, argList{Yu8, Yxm, Yxr}}, }"}, + {vector, "var _yvaeskeygenassist = []ytab{ {zcase: Zvex_i_rm_r, zoffset: 0, args: argList{Yu8, Yxm, Yxr}}, {zcase: Zvex_i_rm_r, zoffset: 2, args: argList{Yi8, Yxm, Yxr}}, }"}, + } { + if !strings.Contains(table.source, strings.Join(strings.Fields(table.definition), "")) { + t.Fatal("actual selected SDK packed-string complete operand table changed", table.definition) + } + } + for _, row := range []struct { + op, opcode, width string + }{ + {"PCMPESTRI", "61", "WIG"}, {"PCMPESTRM", "60", "WIG"}, + {"PCMPISTRI", "63", "WIG"}, {"PCMPISTRM", "62", "W0"}, + } { + legacyRecord := fmt.Sprintf("{A%s,yxshuf,Pq,opBytes{0x3a,0x%s,0}}", row.op, row.opcode) + vectorRecord := fmt.Sprintf("{as:AV%s,ytab:_yvaeskeygenassist,prefix:Pavx,op:opBytes{avxEscape|vex128|vex66|vex0F3A|vex%s,0x%s,}},", row.op, row.width, row.opcode) + if !strings.Contains(legacy, legacyRecord) || !strings.Contains(vector, vectorRecord) { + t.Fatalf("actual selected SDK opcode/prefix/width attributes changed: %s", row.op) + } + } + // Count actual family registrations as well as checking each expected row. + // A shared ytab alone is not evidence of equal WIG or prefix attributes. + pattern := regexp.MustCompile(`\{(?:as:)?A(V?PCMP[EI]STR[[:alnum:]]+),`) + actual := make(map[string]int) + for _, source := range []string{legacy, vector} { + for _, match := range pattern.FindAllStringSubmatch(source, -1) { + actual[match[1]]++ + } + } + if len(actual) != len(amd64PackedStringCompareSpecs) { + t.Fatal("actual selected SDK packed-string family denominator changed", actual) + } + for opcode := range amd64PackedStringCompareSpecs { + if actual[string(opcode)] != 1 { + t.Fatalf("actual selected SDK family registration is missing/duplicate: %s", opcode) + } + } +} + +type x86PackedStringVectorCase struct { + op string + source, dst int + memory bool + imm int +} + +func x86PackedStringVectorCases() []x86PackedStringVectorCase { + var cases []x86PackedStringVectorCase + for _, op := range []string{ + "PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM", + "VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM", + } { + for _, operands := range []struct { + source, dst int + memory bool + }{{0, 1, false}, {1, 0, false}, {0, 0, false}, {1, 1, false}, {2, 1, false}, {2, 0, true}, {2, 1, true}} { + for _, immediate := range []int{0, 8, 64} { + cases = append(cases, x86PackedStringVectorCase{op, operands.source, operands.dst, operands.memory, immediate}) + } + } + } + return cases +} + +func x86PackedStringVectorFixture(cases []x86PackedStringVectorCase, arch string) (string, string) { + var declarations, source strings.Builder + move, pointer, args := "MOVQ", 8, 16 + if arch == "386" { + move, pointer, args = "MOVL", 4, 8 + } + for index, row := range cases { + name := fmt.Sprintf("stringvector%d", index) + fmt.Fprintf(&declarations, "func %s(out *[360]byte, in *[192]byte)\n", name) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s out+0(FP),DI\n%s in+%d(FP),SI\n", name, args, move, move, pointer) + for register := 0; register < 3; register++ { + fmt.Fprintf(&source, "VMOVUPS %d(SI),Z%d\n", register*64, register) + } + source.WriteString("MOVL $5,AX\nMOVL $9,DX\nMOVL $73,CX\n") + operand := fmt.Sprintf("X%d", row.source) + if row.memory { + operand = fmt.Sprintf("%d(SI)", row.source*64) + } + fmt.Fprintf(&source, "%s $%d,%s,X%d\n", row.op, row.imm, operand, row.dst) + for register := 0; register < 3; register++ { + for _, view := range []struct { + bank string + offset int + }{{"X", 0}, {"Y", 16}, {"Z", 48}} { + fmt.Fprintf(&source, "VMOVUPS %s%d,%d(DI)\n", view.bank, register, register*112+view.offset) + } + } + if arch == "386" { + source.WriteString("PUSHFL\nPOPL AX\nMOVL AX,336(DI)\nLAHF\nMOVB AH,344(DI)\nMOVL CX,352(DI)\nRET\n") + } else { + source.WriteString("PUSHFQ\nPOPQ AX\nMOVQ AX,336(DI)\nLAHF\nMOVB AH,344(DI)\nMOVQ CX,352(DI)\nRET\n") + } + } + return declarations.String(), source.String() +} + +func TestX86PackedStringThreeAPIsFiveTargetsO0O2Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + cases := x86PackedStringVectorCases() + for _, target := range x86RawReturnTargets { + t.Run(target.arch+"/"+target.triple, func(t *testing.T) { + declarations, source := x86PackedStringVectorFixture(cases, target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir := x86CPUQueryIRFromDeclaration(t, source, declarations, target.arch, target.triple, api) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + input := ir + if level == "-O2" { + input = x86UndefinedTrapOptimizedIR(t, input) + } + x86PackedStringCompileObject(t, llc, target.triple, input, level) + }) + } + }) + } + }) + } +} + +func TestX86PackedStringResultEffectsAndPreEmissionIdentity(t *testing.T) { + for _, row := range go127PackedStringCompareSpecs() { + spec := amd64PackedStringCompareSpecs[Op(row.op)] + reg, width, zeroUpper := spec.resultEffect() + wantReg, wantWidth := CX, 4 + if row.mask { + wantReg, wantWidth = "X0", 16 + } + if reg != wantReg || width != wantWidth || zeroUpper != (row.vector && row.mask) { + t.Fatalf("%s implicit writer effect = %s/%d/%t", row.op, reg, width, zeroUpper) + } + for _, identity := range []Op{"", Op(row.op), Op(strings.ToLower(row.op)), "MOVOU", "UNKNOWN"} { + for _, arch := range []string{"amd64", "386"} { + t.Run(row.op+"/"+arch+"/"+string(identity), func(t *testing.T) { + ctx, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + instruction := Instr{Op: identity, Args: []Operand{ + {Kind: OpImm, Imm: 8}, {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, + }} + before, temporary := output.String(), ctx.tmp + original := fmt.Sprintf("%#v", instruction) + known, terminated, err := ctx.lowerPackedStringCompare(Op(row.op), instruction) + invalid := identity != "" && !strings.EqualFold(string(identity), row.op) + if !known || terminated || (err != nil) != invalid || invalid && (before != output.String() || temporary != ctx.tmp) || + original != fmt.Sprintf("%#v", instruction) { + t.Fatalf("implicit writer bypassed source identity or emitted before rejection: %v", err) + } + }) + } + } + } +} + +func TestX86PackedStringImmediateCompleteByteDomainOriginalObjects(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT ·query(SB),4,$0-0\n") + for _, row := range go127PackedStringCompareSpecs() { + first := 0 + if row.vector { + first = -128 + } + for immediate := first; immediate <= 255; immediate++ { + fmt.Fprintf(&source, "%s $%d,X0,X1\n", row.op, immediate) + } + } + source.WriteString("RET\n") + x86LEAActualTargetGoObject(t, arch, testTargetTriple("linux", arch), source.String()) + }) + } +} + +func TestX86PackedStringStrictMalformedOperands(t *testing.T) { + for _, row := range go127PackedStringCompareSpecs() { + for _, operands := range []string{ + "$256,X0,X1", "$-129,X0,X1", "$0.0,X0,X1", "$8,Y0,X1", + "$8,X0,X16", "$8,X0,Y1", "$8,X0,(AX)", "$8,X0,X1,X2", + } { + t.Run(row.op+"/"+operands, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-0\n" + row.op + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ir, err := Translate(file, Options{Goarch: "amd64", ResolveSym: func(symbol string) string { + return strings.TrimPrefix(symbol, "·") + }, Sigs: map[string]FuncSig{"query": {Name: "query", Ret: Void}}}) + if err == nil || ir != "" { + t.Fatal("malformed implicit vector writer returned IR", err) + } + }) + } + } +} + +func TestX86PackedStringMode32HighBankNeedsPhysicalContext(t *testing.T) { + for _, op := range []string{"VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM"} { + for _, operands := range []string{"X8,X1", "X1,X8", "X15,X15"} { + t.Run(op+"/"+operands, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·query(SB),4,$0-0\n%s $8,%s\nRET\n", op, operands) + goAssemblerOracleOutput(t, "386", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "query", "package query\nfunc query()\n") + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "386", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", ResolveSym: resolve, Sigs: sigs}) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("frontend high spelling acquired a physical mode32 vector binding: %v", err) + } + }) + } + } +} + +func TestX86PackedStringDoesNotSupplyCallerDefinedness(t *testing.T) { + for _, op := range []string{ + "PCMPESTRI", "PCMPESTRM", "PCMPISTRI", "PCMPISTRM", + "VPCMPESTRI", "VPCMPESTRM", "VPCMPISTRI", "VPCMPISTRM", + } { + for _, operand := range []string{"X0", "(DI)", "(DX)"} { + t.Run(op+"/"+operand, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", op+" $8,"+operand+",X1\nMOVL AX,(DX)", 1) + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unregistered implicit vector inputs/writers acquired caller permission: %v", err) + } + }) + } + } +} + +func TestX86PackedStringDescriptorDoesNotInventInitializedBytes(t *testing.T) { + for _, row := range go127PackedStringCompareSpecs() { + instruction := Instr{Op: Op(row.op), Args: []Operand{ + {Kind: OpImm, Imm: 8}, {Kind: OpReg, Reg: "X0"}, {Kind: OpReg, Reg: "X1"}, + }} + state := x86SourceStackState{vectors: map[Reg]uint64{"Z0": 0xffff, "Z1": 0xff}, masks: map[Reg]bool{"K1": true}} + before := state + before.vectors, before.masks = x86CloneCallerVectorState(state.vectors, state.masks) + if x86CallerVectorMoveKnown(instruction) || x86CallerVectorMoveStep(&instruction, &state, nil) || + x86SourceStackInstructionKnown(instruction, OpMOVQ, "Q", "amd64") { + t.Fatalf("%s descriptor granted an unregistered caller effect", row.op) + } + if !reflect.DeepEqual(state, before) || instruction.x86CallerMemory != nil { + t.Fatalf("%s failed caller effect mutated byte knowledge or native certificates", row.op) + } + } +} + +func TestX86PackedStringIndependentFPFieldsRequireEveryByte(t *testing.T) { + var parameters strings.Builder + for index := 0; index < 16; index++ { + if index != 0 { + parameters.WriteString(",") + } + fmt.Fprintf(¶meters, "a%d uint8", index) + } + pkg := mustGoPackage(t, "strings", "package strings\nfunc query("+parameters.String()+")\n") + for _, row := range go127PackedStringCompareSpecs() { + t.Run(row.op, func(t *testing.T) { + source := "TEXT ·query(SB),4,$0-16\n" + row.op + " $8,a0+0(FP),X1\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: "amd64", ResolveSym: resolve, Sigs: sigs} + if _, err := Translate(file, options); err != nil { + t.Fatalf("complete independent logical FP fields: %v", err) + } + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprint(missing), func(t *testing.T) { + // Each mutation starts from the original complete declaration. + // Logical FP completeness is not native address backing or an + // initialized caller-region witness. + signature := sigs["query"] + signature.Frame.Params = append([]FrameSlot(nil), signature.Frame.Params...) + signature.Frame.Params = append(signature.Frame.Params[:missing], signature.Frame.Params[missing+1:]...) + mutated := options + mutated.Sigs = map[string]FuncSig{"query": signature} + if ir, err := Translate(file, mutated); err == nil || ir != "" { + t.Fatalf("missing logical FP byte%d acquired a complete m128 source: %v", missing, err) + } + }) + } + }) + } +} + +func x86PackedStringCompileObject(t *testing.T, llc, triple, ir, level string) { + t.Helper() + if level != "-O0" && level != "-O2" { + t.Fatal("unexpected packed-string object level", level) + } + dir := t.TempDir() + input, output := filepath.Join(dir, "strings.ll"), filepath.Join(dir, "strings.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+triple, "-filetype=obj", input, "-o", output)) + info, err := os.Lstat(output) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatalf("packed-string object is missing/nonregular/empty: %v", err) + } +} diff --git a/amd64_packed_subtract_fault_runtime_test.go b/amd64_packed_subtract_fault_runtime_test.go new file mode 100644 index 000000000..032d6c1de --- /dev/null +++ b/amd64_packed_subtract_fault_runtime_test.go @@ -0,0 +1,151 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedSubtractNativeMemoryStatus(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, references, checks strings.Builder + sigs := map[string]FuncSig{} + index := 0 + for _, original := range x86PackedSubtractCallerRows() { + parsed, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+original+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := parsed.Funcs[0].Instrs[1] + form, _, err := x86PackedSubtractProperties(instruction, "amd64") + if err != nil { + t.Fatal(err) + } + if form.memory < 0 || form.mask >= 0 && !form.zeroing { + continue + } + masks := []int{0} + if form.mask >= 0 { + masks = []int{0, 1} + } + for _, mask := range masks { + name := fmt.Sprintf("packedsubtractfault%d", index) + reference := fmt.Sprintf("subtractReference%d", index) + index++ + row := strings.ReplaceAll(original, "(DI)", "(BX)") + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-8\nMOVQ input+0(FP),BX\n"+ + "VPXORD Z0,Z0,Z0\nVPXORD Z1,Z1,Z1\nVPXORD Z2,Z2,Z2\n"+ + "VPXORD Z15,Z15,Z15\nVPXORD Z21,Z21,Z21\nVPXORD Z22,Z22,Z22\n"+ + "MOVL $%d,CX\nKMOVQ CX,K1\n%s\nRET\n", name, mask, row) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*);\n", name) + if form.legacy { + mnemonic := strings.ToLower(string(instruction.Op)) + if mnemonic == "psubl" { + mnemonic = "psubd" + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t *p) {\n"+ + " typedef int32_t V __attribute__((vector_size(16)));\n V value = {0,0,0,0};\n"+ + " __asm__ volatile(\"%s (%%1),%%0\" : \"+x\"(value) : \"r\"(p) : \"memory\");\n}\n", + reference, mnemonic) + } else { + bytes := form.memoryBytes + if form.mask >= 0 { + bytes = 0 + if mask != 0 { + bytes = form.laneBits / 8 + } + } + fmt.Fprintf(&references, "__attribute__((noinline)) static void %s(const uint8_t *p) {\n"+ + " for (int i = 0; i < %d; ++i) {\n (void)((const volatile uint8_t*)p)[i];\n }\n}\n", + reference, bytes) + } + fmt.Fprintf(&checks, "for(int n=0;n<16;++n) {\n if(compare(%s,%s,memory+n)) return 1;\n}\n", reference, name) + for _, offset := range []int{0, 1, form.laneBits / 8, 16, 32, 64} { + fmt.Fprintf(&checks, "if(compare(%s,%s,boundary-%d)) return 1;\n", reference, name, offset) + } + fmt.Fprintf(&checks, "if(compare(%s,%s,0)) return 1;\n", reference, name) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + references.String() + ` +typedef struct { int signal; int code; } Status; + +static int observe(void (*fn)(const uint8_t*), const uint8_t *p, Status *out) { + pid_t child = fork(); + if (child < 0) return 1; + if (child == 0) { + struct rlimit limit = {0,0}; + if (setrlimit(RLIMIT_CORE,&limit)) _exit(3); + fn(p); + _exit(0); + } + int status; + if (waitpid(child,&status,0) != child) return 1; + if (WIFSIGNALED(status)) { + *out = (Status){WTERMSIG(status),0}; + return 0; + } + if (WIFEXITED(status)) { + *out = (Status){0,WEXITSTATUS(status)}; + return 0; + } + return 1; +} + +static int compare(void (*original)(const uint8_t*), + void (*translated)(const uint8_t*), const uint8_t *p) { + Status a, b; + if (observe(original,p,&a) || observe(translated,p,&b)) return 1; + if (a.signal != b.signal || a.code != b.code) { + fprintf(stderr,"pointer=%p mod16=%lu original=(%d,%d) translated=(%d,%d)\n", + p,(unsigned long)((uintptr_t)p%16),a.signal,a.code,b.signal,b.code); + return 1; + } + return 0; +} + +int main(void) { + long page = sysconf(_SC_PAGESIZE); + if (page <= 0 || page%64) return 1; + uint8_t *memory = mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANON,-1,0); + if (memory == MAP_FAILED) return 1; + memset(memory,0x87,(size_t)page); + if (mprotect(memory,(size_t)page,PROT_READ) || + mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary = memory+page; +` + checks.String() + ` + if (munmap(memory,(size_t)page*2)) return 1; + return 0; +} +` + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-subtract-fault", triple, text, main, prefix) + }) + } + t.Logf("%d dead-output memory programs at every mod16/NULL/RO guard; native original SSE and independent E4 reads at O0/O2, not AVX512 hardware", index) +} diff --git a/amd64_packed_subtract_forms.go b/amd64_packed_subtract_forms.go new file mode 100644 index 000000000..9c2eabdb4 --- /dev/null +++ b/amd64_packed_subtract_forms.go @@ -0,0 +1,35 @@ +package plan9asm + +import "strings" + +type x86PackedSubtractForm struct { + x86PackedBinaryForm + laneBits int + mode amd64PackedSubtractMode +} + +// Go's legacy subtract rows are yxm, never the add family's MMX rows. +// The V rows share _yvandnpd, but only D/Q enable broadcast in avx_optabs. +func x86PackedSubtractProperties(ins Instr, goarch string) (x86PackedSubtractForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + bits, mode, vector, known := amd64PackedSubtractProperties(base) + if !known { + return x86PackedSubtractForm{}, false, nil + } + if !vector { + form, _, err := x86LegacyIntegerArithmeticProperties(ins.Op, ins, goarch) + return x86PackedSubtractForm{form.x86PackedBinaryForm, bits, mode}, true, err + } + spec := x86PackedBinarySpec{encoding: x86PackedLogicalLegacyFP} + if vector { + spec.encoding, spec.zeroing = x86PackedLogicalVEXOrEVEX, true + if base == "VPSUBD" || base == "VPSUBQ" { + spec.broadcastBytes = bits / 8 + } + } + form, err := x86PackedBinaryProperties(ins, goarch, spec) + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + return x86PackedSubtractForm{form, bits, mode}, true, err +} diff --git a/amd64_packed_subtract_go_oracle_test.go b/amd64_packed_subtract_go_oracle_test.go new file mode 100644 index 000000000..a5b54e7d8 --- /dev/null +++ b/amd64_packed_subtract_go_oracle_test.go @@ -0,0 +1,133 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedSubtractOriginalGoBits(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle + +import "testing" + +func lane(p []byte, n int) uint64 { + var value uint64 + for j := 0; j < n; j++ { + value |= uint64(p[j]) << uint(8*j) + } + return value +} + +func difference(a, b uint64, n, mode int) uint64 { + if mode == 0 { + return b-a + } + if mode == 2 { + if b < a { + return 0 + } + return b-a + } + sign := int64(1) << uint(n*8-1) + left, right := int64(a), int64(b) + if a&uint64(sign) != 0 { + left -= sign*2 + } + if b&uint64(sign) != 0 { + right -= sign*2 + } + result := right-left + if result < -sign { + result = -sign + } + if result > sign-1 { + result = sign-1 + } + return uint64(result) +} + +func TestOriginalSubtract(t *testing.T) { + var patterns [8][16]byte + state := uint32(0x12345678) + for p := range patterns { + for i := range patterns[p] { + state = state*1664525+1013904223 + patterns[p][i] = byte(state>>24) + } + } + for i := range patterns[0] { + patterns[0][i] = 0xff + patterns[1][i] = 1 + patterns[2][i] = 0x80 + patterns[3][i] = 0 + } +`) + opcodes := []string{"PSUBB", "PSUBW", "PSUBL", "PSUBQ", "PSUBSB", "PSUBSW", "PSUBUSB", "PSUBUSW"} + for index, opcode := range opcodes { + bits, mode, _, _ := amd64PackedSubtractProperties(opcode) + name := fmt.Sprintf("packedsubtractgo%d", index) + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\n"+ + "STMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\n"+ + "MOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n"+ + "MOVOU (AX),X1\nMOVOU (BX),X0\nCMPQ AX,AX\nSTC\n"+ + "%s X1,X0\nMOVOU X0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\n"+ + "STMXCSR 18(CX)\nLDMXCSR saved-4(SP)\nRET\n", name, opcode) + fmt.Fprintf(&tests, ` + for _, a := range patterns { + for _, b := range patterns { + for csrMode := 0; csrMode < 4; csrMode++ { + var out [24]byte + for i := range out { + out[i] = 0xc3 + } + csr := uint32(0x1f80|csrMode<<13|csrMode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0] != 0xc3 || out[23] != 0xc3 || out[17] != 1 || out[18] != 0 { + t.Fatalf("%[1]s flags/canary %%x",out) + } + gotCSR := uint32(out[19]) | uint32(out[20])<<8 | uint32(out[21])<<16 | uint32(out[22])<<24 + if gotCSR != csr { + t.Fatalf("%[1]s CSR %%x != %%x",gotCSR,csr) + } + for i := 0; i < 16; i += %[2]d { + want := difference(lane(a[i:],%[2]d),lane(b[i:],%[2]d),%[2]d,%[3]d) + for j := 0; j < %[2]d; j++ { + if out[i+j+1] != byte(want >> uint(8*j)) { + t.Fatalf("%[1]s byte%%d got%%x want%%x",i+j,out,want) + } + } + } + } + } + } +`, name, bits/8, mode) + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go packed subtract oracle: %v\n%s", err, output) + } + t.Logf("%d actual Go SSE functions × 256 independent full-lane wrapping/saturation/MXCSR/flags/canary cases", len(opcodes)) +} diff --git a/amd64_packed_subtract_low_z_test.go b/amd64_packed_subtract_low_z_test.go new file mode 100644 index 000000000..e3edbcb08 --- /dev/null +++ b/amd64_packed_subtract_low_z_test.go @@ -0,0 +1,96 @@ +package plan9asm + +import ( + "fmt" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedSubtractLowZThreeAPIFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + var parameters []string + for _, prefix := range []string{"a", "b"} { + for index := 0; index < 8; index++ { + parameters = append(parameters, fmt.Sprintf("%s%d uint64", prefix, index)) + } + } + declaration := "func Value(" + strings.Join(parameters, ",") + ")" + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + for _, opcode := range []string{"VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW"} { + forms := []struct{ name, instruction string }{ + {"register", opcode + " Z0,Z1,Z2"}, + {"alias", opcode + " Z1,Z1,Z2"}, + {"memory", opcode + " a0+0(FP),Z1,Z2"}, + } + if opcode == "VPSUBD" || opcode == "VPSUBQ" { + forms = append(forms, struct{ name, instruction string }{ + "broadcast", opcode + ".BCST a0+0(FP),Z1,Z2", + }) + } + for _, form := range forms { + t.Run(target.triple+"/"+opcode+"/"+form.name, func(t *testing.T) { + // This fixture establishes form translation and LLVM objects, + // not an initialized numeric return or a caller-region grant. + pkg := mustGoPackage(t, "example.invalid/subtract", "package subtract\n"+declaration+"\n") + sig, err := DeriveGoFuncABI0Signature("Value", pkg.Types.Scope().Lookup("Value").(*types.Func), + target.arch, types.SizesFor("gc", target.arch)) + if err != nil { + t.Fatal(err) + } + body := "VMOVUPS a0+0(FP),Z0\nVMOVUPS b0+64(FP),Z1\n" + + form.instruction + "\nRET" + source := "TEXT ·Value(SB),4,$0-128\n" + body + "\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, target.arch, source, "", true) + options := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Value": sig}, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + before := fmt.Sprintf("%#v %#v", file, sig) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "text") + context := llvm.NewContext() + defer context.Dispose() + module, err := TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, module.String(), "module") + translated, err := translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: options.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, target.triple, translated.Module.String(), "typed-go") + if before != fmt.Sprintf("%#v %#v", file, sig) { + t.Fatal("translation mutated original typed inputs") + } + }) + } + } + } +} diff --git a/amd64_packed_subtract_mode32_test.go b/amd64_packed_subtract_mode32_test.go new file mode 100644 index 000000000..a689f48e1 --- /dev/null +++ b/amd64_packed_subtract_mode32_test.go @@ -0,0 +1,91 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86PackedSubtract386HighRegisterDoesNotAcquirePhysicalBinding(t *testing.T) { + for _, opcode := range []string{"VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW"} { + for _, bank := range []string{"X", "Y"} { + for position := 0; position < 3; position++ { + for _, high := range []int{16, 31} { + args := []string{bank + "0", bank + "1", bank + "2"} + args[position] = fmt.Sprintf("%s%d", bank, high) + row := opcode + " " + strings.Join(args, ",") + t.Run(strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + if _, recognized, err := x86PackedSubtractProperties(instruction, "386"); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go-accepted mode32 high bank became a physical-register permission: recognized=%v err=%v", recognized, err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = "386" + before := output.String() + if recognized, _, err := ctx.lowerPackedIntegerSubtract(instruction.Op, instruction); !recognized || + !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("unproved mode32 high bank emitted IR: recognized=%v err=%v", recognized, err) + } + }) + } + } + } + } +} + +func TestX86PackedSubtract386HighRegisterActualByteIdentities(t *testing.T) { + for _, opcode := range []string{"VPSUBB", "VPSUBW", "VPSUBD", "VPSUBQ", "VPSUBSB", "VPSUBSW", "VPSUBUSB", "VPSUBUSW"} { + for _, bank := range []string{"X", "Y"} { + for operand := 0; operand < 3; operand++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[operand] = bank + "16" + row := opcode + " " + strings.Join(args, ",") + t.Run(strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + code := x86PackedFirstGoInstructionBytes(t, "386", row) + ins, length, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if len(code) != 6 || !recognized || err != nil || length != 6 || ins.Op != Op(opcode) || len(ins.Args) != 3 { + t.Fatalf("actual EVEX byte identity %s/%x: %+v/%d/%t/%v", row, code, ins, length, recognized, err) + } + for index, argument := range args { + if ins.Args[index].Reg != Reg(argument) { + t.Fatalf("%s/%x operand%d: %s != %s", row, code, index, ins.Args[index].Reg, argument) + } + } + if _, _, recognized, err := decodedX86PackedIntegerArithmeticInstruction(code, 32); !recognized || err == nil { + t.Fatalf("raw mode32 high bank obtained false permission: %s/%x", row, code) + } + }) + } + } + } +} + +func TestX86PackedSubtract386BroadcastHighRegisterNeedsContext(t *testing.T) { + for _, opcode := range []string{"VPSUBD", "VPSUBQ"} { + for _, bank := range []string{"X", "Y"} { + for position := 1; position < 3; position++ { + args := []string{"(AX)", bank + "1", bank + "2"} + args[position] = bank + "16" + row := opcode + ".BCST " + strings.Join(args, ",") + t.Run(strings.ReplaceAll(row, " ", "_"), func(t *testing.T) { + source := "TEXT f(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, "386", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, known, err := x86PackedSubtractProperties(file.Funcs[0].Instrs[1], "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("broadcast high bank became a mode32 binding: %v", err) + } + }) + } + } + } +} diff --git a/amd64_packed_subtract_runtime_test.go b/amd64_packed_subtract_runtime_test.go new file mode 100644 index 000000000..75a1e1070 --- /dev/null +++ b/amd64_packed_subtract_runtime_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Only legacy SSE is executed as native instruction evidence. VEX/EVEX +// numerical expectations come from independent scalar lane arithmetic. +const x86PackedSubtractIndependentOracle = ` +static const uint64_t subtractMasks[] = { + 0, 1, UINT64_C(0x55aa55aa55aa55aa), UINT64_MAX +}; + +static uint64_t subtractLane(const uint8_t *p, int bytes) { + uint64_t value = 0; + for (int j = 0; j < bytes; ++j) { + value |= (uint64_t)p[j] << (8*j); + } + return value; +} + +static uint64_t subtractValue(uint64_t a, uint64_t b, int bytes, int mode) { + if (mode == 0) { + return b-a; + } + if (mode == 2) { + return b < a ? 0 : b-a; + } + int bits = bytes*8; + int64_t sign = INT64_C(1) << (bits-1); + int64_t left = (a & sign) ? (int64_t)a - (sign*2) : (int64_t)a; + int64_t right = (b & sign) ? (int64_t)b - (sign*2) : (int64_t)b; + int64_t result = right-left; + if (result < -sign) { + result = -sign; + } + if (result > sign-1) { + result = sign-1; + } + return (uint64_t)result; +} + +__attribute__((noinline)) static int checkSubtract( + void (*fn)(uint8_t*, const uint8_t*, const uint8_t*, uint64_t), + const char *name, int width, int bytes, int arithmetic, + int maskMode, int legacy, int memory, int broadcast) { + for (int round = 0; round < 12; ++round) { + for (int m = 0; m < 4; ++m) { + _Alignas(64) uint8_t aStorage[96], bStorage[96], output[68]; + int shift = legacy && memory ? 16 : 1; + uint8_t *a = aStorage+shift; + uint8_t *b = bStorage+3; + memset(aStorage, 0xa7, sizeof(aStorage)); + memset(bStorage, 0xb9, sizeof(bStorage)); + memset(output, 0xc3, sizeof(output)); + for (int i = 0; i < 64; ++i) { + a[i] = (uint8_t)(i*17+round*31); + b[i] = (uint8_t)(i*23+round*53); + } + if (round < 4) { + memset(a, round == 0 ? 0xff : round == 1 ? 0x80 : round == 2 ? 1 : 0, 64); + memset(b, round == 0 ? 1 : round == 1 ? 0x7f : round == 2 ? 0 : 0xff, 64); + } + uint64_t mask = subtractMasks[m]; + uint32_t saved, before = 0x1f80 | ((round%4)<<13) | round, after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + fn(output+1, a, b, mask); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if (after != before || output[0] != 0xc3 || output[67] != 0xc3 || + output[65] != 1 || output[66] != 0) { + fprintf(stderr, "%s changed MXCSR, flags or output canaries\n", name); + return 1; + } + for (int i = 0; i < 64; i += bytes) { + uint64_t result = subtractValue(subtractLane(a+(broadcast ? 0 : i), bytes), + subtractLane(b+i, bytes), bytes, arithmetic); + int active = maskMode == 0 || ((mask >> (i/bytes)) & 1); + for (int j = 0; j < bytes; ++j) { + uint8_t want = i < width ? + (active ? (uint8_t)(result >> (8*j)) : maskMode == 2 ? 0 : b[i+j]) : + legacy ? b[i+j] : 0; + if (output[i+j+1] != want) { + fprintf(stderr, "%s round=%d mask=%llx byte=%d got=%x want=%x\n", + name, round, (unsigned long long)mask, i+j, output[i+j+1], want); + return 1; + } + } + } + for (int i = 0; i < 96; ++i) { + if ((i < shift || i >= shift+64) && aStorage[i] != 0xa7) { + return 1; + } + if ((i < 3 || i >= 67) && bStorage[i] != 0xb9) { + return 1; + } + } + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedSubtractCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var source, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + rows := x86PackedSubtractCallerRows() + for index, row := range rows { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + instruction := file.Funcs[0].Instrs[1] + form, _, err := x86PackedSubtractProperties(instruction, "amd64") + if err != nil { + t.Fatal(err) + } + mode := 0 + if form.mask >= 0 { + mode = 1 + if form.zeroing { + mode = 2 + } + } + name := fmt.Sprintf("packedsubtract%d", index) + row = strings.ReplaceAll(row, "(DI)", "(BX)") + register, valid := amd64VectorRegisterIndex(instruction.Args[form.destination].Reg, form.byteWidth) + if !valid { + t.Fatal("missing actual vector destination") + } + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n"+ + "MOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\n"+ + "VMOVUPS (BX),Z0\nVMOVUPS (BX),Z20\n"+ + "VMOVUPS (CX),Z1\nVMOVUPS (CX),Z2\nVMOVUPS (CX),Z15\n"+ + "VMOVUPS (CX),Z21\nVMOVUPS (CX),Z22\nKMOVQ DX,K1\n"+ + "CMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", + name, row, register) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, "if(checkSubtract(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d)) return 1;\n", + name, name, form.byteWidth, form.laneBits/8, form.mode, mode, + x86FloatAddSubBoolInt(form.legacy), x86FloatAddSubBoolInt(form.memory >= 0), + x86FloatAddSubBoolInt(form.broadcast)) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := "#include \n#include \n#include \n" + + declarations.String() + x86PackedSubtractIndependentOracle + + "int main(void){\n" + checks.String() + "return 0;}\n" + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed-subtract-complete", triple, text, main, prefix) + }) + } + t.Logf("%d programs × 48 independent wrapping/saturating/mask/upper-view/flags/MXCSR/canary cases at O0/O2; not native AVX512 execution", len(rows)) +} diff --git a/amd64_packed_unpack_fault_runtime_test.go b/amd64_packed_unpack_fault_runtime_test.go new file mode 100644 index 000000000..48d9e7c4c --- /dev/null +++ b/amd64_packed_unpack_fault_runtime_test.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackZeroMaskStillFaults(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, calls strings.Builder + sigs := map[string]FuncSig{} + for index, row := range []struct { + instruction string + bytes int + }{ + {"VPUNPCKLBW.Z (BX),Z0,K1,Z1", 64}, + {"VPUNPCKHWD.Z (BX),Y0,K1,Y1", 32}, + {"VUNPCKHPS.BCST.Z (BX),X0,K1,X1", 4}, + {"VUNPCKHPD.BCST.Z (BX),Z0,K1,Z1", 8}, + } { + name := fmt.Sprintf("unpackfault%d", index) + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-16\nMOVQ input+0(FP),BX\nMOVQ out+8(FP),AX\nVPXORD Z0,Z0,Z0\nMOVL $0,CX\nKMOVQ CX,K1\n%s\nVMOVUPS Z1,(AX)\nRET\n", name, row.instruction) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(const uint8_t*,uint8_t*);\n", name) + fmt.Fprintf(&calls, "if(check_fault(%s,0) || check_fault(%s,boundary-%d+1)) return 1;\n", name, name, row.bytes) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +static int check_fault(void (*program)(const uint8_t*,uint8_t*),const uint8_t *input) { + pid_t child=fork(); + if(child<0) return 1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + uint8_t output[64]; + program(input,output); + _exit(0); + } + int status=0; + if(waitpid(child,&status,0)!=child) return 1; + if(!WIFSIGNALED(status) || (WTERMSIG(status)!=SIGSEGV && WTERMSIG(status)!=SIGBUS)) { + fprintf(stderr,"E4NF zero-mask input %p failed to fault: status=%d\n",input,status); + return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<=0) return 1; + uint8_t *memory=mmap(0,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED || mprotect(memory+page,(size_t)page,PROT_NONE)) return 1; + uint8_t *boundary=memory+page; +` + calls.String() + ` + if(munmap(memory,(size_t)page*2)) return 1; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "unpack-e4nf-fault", triple, ir, main, prefix) +} diff --git a/amd64_packed_unpack_forms.go b/amd64_packed_unpack_forms.go new file mode 100644 index 000000000..9c4e4f3e9 --- /dev/null +++ b/amd64_packed_unpack_forms.go @@ -0,0 +1,75 @@ +package plan9asm + +import "strings" + +type x86PackedUnpackSpec struct { + laneBits int + high bool + grammar x86PackedBinarySpec +} + +// Go's six ymm rows include MMX. Its other six legacy rows use yxm. +// All twelve vector rows use _yvandnpd, but only D/Q/PS/PD permit BCST. +// Their EVEX memory exception class is E4NF: masking does not suppress faults. +var x86PackedUnpackSpecs = map[Op]x86PackedUnpackSpec{ + "PUNPCKLBW": {8, false, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKHBW": {8, true, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKLWL": {16, false, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKHWL": {16, true, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKLLQ": {32, false, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKHLQ": {32, true, x86PackedBinarySpec{x86PackedLogicalLegacyInteger, 0, false}}, + "PUNPCKLQDQ": {64, false, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "PUNPCKHQDQ": {64, true, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKLPS": {32, false, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKHPS": {32, true, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKLPD": {64, false, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "UNPCKHPD": {64, true, x86PackedBinarySpec{x86PackedLogicalLegacyFP, 0, false}}, + "VPUNPCKLBW": {8, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKHBW": {8, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKLWD": {16, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKHWD": {16, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 0, true}}, + "VPUNPCKLDQ": {32, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VPUNPCKHDQ": {32, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VPUNPCKLQDQ": {64, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, + "VPUNPCKHQDQ": {64, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, + "VUNPCKLPS": {32, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VUNPCKHPS": {32, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 4, true}}, + "VUNPCKLPD": {64, false, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, + "VUNPCKHPD": {64, true, x86PackedBinarySpec{x86PackedLogicalVEXOrEVEX, 8, true}}, +} + +type x86PackedUnpackForm struct { + x86PackedBinaryForm + spec x86PackedUnpackSpec + source Operand +} + +func x86PackedUnpackProperties(ins Instr, goarch string) (x86PackedUnpackForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86PackedUnpackSpecs[Op(base)] + if !recognized { + return x86PackedUnpackForm{}, false, nil + } + local := ins + if spec.grammar.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(ins, goarch) + if err != nil { + return x86PackedUnpackForm{}, true, err + } + } + form, err := x86PackedBinaryProperties(local, goarch, spec.grammar) + // Intel's three MMX low-unpack rows are mm/m32, unlike the high + // rows' mm/m64. The unused high source half is not a memory access. + if err == nil && form.mmx && form.memory >= 0 && !spec.high { + form.memoryBytes = 4 + } + if err == nil { + err = x86Mode32VectorRegisterContext(ins, goarch) + } + result := x86PackedUnpackForm{x86PackedBinaryForm: form, spec: spec} + if err == nil { + result.source = local.Args[form.first] + } + return result, true, err +} diff --git a/amd64_packed_unpack_forms_test.go b/amd64_packed_unpack_forms_test.go new file mode 100644 index 000000000..6a4208007 --- /dev/null +++ b/amd64_packed_unpack_forms_test.go @@ -0,0 +1,372 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +func x86UnpackTestLaneBytes(opcode string) int { + switch { + case strings.HasSuffix(opcode, "BW"): + return 1 + case strings.HasSuffix(opcode, "WL"), strings.HasSuffix(opcode, "WD"): + return 2 + case strings.HasSuffix(opcode, "LQ"), strings.HasSuffix(opcode, "DQ") && !strings.HasSuffix(opcode, "QDQ"), strings.HasSuffix(opcode, "PS"): + return 4 + default: + return 8 + } +} + +func TestX86PackedUnpack386EVEXHighRegistersActualBytesNeedContext(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + if !strings.HasPrefix(opcode, "V") { + continue + } + decode := decodedX86FloatingUnpackInstruction + if strings.HasPrefix(opcode, "VPUNPCK") { + decode = decodedX86EVEXPackedUnpackInstruction + } + for _, bank := range []string{"X", "Y"} { + for _, high := range []int{16, 31} { + for index := 0; index < 3; index++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[index] = fmt.Sprintf("%s%d", bank, high) + row := opcode + " " + strings.Join(args, ",") + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual Go mode32 object: %v\n%s", err, listing) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 7 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + for _, field := range fields[1:7] { + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + t.Fatalf("actual Go object byte %q: %v", field, err) + } + code = append(code, byte(value)) + } + break + } + if len(code) != 6 || code[0] != 0x62 { + t.Fatalf("actual Go lost EVEX encoding: %s\n%s", row, listing) + } + ins, length, recognized, err := decode(code, 64) + if !recognized || err != nil || length != len(code) || ins.Op != Op(opcode) || len(ins.Args) != 3 { + t.Fatalf("mode64 actual byte identity: %s/%x: %+v/%d/%t/%v", row, code, ins, length, recognized, err) + } + for index, argument := range args { + if ins.Args[index].Reg != Reg(argument) { + t.Fatalf("actual Go %x decoded argument %d as %s want %s", code, index, ins.Args[index].Reg, argument) + } + } + if _, _, recognized, err := decode(code, 32); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s/%x invented raw mode32 high registers: %t/%v", row, code, recognized, err) + } + } + } + } + } +} + +func TestX86PackedUnpack386NamedHighRegistersKeepPhysicalContext(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + if !strings.HasPrefix(opcode, "V") { + continue + } + for _, bank := range []string{"X", "Y"} { + for _, high := range []int{8, 15, 16, 31} { + for index := 0; index < 3; index++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[index] = fmt.Sprintf("%s%d", bank, high) + row := opcode + " " + strings.Join(args, ",") + // This invokes the actual Go tool on both compatibility + // versions. Syntax acceptance alone is not a native binding. + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil || !strings.Contains(listing, row[:len(opcode)]) { + t.Fatalf("actual Go lost accepted mode32 source %s: %v\n%s", row, err, listing) + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s named high register acquired a physical mode32 binding: %v", row, err) + } + } + } + } + } + } +} + +func x86UnpackTestHasMMX(opcode string) bool { + return strings.HasPrefix(opcode, "PUNPCK") && !strings.HasSuffix(opcode, "QDQ") +} + +func x86UnpackAcceptedForms(opcode, arch string) []string { + if !strings.HasPrefix(opcode, "V") { + last := 15 + if arch == "386" { + last = 7 + } + rows := []string{fmt.Sprintf("%s X0,X%d", opcode, last), fmt.Sprintf("%s 8(AX),X%d", opcode, last)} + if arch == "amd64" && x86UnpackTestHasMMX(opcode) { + rows = append(rows, opcode+" M0,M7", opcode+" 8(AX),M7") + } + return rows + } + var rows []string + for _, bank := range []string{"X", "Y", "Z"} { + last := 31 + if arch == "386" { + last = 7 + } + for _, first := range []string{bank + "0", "8(AX)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s%d", opcode, first, bank, bank, last)) + if arch == "amd64" { + rows = append(rows, + fmt.Sprintf("%s %s,%s1,K7,%s%d", opcode, first, bank, bank, last), + fmt.Sprintf("%s.Z %s,%s1,K1,%s%d", opcode, first, bank, bank, last)) + } + } + if x86UnpackTestLaneBytes(opcode) >= 4 { + rows = append(rows, fmt.Sprintf("%s.BCST 8(AX),%s1,%s%d", opcode, bank, bank, last)) + if arch == "amd64" { + rows = append(rows, + fmt.Sprintf("%s.BCST 8(AX),%s1,K7,%s%d", opcode, bank, bank, last), + fmt.Sprintf("%s.BCST.Z 8(AX),%s1,K1,%s%d", opcode, bank, bank, last)) + } + } + } + return rows +} + +func TestX86PackedUnpackSpecsBindEveryGoOpcodeAttribute(t *testing.T) { + if len(x86PackedUnpackSpecs) != len(x86CallerUnpackOpcodes()) { + t.Fatal("incomplete interleave family") + } + dir := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86") + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, opcode := range x86CallerUnpackOpcodes() { + spec, ok := x86PackedUnpackSpecs[Op(opcode)] + if !ok || spec.laneBits != 8*x86UnpackTestLaneBytes(opcode) { + t.Fatalf("missing or wrong source-defined lane width: %s", opcode) + } + if !strings.HasPrefix(opcode, "V") { + start := strings.Index(string(legacy), "{A"+opcode+", ") + if start < 0 { + t.Fatalf("actual Go omits %s", opcode) + } + line := strings.SplitN(string(legacy)[start:], "\n", 2)[0] + table := "yxm" + if x86UnpackTestHasMMX(opcode) { + table = "ymm" + } + if !strings.Contains(line, ", "+table+",") { + t.Fatalf("actual Go table differs for %s: %s", opcode, line) + } + continue + } + prefix := "{as: A" + opcode + ", ytab: _yvandnpd," + start := strings.Index(string(avx), prefix) + if start < 0 { + t.Fatalf("actual Go omits %s family table", opcode) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if strings.Count(row, "avxEscape | vex") != 2 || strings.Count(row, "avxEscape | evex") != 3 { + t.Fatalf("actual Go encoding generations changed: %s", row) + } + for _, line := range strings.Split(row, "\n") { + if !strings.Contains(line, "avxEscape | evex") { + continue + } + broadcast := "" + if x86UnpackTestLaneBytes(opcode) >= 4 { + broadcast = fmt.Sprintf("evexBcstN%d", x86UnpackTestLaneBytes(opcode)) + } + if !strings.Contains(line, "evexZeroingEnabled") || strings.Contains(line, "evexSae") || strings.Contains(line, "evexRounding") || + broadcast == "" && strings.Contains(line, "evexBcst") || broadcast != "" && !strings.Contains(line, broadcast) { + t.Fatalf("actual opcode attributes differ: %s", line) + } + } + } +} + +func TestX86PackedUnpackCompleteGoRowsAndFiveLLVMObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT unpackforms(SB),4,$0-0\n") + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + for _, row := range x86UnpackAcceptedForms(opcode, target.arch) { + count++ + fmt.Fprintln(&source, row) + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, ins := range file.Funcs[0].Instrs[1 : len(file.Funcs[0].Instrs)-1] { + if _, recognized, err := x86PackedUnpackProperties(ins, target.arch); !recognized || err != nil { + t.Fatalf("actual Go accepted %s but grammar rejected: %v", ins.Raw, err) + } + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"unpackforms": {Name: "unpackforms", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "unpack-forms.ll", "unpack-forms.o", ir) + t.Logf("%d actual Go forms in one LLVM22 target object", count) + }) + } +} + +func TestX86PackedUnpackDispatchIdentityBeforeEmission(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\nVPUNPCKLBW X0,X1,X2\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, opcode := range []Op{"VPUNPCKLBW", "vpunpcklbw"} { + for _, sourceOp := range []Op{"", "VPUNPCKLBW", "vpunpcklbw", "VPUNPCKHBW", "UNKNOWN"} { + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + ins := file.Funcs[0].Instrs[1] + ins.Op = sourceOp + _, _, err := ctx.lowerPackedUnpack(opcode, ins) + wantError := sourceOp == "VPUNPCKHBW" || sourceOp == "UNKNOWN" + if (err != nil) != wantError || wantError && output.String() != before || !wantError && output.String() == before { + t.Fatalf("dispatch identity %s/%s: %v, %d emitted bytes", opcode, sourceOp, err, output.Len()) + } + } + } +} + +func TestX86PackedUnpackRejectsMalformedGoFormsBeforeEmission(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, opcode := range x86CallerUnpackOpcodes() { + rows := []string{opcode + " X0,(AX)", opcode + ".SAE X0,X1"} + if strings.HasPrefix(opcode, "V") { + rows = []string{ + opcode + " X0,X1,(AX)", + opcode + " X0,Y1,Y2", + opcode + " X0,X1,K0,X2", + opcode + ".Z X0,X1,X2", + opcode + ".SAE X0,X1,X2", + opcode + ".BCST X0,X1,X2", + } + if x86UnpackTestLaneBytes(opcode) < 4 { + rows = append(rows, opcode+".BCST (AX),X1,X2") + } + } else { + rows = append(rows, opcode+" X0,Y1") + if arch == "386" { + rows = append(rows, opcode+" X8,X1") + } + } + for _, row := range rows { + t.Run(arch+"/"+row, func(t *testing.T) { + source := "TEXT bad(SB),4,$0-0\n" + row + "\nRET\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + return + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.String() + ins := file.Funcs[0].Instrs[1] + known, _, err := ctx.lowerPackedUnpack(ins.Op, ins) + if !known || err == nil || output.String() != before { + t.Fatalf("malformed source emitted IR: known=%t err=%v", known, err) + } + }) + } + } + } +} + +func TestX86PackedUnpackMMXMemoryNativeExtent(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + if !x86UnpackTestHasMMX(opcode) { + continue + } + source := "TEXT f(SB),4,$0-0\n" + opcode + " (AX),M0\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + form, recognized, err := x86PackedUnpackProperties(file.Funcs[0].Instrs[1], "amd64") + want := 4 + if strings.Contains(opcode, "UNPCKH") { + want = 8 + } + if !recognized || err != nil || !form.mmx || form.memoryBytes != want { + t.Fatalf("%s native memory extent=%d want=%d: %v", opcode, form.memoryBytes, want, err) + } + } +} + +func TestX86PackedUnpackTypedFPMemoryExtentBeforeEmission(t *testing.T) { + for _, opcode := range x86CallerUnpackOpcodes() { + for _, row := range x86UnpackAcceptedForms(opcode, "amd64") { + if !strings.Contains(row, "8(AX)") { + continue + } + row = strings.Replace(row, "8(AX)", "a+0(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedUnpackProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for _, complete := range []bool{false, true} { + sig := FuncSig{Ret: Void} + for offset := 0; offset < form.memoryBytes; offset += 4 { + if !complete && offset+4 == form.memoryBytes { + break + } + index := len(sig.Args) + sig.Args = append(sig.Args, I32) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(offset), Type: I32, Index: index, Field: -1}) + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + known, _, err := ctx.lowerPackedUnpack(ins.Op, ins) + if !known || complete && err != nil || !complete && (err == nil || output.String() != before) { + t.Fatalf("%s complete=%t native bytes=%d: %v", row, complete, form.memoryBytes, err) + } + } + } + } +} diff --git a/amd64_packed_unpack_go_oracle_test.go b/amd64_packed_unpack_go_oracle_test.go new file mode 100644 index 000000000..2e90d3993 --- /dev/null +++ b/amd64_packed_unpack_go_oracle_test.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackOriginalGoBitOracle(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\n") + tests.WriteString(`package oracle +import "testing" +func TestOriginalGoInterleaveBits(t *testing.T) { + patterns:=[][16]byte{ + {0x45,0x23,0x81,0x7f,0x78,0x56,0xc4,0xff,0xbc,0x9a,0x78,0x56,0x34,0x12,0xf0,0x7f}, + {0x54,0x76,0x98,0xba,0xdc,0xfe,0xf8,0xff,0x45,0x23,0xc1,0x7f,0,0,0,0x80}, + {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}, + {0xff,0xfe,0xfd,0xfc,0xfb,0xfa,0xf9,0xf8,0xf7,0xf6,0xf5,0xf4,0xf3,0xf2,0xf1,0xf0}, + } +`) + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + if strings.HasPrefix(opcode, "V") { + continue // Native VEX/EVEX execution needs separate ISA witnesses. + } + banks := []string{"X"} + if x86UnpackTestHasMMX(opcode) { + banks = append(banks, "M") + } + for _, bank := range banks { + forms := []string{"register"} + if bank == "M" { + forms = append(forms, "memory") + } + for _, form := range forms { + name := fmt.Sprintf("unpack%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte,csr uint32)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$8-28\nSTMXCSR saved-4(SP)\nLDMXCSR csr+24(FP)\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\n", name) + move, bytes := "MOVOU", 16 + if bank == "M" { + move, bytes = "MOVQ", 8 + } + fmt.Fprintf(&assembly, "%s (BX),%s0\n", move, bank) + first := bank + "1" + if form == "register" { + fmt.Fprintf(&assembly, "%s (AX),%s1\n", move, bank) + } else { + first = "(AX)" + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\n%s %s,%s0\n%s %s0,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nSTMXCSR 18(CX)\n", opcode, first, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("LDMXCSR saved-4(SP)\nRET\n") + high := 0 + if strings.Contains(opcode, "UNPCKH") { + high = bytes / 2 + } + fmt.Fprintf(&tests, ` + for _,a:=range patterns { for _,b:=range patterns { for mode:=0;mode<4;mode++ { + var out [24]byte + for i:=range out {out[i]=0xc3} + csr:=uint32(0x1f80|mode<<13|mode) + %[1]s(&a[0],&b[0],&out[1],csr) + if out[0]!=0xc3 || out[23]!=0xc3 || out[17]!=1 || out[18]!=0 {t.Fatalf("%[1]s canary/flags: %%x",out)} + gotCSR:=uint32(out[19])|uint32(out[20])<<8|uint32(out[21])<<16|uint32(out[22])<<24 + if gotCSR!=csr {t.Fatalf("%[1]s changed MXCSR %%x want %%x",gotCSR,csr)} + for i:=0;i<16;i++ { + want:=byte(0xc3) + if i<%[2]d { + source:=%[3]d+(i/(2*%[4]d))*%[4]d+i%%%[4]d + want=b[source] + if i/%[4]d%%2!=0 {want=a[source]} + } + if out[i+1]!=want {t.Fatalf("%[1]s byte %%d got %%x want %%x",i,out[i+1],want)} + } + } } } +`, name, bytes, high, x86UnpackTestLaneBytes(opcode)) + } + } + } + tests.WriteString("}\n") + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", + "value.go": declarations.String(), "value_amd64.s": assembly.String(), "value_test.go": tests.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go interleave bit oracle: %v\n%s", err, output) + } + t.Logf("%d original SSE/MMX Go functions; 64 independent bitwise/MXCSR/flag/canary inputs each", count) +} diff --git a/amd64_packed_unpack_mmx_runtime_test.go b/amd64_packed_unpack_mmx_runtime_test.go new file mode 100644 index 000000000..dd7fa32df --- /dev/null +++ b/amd64_packed_unpack_mmx_runtime_test.go @@ -0,0 +1,108 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackMMXNativeMemoryAndFP(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, calls strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + if !x86UnpackTestHasMMX(opcode) { + continue + } + name, fpName := fmt.Sprintf("unpackmmx%d", count), fmt.Sprintf("unpackmmxfp%d", count) + count++ + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-24\nMOVQ input+0(FP),AX\nMOVQ second+8(FP),M0\n%s (AX),M0\nMOVQ M0,ret+16(FP)\nEMMS\nRET\n", name, opcode) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, I64}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: I64, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: 16, Type: I64, Index: 0, Field: -1}}, + }} + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-16\nMOVQ $0,AX\nMOVQ AX,M0\n%s a+0(FP),M0\nMOVQ M0,ret+8(FP)\nEMMS\nRET\n", fpName, opcode) + sigs[fpName] = FuncSig{Name: fpName, Args: []LLVMType{I32, I32}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}, {Offset: 4, Type: I32, Index: 1, Field: -1}}, + Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }} + fmt.Fprintf(&declarations, "extern uint64_t %s(const uint8_t*,uint64_t);\nextern uint64_t %s(uint32_t,uint32_t);\n", name, fpName) + high, extent := 0, 4 + if strings.Contains(opcode, "UNPCKH") { + high, extent = 4, 8 + } + fmt.Fprintf(&calls, "if(checkMMX(%s,%s,%d,%d,%d,begin,end)) return 1;\n", name, fpName, x86UnpackTestLaneBytes(opcode), high, extent) + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +#include +#include +#include +#include +` + declarations.String() + ` +static uint64_t expectedMMX(const uint8_t *first,uint64_t second,int lane,int high) { + uint8_t old[8],out[8]; + memcpy(old,&second,8); + for(int i=0;i<8;++i) { + int source=high+(i/(2*lane))*lane+i%lane; + out[i]=(i/lane)%2 ? first[source] : old[source]; + } + uint64_t value; + memcpy(&value,out,8); + return value; +} +static int checkMMX(uint64_t (*program)(const uint8_t*,uint64_t), + uint64_t (*fp)(uint32_t,uint32_t),int lane,int high,int extent,uint8_t *begin,uint8_t *end) { + for(int seed=0;seed<16;++seed) { + uint8_t *first=end-extent,source[8]; + for(int i=0;i<8;++i) source[i]=(uint8_t)(seed*73+i*19+7); + memcpy(first,source,extent); + uint64_t second=UINT64_C(0xcafe11223344ba5e)^((uint64_t)seed*UINT64_C(0x1234567890)); + if(program(first,second)!=expectedMMX(source,second,lane,high) || memcmp(first,source,extent)) return 1; + uint32_t a,b; + memcpy(&a,source,4); + memcpy(&b,source+4,4); + if(fp(a,b)!=expectedMMX(source,0,lane,high)) return 1; + } + const uint8_t *bad[]={end-extent+1,high ? begin-4 : end-3}; + for(int i=0;i<2;++i) { + pid_t child=fork(); + if(child<0) return 1; + if(child==0) { + struct rlimit limit={0,0}; + if(setrlimit(RLIMIT_CORE,&limit)) _exit(2); + volatile uint64_t observed=program(bad[i],UINT64_C(0xabcdef1234567890)); + (void)observed; + _exit(0); + } + int status=0; + if(waitpid(child,&status,0)!=child || !WIFSIGNALED(status) || + (WTERMSIG(status)!=SIGSEGV && WTERMSIG(status)!=SIGBUS)) return 1; + } + return 0; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<=0) return 1; + uint8_t *memory=mmap(0,(size_t)page*3,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(memory==MAP_FAILED || mprotect(memory,(size_t)page,PROT_NONE) || + mprotect(memory+2*page,(size_t)page,PROT_NONE)) return 1; + uint8_t *begin=memory+page,*end=memory+2*page; +` + calls.String() + ` + if(munmap(memory,(size_t)page*3)) return 1; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "unpack-mmx-native-memory-fp", triple, ir, main, prefix) +} diff --git a/amd64_packed_unpack_runtime_test.go b/amd64_packed_unpack_runtime_test.go new file mode 100644 index 000000000..230412238 --- /dev/null +++ b/amd64_packed_unpack_runtime_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// One independent reference is shared by all format programs. Keeping this +// out of main avoids compiling hundreds of copies of the same canary loops. +const x86UnpackBitwiseRuntimeOracle = ` +static uint8_t patterns[4][64]; +static const uint64_t masks[]={0,1,UINT64_C(0x55aa55aa55aa55aa),UINT64_MAX}; +__attribute__((noinline)) static int checkUnpack( + void (*program)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t), + const char *name,int width,int lane,int high,int mode,int broadcast,int legacy,int destination,int shift) { + for(int round=0;round<4;++round) for(int m=0;m<4;++m) { + _Alignas(64) uint8_t aStorage[96],bStorage[96],output[68]; + uint8_t *a=aStorage+shift,*b=bStorage+3; + memset(aStorage,0xa7,sizeof(aStorage)); + memset(bStorage,0xb9,sizeof(bStorage)); + memset(output,0xc3,sizeof(output)); + memcpy(a,patterns[round],64); + memcpy(b,patterns[(round+1)%4],64); + uint64_t mask=masks[m]; + uint32_t saved,before=0x1f80|(round<<13)|round,after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + program(output+1,a,b,mask); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if(after!=before || output[0]!=0xc3 || output[67]!=0xc3 || output[65]!=1 || output[66]!=0) { + fprintf(stderr,"%s round=%d mask=%llx CSR/flags/canary failure\n",name,round,(unsigned long long)mask); + return 1; + } + for(int i=0;i<64;++i) { + int source=(i/16)*16+high+((i%16)/(2*lane))*lane+i%lane; + uint8_t selected=(i/lane)%2 ? a[broadcast ? source%lane : source] : b[source]; + const uint8_t *old=destination==1 ? a : b; + int active=mode==0 || ((mask>>(i/lane))&1); + uint8_t want=i=shift+64) && aStorage[i]!=0xa7) return 1; + if((i<3 || i>=67) && bStorage[i]!=0xb9) return 1; + } + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedUnpackCompleteSemantics(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, opcode := range x86CallerUnpackOpcodes() { + legacy := !strings.HasPrefix(opcode, "V") + widths := []int{16} + if !legacy { + widths = []int{16, 32, 64} + } + lane := x86UnpackTestLaneBytes(opcode) + for _, width := range widths { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, memory := range []string{"register", "memory", "broadcast"} { + if memory == "broadcast" && (legacy || lane < 4) { + continue + } + for _, mode := range []string{"unmasked", "merge", "zero"} { + if legacy && mode != "unmasked" { + continue + } + for _, destination := range []int{0, 1, 2} { + if legacy && destination != 0 { + continue + } + name := fmt.Sprintf("unpack%d", count) + count++ + first := bank + "1" + if memory != "register" { + first = "(BX)" + } + operation := opcode + if memory == "broadcast" { + operation += ".BCST" + } + if mode == "zero" { + operation += ".Z" + } + row := operation + " " + first + "," + bank + "0" + if !legacy { + if mode != "unmasked" { + row += ",K1" + } + row += fmt.Sprintf(",%s%d", bank, destination) + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (CX),Z2\nKMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", name, row, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + high := 0 + if strings.Contains(opcode, "UNPCKH") { + high = 8 + } + // Legacy m128 requires aligned storage. All other forms + // exercise unaligned input/output pointers and canaries. + shift := 1 + if legacy && memory == "memory" { + shift = 16 + } + modeValue := map[string]int{"unmasked": 0, "merge": 1, "zero": 2}[mode] + fmt.Fprintf(&checks, "if(checkUnpack(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d,%d)) return 1;\n", name, name, + width, lane, high, modeValue, x86FloatAddSubBoolInt(memory == "broadcast"), x86FloatAddSubBoolInt(legacy), destination, shift) + } + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + main := `#include +#include +#include +` + declarations.String() + x86UnpackBitwiseRuntimeOracle + ` +int main(void) { + for(int p=0;p<4;++p) for(int i=0;i<64;++i) patterns[p][i]=(uint8_t)(i*19+p*83+7); + // These include float32/float64 signaling/quiet NaN payloads. An unpack + // must copy their exact bits and leave even pre-existing MXCSR flags alone. + const uint64_t specials[]={UINT64_C(0x7ff0123456789abc),UINT64_C(0xfff8fedcba987654),UINT64_C(0x7f812345ffc45678),UINT64_C(0x800000007fc12345)}; + memcpy(patterns[2],specials,sizeof(specials)); + memcpy(patterns[3]+32,specials,sizeof(specials)); +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "unpack-complete", triple, ir, main, prefix) + t.Logf("%d opcode/width/memory/mask/alias programs, 16 independent bitwise/MXCSR/flag/canary inputs each", count) +} diff --git a/amd64_packed_vector_move_aligned.go b/amd64_packed_vector_move_aligned.go new file mode 100644 index 000000000..849cf9fb9 --- /dev/null +++ b/amd64_packed_vector_move_aligned.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Alignment faults are part of the original instruction, including its exact +// mask and zeroing behavior. Neither LLVM align metadata nor a masked load can +// replace this operation: the former asserts alignment instead of checking it, +// and the latter can remove a dead read. Pass an opaque address in a register, +// without granting the optimizer any pointee, extent or alignment assumption. +// Intel SDM Vol. 2A Table 2-46 keeps E1 alignment separate from page-fault +// suppression; the runtime oracle compares both against original Go opcodes. +func (c *amd64Ctx) lowerAlignedPackedMoveMemory(ins Instr, form amd64PackedVectorMoveForm) error { + pointer, pointerType, _, err := c.packedVectorMemoryPointer(ins.Args[form.memory]) + if err != nil { + return err + } + segment := "" + if pointerType == "ptr addrspace(256)" { + segment = "%gs:" + } else if pointerType == "ptr addrspace(257)" { + segment = "%fs:" + } + name, _, _ := strings.Cut(strings.ToLower(string(ins.Op)), ".") + constraint, prefix := "x", "" + if form.nativeEVEX(ins) { + constraint = "v" + // Retain EVEX for high source registers even if LLVM allocates a low + // carrier. APD/APS otherwise also have a VEX encoding at X/Y widths. + prefix = "{evex} " + } + vectorType := fmt.Sprintf("<%d x float>", form.byteWidth/4) + vector := func(operand Operand) (string, error) { + value, err := c.loadPackedCompareBytes(operand, form.byteWidth) + if err != nil { + return "", err + } + carrier := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast <%d x i8> %s to %s\n", carrier, form.byteWidth, value, vectorType) + return "%" + carrier, nil + } + mask := "" + if form.mask >= 0 { + value, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + narrow := c.newTmp() + fmt.Fprintf(c.b, " %%%s = trunc i64 %s to i32\n", narrow, value) + mask = "%" + narrow + } + if form.memoryWrite { + value, err := vector(ins.Args[0]) + if err != nil { + return err + } + assembly := prefix + name + " $0, " + segment + "($1)" + constraints := constraint + ",r" + arguments := fmt.Sprintf("%s %s, %s %s", vectorType, value, pointerType, pointer) + if mask != "" { + assembly = "kmovw ${2:k}, %k1; " + assembly + " {%k1}" + constraints += ",r,~{k1}" + arguments += ", i32 " + mask + } + fmt.Fprintf(c.b, " call void asm sideeffect %q, %q(%s)\n", assembly, constraints+",~{memory}", arguments) + return nil + } + assembly := prefix + name + " " + segment + "($1), $0" + constraints := "=" + constraint + ",r" + arguments := pointerType + " " + pointer + if mask != "" { + if form.zeroing { + assembly = "kmovw ${2:k}, %k1; " + assembly + " {%k1} {z}" + } else { + old, err := vector(ins.Args[form.destination]) + if err != nil { + return err + } + assembly = "kmovw ${3:k}, %k1; " + prefix + name + " " + segment + "($2), $0 {%k1}" + constraints = "=" + constraint + ",0,r" + arguments = vectorType + " " + old + ", " + arguments + } + constraints += ",r,~{k1}" + arguments += ", i32 " + mask + } + loaded, bytes := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = call %s asm sideeffect %q, %q(%s)\n", loaded, vectorType, assembly, constraints+",~{memory}", arguments) + fmt.Fprintf(c.b, " %%%s = bitcast %s %%%s to <%d x i8>\n", bytes, vectorType, loaded, form.byteWidth) + return c.storePackedMoveOperand(ins.Args[form.destination], form.byteWidth, "%"+bytes) +} diff --git a/amd64_packed_vector_move_aligned_oracle_unix_test.go b/amd64_packed_vector_move_aligned_oracle_unix_test.go new file mode 100644 index 000000000..3010e50cb --- /dev/null +++ b/amd64_packed_vector_move_aligned_oracle_unix_test.go @@ -0,0 +1,27 @@ +//go:build unix + +package plan9asm + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Every host compiles the actual Go-byte original/LLVM oracle. Only the required +// Linux/amd64 runtime root executes it; this test is not native ISA evidence. +func TestX86PackedMoveAlignedOriginalGoOracleCompileOnly(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + ir, main, count := x86PackedMoveObservableFixture(t, true, true, execution.triple) + compileLLVMToObject(t, execution.llc, execution.triple, "aligned-oracle.ll", "aligned-oracle.o", ir) + dir := t.TempDir() + source, object := filepath.Join(dir, "oracle.c"), filepath.Join(dir, "oracle.o") + if err := os.WriteFile(source, []byte(main), 0600); err != nil { + t.Fatal(err) + } + command := append([]string(nil), execution.compiler...) + command = append(command, "-O2", "-c", source, "-o", object) + x86FloatCompareCaptureCommand(t, exec.Command(command[0], command[1:]...)) + t.Logf("compiled %d original-Go aligned cases; native numerical/fault execution NOT performed by this test", count) +} diff --git a/amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go b/amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go new file mode 100644 index 000000000..a93a986f7 --- /dev/null +++ b/amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go @@ -0,0 +1,15 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedMaskedMemory(t *testing.T) { + x86PackedMoveMaskedMemoryRuntime(t, true) +} + +func TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedRegisterViews(t *testing.T) { + x86PackedMoveRegisterViewsRuntime(t, true) +} + +func TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedOriginalGo(t *testing.T) { + x86PackedMoveObservableRuntime(t, true, true) +} diff --git a/amd64_packed_vector_move_aligned_test.go b/amd64_packed_vector_move_aligned_test.go new file mode 100644 index 000000000..f5d4b0154 --- /dev/null +++ b/amd64_packed_vector_move_aligned_test.go @@ -0,0 +1,301 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86PackedMoveAlignedNativeOpcodeAndFeatures(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, mode := range []string{"load", "store", "merge", "zero", "masked-store"} { + masked := mode != "load" && mode != "store" + if row.vexOnly && masked { + continue + } + t.Run(row.name+"/"+bank+"/"+mode, func(t *testing.T) { + op, mask := row.name, "" + if masked { + mask = "K7," + } + if mode == "zero" { + op += ".Z" + } + operation := fmt.Sprintf("%s (AX),%s%s0", op, mask, bank) + if strings.HasSuffix(mode, "store") { + operation = fmt.Sprintf("%s %s0,%s(AX)", op, bank, mask) + } + source := "TEXT ·view(SB),4,$0-8\nMOVQ p+0(FP),AX\nMOVL $3,CX\nKMOVD CX,K7\nVPXORQ Z0,Z0,Z0\n" + operation + "\nRET\n" + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "asm sideeffect") || !strings.Contains(ir, strings.ToLower(row.name)+" ") { + t.Fatal("aligned memory lost its original observable opcode", operation) + } + feature := "+avx" + if masked || bank == "Z" || strings.HasPrefix(row.name, "VMOVDQA3") || row.name == "VMOVDQA64" { + feature = "+avx512f" + } + if !strings.Contains(ir, feature) { + t.Fatal("missing required native feature", feature) + } + }) + } + } + } +} + +func TestX86PackedMoveAlignedVirtualMemoryNeedsContext(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, memory := range []string{"a+0(FP)", "local-64(SP)", "(BP)"} { + for _, operands := range []string{memory + "," + bank + "0", bank + "0," + memory} { + source := fmt.Sprintf("TEXT ·view(SB),4,$64-64\n%s %s\nRET\n", row.name, operands) + _, err := x86ScalarFPBitViewAPI(t, source, "func view(a,b,c,d,e,f,g,h uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s %s acquired native alignment/backing: %v", row.name, operands, err) + } + } + } + } + } +} + +func TestX86PackedMoveAlignedResultFiveTargetsThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, move := 8, "MOVQ" + if target.arch == "386" { + word, move = 4, "MOVL" + } + var source, declarations strings.Builder + count := 0 + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, mode := range []string{"plain", "merge", "zero"} { + if row.vexOnly && mode != "plain" { + continue + } + name, mask := row.name, "" + if mode != "plain" { + mask = "K1," + } + if mode == "zero" { + name += ".Z" + } + fmt.Fprintf(&declarations, "func view%d(p *[64]byte) uint32\n", count) + fmt.Fprintf(&source, "TEXT ·view%d(SB),4,$0-%d\n%s p+0(FP),SI\nMOVL $1,AX\nKMOVD AX,K1\nVPXORQ Z1,Z1,Z1\n%s (SI),%s%s1\nPEXTRD $3,X1,AX\nMOVL AX,ret+%d(FP)\nRET\n", count, word+4, move, name, mask, bank, word) + count++ + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + compileLLVMToObject(t, llc, target.triple, "aligned-result.ll", "aligned-result.o", ir) + } + }) + } +} + +func TestX86PackedMoveAlignedNativeCompleteMemoryAxes(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("GLOBL table<>(SB),8,$128\n") + sigs := make(map[string]FuncSig) + count := 0 + add := func(instruction string) { + name := fmt.Sprintf("aligned%d", count) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n%s\nRET\n", name, instruction) + sigs[name] = FuncSig{Name: name, Ret: Void} + count++ + } + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + // Every mask register, both directions, merge and zero. + for mask := 0; mask <= 7; mask++ { + if row.vexOnly && mask != 0 { + continue + } + predicate := "" + if mask != 0 { + predicate = fmt.Sprintf("K%d,", mask) + } + add(fmt.Sprintf("%s (AX),%s%s7", row.name, predicate, bank)) + add(fmt.Sprintf("%s %s7,%s(AX)", row.name, bank, predicate)) + if mask != 0 { + add(fmt.Sprintf("%s.Z (AX),%s%s7", row.name, predicate, bank)) + } + } + // Orthogonal address-space and high-bank encoding classes. + indexes := []int{0, 7} + if target.arch == "amd64" { + indexes = append(indexes, 8, 15) + if !row.vexOnly { + indexes = append(indexes, 16, 31) + } + } + for _, index := range indexes { + for _, address := range []string{"16(AX)(CX*2)", "table<>+16(SB)", "16(FS)", "16(GS)"} { + add(fmt.Sprintf("%s %s,%s%d", row.name, address, bank, index)) + add(fmt.Sprintf("%s %s%d,%s", row.name, bank, index, address)) + } + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, segment := range []string{"%fs:", "%gs:", "ptr addrspace(256)", "ptr addrspace(257)"} { + if !strings.Contains(ir, segment) { + t.Fatal("native address-space lost", segment) + } + } + if got := strings.Count(ir, "asm sideeffect"); got != count { + t.Fatalf("native memory witness count got %d want %d", got, count) + } + compileLLVMToObject(t, llc, target.triple, "aligned-memory.ll", "aligned-memory.o", ir) + t.Logf("%d actual-Go/LLVM aligned memory forms", count) + }) + } +} + +func TestX86PackedMoveAlignedNativeAddressSizeBoundary(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, addressBits := range []int{16, 32, 64} { + ins := x86ScalarLaneInstruction(t, row.name+" (AX),X0") + ins.x86AddressBits = addressBits + ctx := &amd64Ctx{goarch: "amd64"} + if _, _, err := ctx.lowerPackedVectorMove(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unbound original address width borrowed ordinary memory", row.name, addressBits, err) + } + } + } +} + +func TestX86PackedMoveAlignedDATAReadIsNotWriteAuthority(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, masked := range []bool{false, true} { + if row.vexOnly && masked { + continue + } + predicate := "" + if masked { + predicate = "K1," + } + source := "DATA table<>+0(SB)/8,$1\nGLOBL table<>(SB),8,$64\nTEXT ·view(SB),4,$0-4\nMOVL $1,AX\nKMOVD AX,K1\n" + + row.name + " table<>(SB),X1\n" + row.name + " X1," + predicate + "table<>(SB)\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n" + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "func view() uint32\n", "amd64", "x86_64-unknown-linux-gnu", api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("readable DATA gained write authority", row.name, masked, api, err) + } + } + } + } +} + +func TestX86PackedMoveAlignedRawSourceCannotBorrowConditionalRead(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[64]byte) uint32", + "MOVQ p+0(FP),AX\n"+row.name+" (AX),X1\nPEXTRD $0,X1,CX\nMOVL CX,ret0+8(FP)\nRET") + for _, mutation := range []string{"decoded", "address size", "unbound address", "raw text", "source stack", "fixed local", "segment", "symbol"} { + fn := file.Funcs[0] + fn.Instrs = append([]Instr(nil), fn.Instrs...) + fn.Instrs[2].Args = append([]Operand(nil), fn.Instrs[2].Args...) + switch mutation { + case "decoded": + fn.Instrs[2].x86Encoded = true + case "address size": + fn.Instrs[2].x86AddressBits = 32 + case "unbound address": + fn.Instrs[2].x86UnboundAddressOverride = true + case "raw text": + fn.X86RawText = []byte{0xc3} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + case "segment": + fn.Instrs[2].Args[0].Mem.Segment = "FS" + case "symbol": + fn.Instrs[2].Args[0].Mem.Sym = "table<>(SB)" + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("aligned read bypassed source identity", row.name, mutation, err) + } + } + } +} + +func TestX86PackedMoveAlignedRegisterCopyRemainsPortable(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if !row.aligned { + continue + } + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + ins := x86ScalarLaneInstruction(t, row.name+" "+bank+"0,"+bank+"1") + if features := inferFuncTargetFeaturesForGOARCH(ArchAMD64, "amd64", Func{Instrs: []Instr{ins}}); features != "" { + t.Fatal("register bit-copy acquired physical memory ISA", ins, features) + } + } + } +} diff --git a/amd64_packed_vector_move_complete_grammar_test.go b/amd64_packed_vector_move_complete_grammar_test.go new file mode 100644 index 000000000..00810eb8f --- /dev/null +++ b/amd64_packed_vector_move_complete_grammar_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/plan9asm/internal/goasmrules" +) + +func TestX86PackedMoveCompleteIndependentEncoderRows(t *testing.T) { + rules, err := goasmrules.LoadX86AVXRules(runtime.GOROOT()) + if err != nil { + t.Fatal(err) + } + for _, expected := range []struct { + name, table, pp, evexW, store, load string + vex, evex bool + }{ + {"VMOVDQA", "_yvmovdqa", "66", "", "0x7F", "0x6F", true, false}, + {"VMOVDQU", "_yvmovdqa", "F3", "", "0x7F", "0x6F", true, false}, + {"VMOVAPD", "_yvmovapd", "66", "1", "0x29", "0x28", true, true}, + {"VMOVAPS", "_yvmovapd", "", "0", "0x29", "0x28", true, true}, + {"VMOVUPD", "_yvmovapd", "66", "1", "0x11", "0x10", true, true}, + {"VMOVUPS", "_yvmovapd", "", "0", "0x11", "0x10", true, true}, + {"VMOVDQA32", "_yvmovdqa32", "66", "0", "0x7F", "0x6F", false, true}, + {"VMOVDQA64", "_yvmovdqa32", "66", "1", "0x7F", "0x6F", false, true}, + {"VMOVDQU8", "_yvmovdqa32", "F2", "0", "0x7F", "0x6F", false, true}, + {"VMOVDQU16", "_yvmovdqa32", "F2", "1", "0x7F", "0x6F", false, true}, + {"VMOVDQU32", "_yvmovdqa32", "F3", "0", "0x7F", "0x6F", false, true}, + {"VMOVDQU64", "_yvmovdqa32", "F3", "1", "0x7F", "0x6F", false, true}, + } { + var classes, zcases, opbytes []string + var offsets []int + for _, encoding := range []string{"vex", "evex"} { + if encoding == "vex" && !expected.vex || encoding == "evex" && !expected.evex { + continue + } + for direction, opcode := range []string{expected.store, expected.load} { + widths := []int{128, 256} + if encoding == "evex" { + widths = append(widths, 512) + } + for _, width := range widths { + bank := map[int]string{128: "x", 256: "y", 512: "z"}[width] + register, memory := "Y"+bank+"r", "Y"+bank+"m" + word := "0" + if encoding == "evex" { + word = expected.evexW + if width != 512 { + register, memory = register+"Evex", memory+"Evex" + } + } + prefix := fmt.Sprintf("avxEscape | %s%d", encoding, width) + if expected.pp != "" { + prefix += " | " + encoding + expected.pp + } + prefix += " | " + encoding + "0F | " + encoding + "W" + word + opbytes = append(opbytes, prefix) + if encoding == "evex" { + opbytes = append(opbytes, fmt.Sprintf("evexN%d | evexZeroingEnabled", width/8)) + } + opbytes = append(opbytes, opcode) + for _, masked := range []bool{false, true} { + if masked && encoding == "vex" { + continue + } + first, last, zcase := register, memory, "r_v_rm" + if direction == 1 { + first, last, zcase = memory, register, "rm_v_r" + } + args, offset := first+","+last, 2 + if encoding == "evex" { + offset = 0 + if masked { + args, offset = first+",Yknot0,"+last, 3 + zcase = strings.Replace(zcase, "_v_", "_k_", 1) + } + } + classes, zcases, offsets = append(classes, args), append(zcases, "Z"+encoding+"_"+zcase), append(offsets, offset) + } + } + } + } + count, encodingOffset := 0, 0 + for _, rule := range rules { + if rule.Opcode != expected.name { + continue + } + if count >= len(classes) || rule.RowIndex != count || rule.Ytab != expected.table || rule.Prefix != "Pavx" || + strings.Join(rule.Classes, ",") != classes[count] || rule.ZCase != zcases[count] || rule.ZOffset != offsets[count] || + rule.EncodingOffset != encodingOffset || !reflect.DeepEqual(rule.OpBytes, opbytes) { + t.Fatalf("%s complete Go row/attributes changed: %+v", expected.name, rule) + } + encodingOffset += offsets[count] + count++ + } + if count != len(classes) { + t.Fatal("missing independent encoder rows", expected.name, count, len(classes)) + } + } +} + +func TestX86PackedMoveCompleteRegisterMaskAndAddressAxes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + var rows []string + for _, row := range x86PackedMoveResultOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + limit := 32 + if row.vexOnly { + limit = 16 + } + if arch == "386" && bank == "Z" { + limit = 8 + } + for index := 0; index < limit; index++ { + rows = append(rows, fmt.Sprintf("%s %s%d,%s1", row.name, bank, index, bank), fmt.Sprintf("%s %s0,%s%d", row.name, bank, bank, index)) + } + if !row.vexOnly { + for mask := 1; mask <= 7; mask++ { + for _, suffix := range []string{"", ".Z"} { + rows = append(rows, fmt.Sprintf("%s%s %s0,K%d,%s1", row.name, suffix, bank, mask, bank), + fmt.Sprintf("%s%s (AX),K%d,%s1", row.name, suffix, mask, bank), + fmt.Sprintf("%s%s %s0,K%d,(AX)", row.name, suffix, bank, mask)) + } + } + } + for _, address := range []string{"(AX)", "-16(BX)(CX*2)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1", row.name, address, bank), fmt.Sprintf("%s %s1,%s", row.name, bank, address)) + } + } + } + var source strings.Builder + for index, row := range rows { + fmt.Fprintf(&source, "TEXT f%d(SB),4,$64-64\n%s\nRET\n", index, row) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for index, fn := range file.Funcs { + ins := fn.Instrs[1] + _, known, err := amd64PackedVectorMoveProperties(ins, arch) + physicalContext := false + for _, arg := range ins.Args { + register, vector := amd64VectorRegisterIndex(arg.Reg, amd64VectorByteWidth(arg.Reg)) + physicalContext = physicalContext || arch == "386" && vector && register >= 8 + } + if !known || physicalContext && !errors.Is(err, ErrProbeNeedsContext) || !physicalContext && err != nil { + t.Fatal("independent Go family grammar mismatch", arch, rows[index], err) + } + } + t.Logf("%s: %d independent actual-Go accepted form axes", arch, len(rows)) + } +} + +func TestX86PackedMoveConstructedMemoryCannotBypassGrammar(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for _, arch := range []string{"amd64", "386"} { + for _, change := range []string{"unknown offset", "unknown index", "bad scale", "stack index", "vector base", "vector index", "unknown register", "address literal"} { + ins := x86ScalarLaneInstruction(t, row.name+" (AX),X1") + switch change { + case "unknown offset": + ins.Args[0].Mem.OffRaw = "missing" + case "unknown index": + ins.Args[0].Mem.IndexExt = "missing" + case "bad scale": + ins.Args[0].Mem.Index, ins.Args[0].Mem.Scale = CX, 3 + case "stack index": + ins.Args[0].Mem.Index, ins.Args[0].Mem.Scale = SP, 1 + case "vector base": + ins.Args[0].Mem.Base = "X0" + case "vector index": + ins.Args[0].Mem.Index, ins.Args[0].Mem.Scale = "X0", 1 + case "unknown register": + ins.Args[0].Mem.Base = "UNKNOWN" + case "address literal": + ins.Args[0] = Operand{Kind: OpSym, Sym: "$table<>(SB)"} + } + if _, known, err := amd64PackedVectorMoveProperties(ins, arch); !known || err == nil { + t.Fatal("constructed memory bypassed grammar", row.name, arch, change, err) + } + if x86ScalarFPResultTransfer(ins, &x86ResultState{}, nil, 0, arch) { + t.Fatal("invalid memory acquired a result effect", row.name, arch, change) + } + } + } + } +} diff --git a/amd64_packed_vector_move_forms.go b/amd64_packed_vector_move_forms.go new file mode 100644 index 000000000..9fd1e42b3 --- /dev/null +++ b/amd64_packed_vector_move_forms.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type amd64PackedMoveEncoding uint8 + +const ( + amd64PackedMoveVEX amd64PackedMoveEncoding = iota + amd64PackedMoveVEXOrEVEX + amd64PackedMoveEVEX +) + +type amd64PackedVectorMoveSpec struct { + elementBits int + encoding amd64PackedMoveEncoding + aligned bool +} + +// This is the complete _yvmovdqa, _yvmovapd and _yvmovdqa32 family. +// Encoding generation controls widths/registers/masks; elementBits controls +// mask granularity, not the native memory access's complete byte envelope. +var amd64PackedVectorMoveSpecs = map[Op]amd64PackedVectorMoveSpec{ + "VMOVDQA": {8, amd64PackedMoveVEX, true}, + "VMOVDQU": {8, amd64PackedMoveVEX, false}, + "VMOVAPD": {64, amd64PackedMoveVEXOrEVEX, true}, + "VMOVAPS": {32, amd64PackedMoveVEXOrEVEX, true}, + "VMOVUPD": {64, amd64PackedMoveVEXOrEVEX, false}, + "VMOVUPS": {32, amd64PackedMoveVEXOrEVEX, false}, + "VMOVDQA32": {32, amd64PackedMoveEVEX, true}, + "VMOVDQA64": {64, amd64PackedMoveEVEX, true}, + "VMOVDQU8": {8, amd64PackedMoveEVEX, false}, + "VMOVDQU16": {16, amd64PackedMoveEVEX, false}, + "VMOVDQU32": {32, amd64PackedMoveEVEX, false}, + "VMOVDQU64": {64, amd64PackedMoveEVEX, false}, +} + +type amd64PackedVectorMoveForm struct { + spec amd64PackedVectorMoveSpec + byteWidth int + destination int + mask int // -1 for unmasked forms + memory int // -1 for register-to-register forms + memoryWrite bool + zeroing bool // native EVEX.z; memory destinations raise #UD +} + +// Go's frontend accepts .Z memory stores and sets EVEX.z in their objects, +// although the native instruction requires EVEX.z=0 for a memory destination. +// Keep syntax acceptance separate from its terminal exception effect. +func (form amd64PackedVectorMoveForm) undefinedOpcode() bool { + return form.zeroing && form.memoryWrite +} + +// Only the written vector bytes are described here. Callers independently +// supply numerical source/merge bytes and validate every mask and native +// memory role. VEX/EVEX upper-zeroing is applied by the state consumer. +func (form amd64PackedVectorMoveForm) definedActiveBytes(source, previous uint64) uint64 { + known := source & x86CallerVectorByteMask(int64(form.byteWidth)) + if form.mask >= 0 && !form.zeroing { + known &= previous + } + return known +} + +// Both the lowerer and caller-effect proof consume this grammar. It supplies +// no memory bounds, alignment, initialized bytes, mask value or continuation. +func amd64PackedVectorMoveProperties(ins Instr, goarch string) (amd64PackedVectorMoveForm, bool, error) { + name := strings.ToUpper(string(ins.Op)) + base, suffix, dotted := strings.Cut(name, ".") + spec, recognized := amd64PackedVectorMoveSpecs[Op(base)] + if !recognized { + return amd64PackedVectorMoveForm{}, false, nil + } + form := amd64PackedVectorMoveForm{ + spec: spec, destination: len(ins.Args) - 1, + mask: -1, memory: -1, zeroing: suffix == "Z", + } + fail := func() (amd64PackedVectorMoveForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix are outside the packed vector-move grammar: %q", goarch, base, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || dotted && suffix != "Z" || + len(ins.Args) != 2 && len(ins.Args) != 3 { + return fail() + } + if len(ins.Args) == 3 { + if spec.encoding == amd64PackedMoveVEX || !amd64NonzeroKOperand(ins.Args[1]) { + return fail() + } + form.mask = 1 + } else if form.zeroing { + return fail() + } + vector := func(operand Operand, width int) bool { + if !amd64GoPackedVectorMoveRegister(operand, width, goarch) { + return false + } + if spec.encoding == amd64PackedMoveVEX { + return width != 64 && amd64VEXVectorRegister(operand, width) + } + return true + } + source, destination := ins.Args[0], ins.Args[form.destination] + switch { + case source.Kind == OpReg: + form.byteWidth = amd64VectorByteWidth(source.Reg) + if !vector(source, form.byteWidth) { + return fail() + } + if isAMD64MemoryOperand(destination) { + form.memory, form.memoryWrite = form.destination, true + } else if !vector(destination, form.byteWidth) { + return fail() + } + case isAMD64MemoryOperand(source): + form.byteWidth = amd64VectorByteWidth(destination.Reg) + if !vector(destination, form.byteWidth) { + return fail() + } + form.memory = 0 + default: + return fail() + } + if form.memory >= 0 { + memory := ins.Args[form.memory] + if !x86PackedShiftMemoryValid(memory, goarch) { + return fail() + } + if memory.Kind == OpMem { + if err := x86SymbolIndexedMemoryError(memory.Mem, goarch); err != nil { + return form, true, err + } + if memory.Mem.OffRaw != "" || memory.Mem.IndexExt != "" { + return fail() + } + } + } + return form, true, x86Mode32VectorRegisterContext(ins, goarch) +} + +// This selects actual ordinary memory, not an extent or a continuation proof. +// The lowerer and private conditional result reader must select the same path. +func (form amd64PackedVectorMoveForm) unalignedOrdinaryMemory(ins Instr, arch string) bool { + return !form.spec.aligned && form.ordinaryMemory(ins, arch) +} + +func (form amd64PackedVectorMoveForm) ordinaryMemory(ins Instr, arch string) bool { + return form.memory >= 0 && + x86OrdinaryNativeMemoryOperand(ins.Args[form.memory], arch) +} + +func (form amd64PackedVectorMoveForm) nativeEVEX(ins Instr) bool { + if form.spec.encoding == amd64PackedMoveEVEX || form.mask >= 0 || form.byteWidth == 64 { + return true + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && amd64VectorByteWidth(operand.Reg) != 0 { + index, ok := amd64VectorRegisterIndex(operand.Reg, form.byteWidth) + if ok && index >= 16 { + return true + } + } + } + return false +} + +func amd64GoPackedVectorMoveRegister(operand Operand, width int, goarch string) bool { + if !amd64EVEXVectorRegister(operand, width) { + return false + } + index, _ := amd64VectorRegisterIndex(operand.Reg, width) + return goarch != "386" || width != 64 || index < 8 +} diff --git a/amd64_packed_vector_move_forms_test.go b/amd64_packed_vector_move_forms_test.go new file mode 100644 index 000000000..b6026faf8 --- /dev/null +++ b/amd64_packed_vector_move_forms_test.go @@ -0,0 +1,229 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "reflect" + "runtime" + "sort" + "strings" + "testing" +) + +type x86PackedMoveExpectedAxes struct { + operands string + memory, mask int + write, zero bool +} + +func TestX86PackedMoveGrammarCoversEveryCurrentGoFamilyMember(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go"), nil, 0) + if err != nil { + t.Fatal(err) + } + tables := map[string]amd64PackedMoveEncoding{ + "_yvmovdqa": amd64PackedMoveVEX, "_yvmovapd": amd64PackedMoveVEXOrEVEX, + "_yvmovdqa32": amd64PackedMoveEVEX, + } + wanted := make(map[Op]amd64PackedMoveEncoding) + rows := make(map[string]int) + ast.Inspect(file, func(node ast.Node) bool { + if value, ok := node.(*ast.ValueSpec); ok && len(value.Names) == 1 && len(value.Values) == 1 { + name := value.Names[0].Name + if _, family := tables[name]; family { + literal, ok := value.Values[0].(*ast.CompositeLit) + if !ok { + t.Fatalf("Go packed-move table is no longer literal: %s", name) + } + rows[name] = len(literal.Elts) + } + } + literal, ok := node.(*ast.CompositeLit) + if !ok { + return true + } + var opcode, table string + for _, element := range literal.Elts { + field, ok := element.(*ast.KeyValueExpr) + if !ok { + continue + } + key, keyOK := field.Key.(*ast.Ident) + value, valueOK := field.Value.(*ast.Ident) + if !keyOK || !valueOK { + continue + } + switch key.Name { + case "as": + opcode = value.Name + case "ytab": + table = value.Name + } + } + if encoding, family := tables[table]; family { + if !strings.HasPrefix(opcode, "AVMOV") { + t.Fatalf("unregistered spelling in shared Go move family: %s", opcode) + } + wanted[Op(strings.TrimPrefix(opcode, "A"))] = encoding + } + return true + }) + if !reflect.DeepEqual(rows, map[string]int{"_yvmovdqa": 4, "_yvmovapd": 16, "_yvmovdqa32": 12}) || len(wanted) != len(amd64PackedVectorMoveSpecs) { + t.Fatalf("current Go family changed: rows=%v opcodes=%v", rows, wanted) + } + for opcode, encoding := range wanted { + spec, present := amd64PackedVectorMoveSpecs[opcode] + if !present || spec.encoding != encoding { + t.Fatalf("Go opcode/encoding omitted or changed: %s/%d", opcode, encoding) + } + } +} + +func TestX86PackedMoveGrammarAxesAndFiveLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var opcodes []string + for opcode := range amd64PackedVectorMoveSpecs { + opcodes = append(opcodes, string(opcode)) + } + sort.Strings(opcodes) + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + sigs := make(map[string]FuncSig) + for _, opcode := range opcodes { + spec := amd64PackedVectorMoveSpecs[Op(opcode)] + for index, bank := range []string{"X", "Y", "Z"} { + if bank == "Z" && spec.encoding == amd64PackedMoveVEX { + continue + } + width, high := 16< %s, %s %s, align 1\n", width, value, pointerType, pointer) + return nil + } + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile <%d x i8>, %s %s, align 1\n", value, width, pointerType, pointer) + return c.storePackedMoveOperand(ins.Args[form.destination], width, "%"+value) + } + + mask, err := c.loadK(ins.Args[form.mask].Reg) + if err != nil { + return err + } + bits := form.spec.elementBits + lanes := width * 8 / bits + vectorType := fmt.Sprintf("<%d x i%d>", lanes, bits) + value := "zeroinitializer" + if form.memoryWrite { + value, err = c.loadPackedCompareLanes(ins.Args[0], width, bits, false) + } else if !form.zeroing { + value, err = c.loadPackedCompareLanes(ins.Args[form.destination], width, bits, false) + } + if err != nil { + return err + } + + for lane := 0; lane < lanes; lane++ { + condition := amd64MaskBitI1(c, mask, lane) + address := c.newTmp() + // No inbounds GEP: inactive lanes, including a null base, need no + // allocation. Only the guarded branch accesses the selected bytes. + fmt.Fprintf(c.b, " %%%s = getelementptr i8, %s %s, i64 %d\n", address, pointerType, pointer, lane*bits/8) + if form.memoryWrite { + element := c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", element, vectorType, value, lane) + storeLabel, joinLabel := c.newTmp()+"_packed_store", c.newTmp()+"_packed_join" + fmt.Fprintf(c.b, " br i1 %s, label %%%s, label %%%s\n%s:\n", condition, storeLabel, joinLabel, storeLabel) + fmt.Fprintf(c.b, " store volatile i%d %%%s, %s %%%s, align 1\n", bits, element, pointerType, address) + fmt.Fprintf(c.b, " br label %%%s\n%s:\n", joinLabel, joinLabel) + continue + } + loaded, err := c.loadX86ScalarIf(bits, condition, func() (string, error) { + loaded := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load volatile i%d, %s %%%s, align 1\n", loaded, bits, pointerType, address) + return "%" + loaded, nil + }) + if err != nil { + return err + } + if !form.zeroing { + old, selected := c.newTmp(), c.newTmp() + fmt.Fprintf(c.b, " %%%s = extractelement %s %s, i32 %d\n", old, vectorType, value, lane) + fmt.Fprintf(c.b, " %%%s = select i1 %s, i%d %s, i%d %%%s\n", selected, condition, bits, loaded, bits, old) + loaded = "%" + selected + } + inserted := c.newTmp() + fmt.Fprintf(c.b, " %%%s = insertelement %s %s, i%d %s, i32 %d\n", inserted, vectorType, value, bits, loaded, lane) + value = "%" + inserted + } + if form.memoryWrite { + return nil + } + bytes := c.newTmp() + fmt.Fprintf(c.b, " %%%s = bitcast %s %s to <%d x i8>\n", bytes, vectorType, value, width) + return c.storePackedMoveOperand(ins.Args[form.destination], width, "%"+bytes) +} diff --git a/amd64_packed_vector_move_observable_runtime_linux_amd64_test.go b/amd64_packed_vector_move_observable_runtime_linux_amd64_test.go new file mode 100644 index 000000000..9481a7700 --- /dev/null +++ b/amd64_packed_vector_move_observable_runtime_linux_amd64_test.go @@ -0,0 +1,7 @@ +package plan9asm + +import "testing" + +func TestCrossLinuxRuntimeMatrixX86PackedMoveUnalignedOriginalGo(t *testing.T) { + x86PackedMoveUnalignedRuntime(t, true) +} diff --git a/amd64_packed_vector_move_observable_runtime_unix_test.go b/amd64_packed_vector_move_observable_runtime_unix_test.go new file mode 100644 index 000000000..0e6eaadca --- /dev/null +++ b/amd64_packed_vector_move_observable_runtime_unix_test.go @@ -0,0 +1,256 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86PackedMoveUnalignedPortableFaults(t *testing.T) { + x86PackedMoveUnalignedRuntime(t, false) +} + +func x86PackedMoveUnalignedRuntime(t *testing.T, native bool) { + x86PackedMoveObservableRuntime(t, native, false) +} + +func x86PackedMoveObservableRuntime(t *testing.T, native, aligned bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + if native { + execution = x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + } + ir, main, count := x86PackedMoveObservableFixture(t, native, aligned, execution.triple) + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%t", optimized), func(t *testing.T) { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + execution.compileAndRunLLVM(t, "packed-move-observable", text, main, "-O2") + }) + } + t.Logf("%d independent move cases; original native Go execution=%t; aligned native semantics=%t", count, native, aligned) +} + +func x86PackedMoveObservableFixture(t *testing.T, native, aligned bool, triple string) (string, string, int) { + t.Helper() + if aligned && !native { + t.Fatal("aligned fault semantics require the original native counterpart") + } + var source, declarations, originals, pairs strings.Builder + sigs := make(map[string]FuncSig) + count := 0 + for _, row := range x86PackedMoveResultOracle { + if row.aligned != aligned { + continue + } + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + for _, mode := range []string{"plain-live", "plain-dead", "plain-store", "merge-live", "zero-live", "zero-dead", "mask-store"} { + masked := strings.HasPrefix(mode, "merge") || strings.HasPrefix(mode, "zero") || mode == "mask-store" + if row.vexOnly && masked { + continue + } + store, dead, zero := strings.HasSuffix(mode, "store"), strings.HasSuffix(mode, "dead"), strings.HasPrefix(mode, "zero") + op, mask := row.name, "" + if masked { + mask = "K1," + } + if zero { + op += ".Z" + } + operation := fmt.Sprintf("%s (AX),%s%s0\n", op, mask, bank) + if store { + operation = fmt.Sprintf("%s %s0,%s(AX)\n", op, bank, mask) + } + name := fmt.Sprintf("packed_observable%d", count) + body := "KMOVQ CX,K1\nVMOVUPS (DI),Z0\nSTC\n" + operation + if !store && !dead { + body += "VMOVUPS Z0,(DI)\n" + } + body += "SETCS 64(DI)\n" + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ p+0(FP),AX\nMOVQ out+8(FP),DI\nMOVQ mask+16(FP),CX\n%sRET\n", name, body) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, I64}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,uint8_t*,uint64_t);\n", name) + original := "0" + if native { + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+body+"RET\n") + if len(code) < 2 || code[len(code)-1] != 0xc3 { + t.Fatal("missing original Go opcode and RET", name, code) + } + var bytes []string + for _, value := range code[:len(code)-1] { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + original = "original_" + name + fmt.Fprintf(&originals, `__attribute__((target("avx512f,avx512bw,avx512vl"))) +static void %s(uint8_t *p,uint8_t *out,uint64_t mask) { + __asm__ volatile(".byte %s" : : "a"(p),"D"(out),"c"(mask) : "zmm0","k1","memory","cc"); +} +`, original, strings.Join(bytes, ",")) + } + flag := func(value bool) int { + if value { + return 1 + } + return 0 + } + fmt.Fprintf(&pairs, "{%s,%s,%d,%d,%d,%d,%d,%d},\n", original, name, width, row.bits/8, flag(masked), flag(store), flag(dead), flag(zero)) + count++ + } + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + alignment := "0" + if aligned { + alignment = "1" + } + main := "#define ALIGNED " + alignment + "\n" + x86PackedMoveObservableRuntimeC + declarations.String() + originals.String() + "static const struct pair pairs[]={\n" + pairs.String() + `}; +int main(void) { + if (pairs[0].original) { + unsigned a,b,c,d; + if (!__get_cpuid_count(7,0,&a,&b,&c,&d) || !(b&(1u<<30))) return 78; + } + long page=sysconf(_SC_PAGESIZE); + if(page<256) return 1; + uint8_t *region=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(region==MAP_FAILED || mprotect(region+page,page,PROT_NONE)) return 2; + for(unsigned op=0;op +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef void (*operation)(uint8_t*,uint8_t*,uint64_t); +struct pair { operation original,translated; int width,lane,masked,store,dead,zero; }; +static int active(struct pair pair,uint64_t mask,int byte) { + return !pair.masked || ((mask>>(byte/pair.lane))&1); +} +static int expected_fault(struct pair pair,uint64_t mask,int extent) { + for(int byte=extent;byte=pair.width?0:active(pair,masks[m],i)?a[in+i]:pair.zero?0:x[out+i]; + } + } + want_x[out+64]=1; + unsigned saved=_mm_getcsr(),csr=0x1f80|((sample&3)<<13)|(sample&63); + _mm_setcsr(csr); pair.translated(a+in,x+out,masks[m]); unsigned after=_mm_getcsr(); + _mm_setcsr(saved); + if(after!=csr || memcmp(a,want_a,256) || memcmp(x,want_x,256)) return 1; + if(pair.original) { + _mm_setcsr(csr); pair.original(b+in,y+out,masks[m]); after=_mm_getcsr(); + _mm_setcsr(saved); + if(after!=csr || memcmp(a,b,256) || memcmp(x,y,256)) return 2; + } + } + return 0; +} +` + +func TestX86PackedMoveObservableFaultChildrenDisableCore(t *testing.T) { + if err := runtimeOracleFaultCorePolicy(runtimeOracleFaultC(t, "amd64_packed_vector_move_observable_runtime_unix_test.go")); err != nil { + t.Fatal(err) + } +} diff --git a/amd64_packed_vector_move_result.go b/amd64_packed_vector_move_result.go new file mode 100644 index 000000000..13c2ab0d4 --- /dev/null +++ b/amd64_packed_vector_move_result.go @@ -0,0 +1,54 @@ +package plan9asm + +// Memory availability is separate from the register's defined-byte transfer. +// conditionalRead is only supplied by the lowering-only, source-wide observer +// paired with an exact native aligned or unaligned volatile read. It is never a +// public observer, initialized-pointee, extent, alignment or write permission. +func x86ResultPackedVectorMoveTransfer(ins Instr, form amd64PackedVectorMoveForm, state *x86ResultState, owned []bool, resultOffset int64, arch string, conditionalRead bool) bool { + if form.undefinedOpcode() { + return false + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false + } + } + width := int64(form.byteWidth) + widthMask := x86CallerVectorByteMask(width) + known := widthMask + if form.memory >= 0 && form.spec.aligned && !form.ordinaryMemory(ins, arch) { + // Virtual FP/AUTO storage is not the original aligned memory. Ordinary + // memory instead retains its actual opcode: continuation defines its + // active bytes without assuming that its native address was aligned. + return false + } + if form.memory == 0 { + if !conditionalRead && !x86ResultRead(ins.Args[0], width, state, owned, arch) { + return false + } + } else { + index, valid := amd64VectorRegisterIndex(ins.Args[0].Reg, form.byteWidth) + if !valid { + return false + } + known = state.vectorBytes(index) & widthMask + } + if form.memoryWrite { + if known != widthMask { + return false + } + destination := ins.Args[form.destination] + if form.mask >= 0 && !x86ResultRead(destination, width, state, owned, arch) { + return false // An unknown mask may retain any old destination byte. + } + return x86ResultStore(destination, width, state, owned, resultOffset, arch) + } + destination, valid := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + if !valid { + return false + } + known = form.definedActiveBytes(known, state.vectorBytes(destination)) + state.writeVector(destination, form.byteWidth, known, false) + return true +} diff --git a/amd64_packed_vector_move_result_test.go b/amd64_packed_vector_move_result_test.go new file mode 100644 index 000000000..7e17de575 --- /dev/null +++ b/amd64_packed_vector_move_result_test.go @@ -0,0 +1,370 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// Independent family inventory: do not derive expected permissions or widths +// from the production registry whose completeness these tests check. +var x86PackedMoveResultOracle = []struct { + name string + bits int + aligned, vexOnly bool +}{ + {"VMOVDQA", 8, true, true}, {"VMOVDQU", 8, false, true}, + {"VMOVAPD", 64, true, false}, {"VMOVAPS", 32, true, false}, + {"VMOVUPD", 64, false, false}, {"VMOVUPS", 32, false, false}, + {"VMOVDQA32", 32, true, false}, {"VMOVDQA64", 64, true, false}, + {"VMOVDQU8", 8, false, false}, {"VMOVDQU16", 16, false, false}, + {"VMOVDQU32", 32, false, false}, {"VMOVDQU64", 64, false, false}, +} + +func TestX86PackedMoveResultEveryByteAndWriteEncoding(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue // The independent Go table has no VEX.512 row. + } + width := 16 << axis + for _, mode := range []string{"plain", "merge", "zero"} { + if row.vexOnly && mode != "plain" { + continue + } + name, mask := row.name, "" + if mode != "plain" { + mask = ",K1" + } + if mode == "zero" { + name += ".Z" + } + instruction := fmt.Sprintf("%s %s0%s,%s1", name, bank, mask, bank) + ins := x86ScalarLaneInstruction(t, instruction) + for missing := 0; missing < 64; missing++ { + for source := 0; source < 2; source++ { + state := x86ResultState{flags: x86ByteFlagsAll, masks: [8]bool{1: true}} + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + state.writeVector(source, 64, ^(uint64(1) << uint(missing)), false) + before := state.clone() + if !x86ScalarFPResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatalf("typed packed move is missing: %s", instruction) + } + want := ^uint64(0) + if missing < width && (source == 0 || mode == "merge") { + want &^= uint64(1) << uint(missing) + } + if state.vectorBytes(1) != want || state.vectorBytes(0) != before.vectorBytes(0) || state.flags != before.flags { + t.Fatalf("%s source%d missing%d: got %016x want %016x", instruction, source, missing, state.vectorBytes(1), want) + } + } + } + } + } + } +} + +func TestX86PackedMoveResultMemoryAndMaskBoundaries(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + for _, memory := range []string{"in+0(FP)", fmt.Sprintf("local-%d(SP)", width)} { + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s %s,%s1", row.name, memory, bank)) + owned := make([]bool, width) + for i := range owned { + owned[i] = true + } + for missing := -1; missing < width; missing++ { + state := x86ResultState{fp: append([]bool(nil), owned...), auto: append([]bool(nil), owned...)} + if missing >= 0 { + state.fp[missing], state.auto[missing] = false, false + } + got := x86ScalarFPResultTransfer(ins, &state, owned, 0, "amd64") + if got != (!row.aligned && missing < 0) { + t.Fatalf("%s %s missing%d: accepted=%t", row.name, memory, missing, got) + } + } + } + if row.vexOnly { + continue + } + for _, suffix := range []string{"", ".Z"} { + state := x86ResultState{} + state.writeVector(0, 64, ^uint64(0), false) + state.writeVector(1, 64, ^uint64(0), false) + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s%s %s0,K1,%s1", row.name, suffix, bank, bank)) + if x86ScalarFPResultTransfer(ins, &state, nil, 0, "amd64") { + t.Fatal("undefined mask acquired a numerical value", ins) + } + } + } + } +} + +func TestX86PackedMoveResultFiveTargetsThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, pointerMove := 8, "MOVQ" + if target.arch == "386" { + word, pointerMove = 4, "MOVL" + } + var source, declarations strings.Builder + count := 0 + for _, row := range x86PackedMoveResultOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + for _, memory := range []bool{false, true} { + for _, mode := range []string{"plain", "merge", "zero"} { + if row.vexOnly && mode != "plain" { + continue + } + name, mask, input := row.name, "", bank+"0" + if memory { + input = "(SI)" + } + if mode != "plain" { + mask = ",K1" + } + if mode == "zero" { + name += ".Z" + } + fmt.Fprintf(&declarations, "func view%d(p *[64]byte) uint32\n", count) + fmt.Fprintf(&source, "TEXT ·view%d(SB),4,$0-%d\n%s p+0(FP),SI\nMOVL $1,AX\nKMOVD AX,K1\nVPXORQ %s0,%s0,%s0\nVPXORQ %s1,%s1,%s1\n%s %s%s,%s1\nPEXTRD $3,X1,AX\nMOVL AX,ret+%d(FP)\nRET\n", + count, word+4, pointerMove, bank, bank, bank, bank, bank, bank, name, input, mask, bank, word) + count++ + } + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + compileLLVMToObject(t, llc, target.triple, "packed-result.ll", "packed-result.o", ir) + } + }) + } +} + +func TestX86PackedMoveResultPointerReadIsLoweringOnly(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for _, arch := range []string{"amd64", "386"} { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + body := fmt.Sprintf("%s p+0(FP),AX\n%s (AX),X1\nPEXTRD $0,X1,CX\nMOVL CX,ret0+%d(FP)\nRET", move, row.name, word) + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value(p *[64]byte) uint32", body) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("declaration-only observer acquired a physical read", row.name, arch, err) + } + } + } + } +} + +func TestX86PackedMoveUnalignedDeadReadIsObservable(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if row.aligned { + continue + } + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + source := fmt.Sprintf("TEXT ·view(SB),4,$0-8\nMOVQ p+0(FP),AX\n%s (AX),%s0\nRET\n", row.name, bank) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, fmt.Sprintf("load volatile <%d x i8>", width)) { + t.Fatal("dead full-tuple native read is removable", row.name, bank) + } + } + } +} + +func TestX86PackedMoveMaskedUnalignedReadIsObservable(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + if row.aligned || row.vexOnly { + continue + } + for axis, bank := range []string{"X", "Y", "Z"} { + width := 16 << axis + source := fmt.Sprintf("TEXT ·view(SB),4,$0-16\nMOVQ p+0(FP),AX\nMOVQ mask+8(FP),CX\nKMOVQ CX,K1\n%s.Z (AX),K1,%s0\nRET\n", row.name, bank) + ir, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte, mask uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "text") + if err != nil { + t.Fatal(err) + } + if got := strings.Count(ir, fmt.Sprintf("load volatile i%d,", row.bits)); got != width*8/row.bits { + t.Fatalf("%s %s masked dead read has %d observable lanes; want %d", row.name, bank, got, width*8/row.bits) + } + } + } +} + +func TestX86PackedMoveResultStoreAndStaticExtentBoundaries(t *testing.T) { + for _, row := range x86PackedMoveResultOracle { + for axis, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + width := 16 << axis + for _, destination := range []string{"ret+0(FP)", fmt.Sprintf("local-%d(SP)", width)} { + for _, masked := range []bool{false, true} { + if row.vexOnly && masked { + continue + } + mask := "" + if masked { + mask = "K1," + } + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s %s0,%s%s", row.name, bank, mask, destination)) + for _, oldKnown := range []bool{false, true} { + for missing := -1; missing < width; missing++ { + owned := make([]bool, width) + state := x86ResultState{fp: make([]bool, width), auto: make([]bool, width), masks: [8]bool{1: true}} + for i := range owned { + owned[i], state.fp[i], state.auto[i] = true, oldKnown, oldKnown + } + known := ^uint64(0) + if missing >= 0 { + known &^= uint64(1) << uint(missing) + } + state.writeVector(0, 64, known, false) + want := !row.aligned && missing < 0 && (!masked || oldKnown) + if got := x86ScalarFPResultTransfer(ins, &state, owned, 0, "amd64"); got != want { + t.Fatal("store borrowed source or inactive destination bytes", ins, oldKnown, missing, got, want) + } + } + } + } + } + ins := x86ScalarLaneInstruction(t, fmt.Sprintf("%s table<>+16(SB),%s1", row.name, bank)) + for _, available := range []int64{int64(width - 1), int64(width)} { + state := x86ResultState{loweredStaticReads: map[x86StaticSBAddress]int64{{symbol: "table<>", offset: 16}: available}} + if got := x86ScalarFPResultTransfer(ins, &state, nil, 0, "amd64"); got != (available == int64(width)) { + t.Fatal("static read ignored complete extent", row.name, bank, available, got) + } + } + } + } +} + +func TestX86PackedMoveResultSourceBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, move, triple := 8, "MOVQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + word, move, triple = 4, "MOVL", "i386-unknown-linux-gnu" + } + for _, row := range x86PackedMoveResultOracle { + load := move + " p+0(FP),AX\n" + read := row.name + " (AX),X1\n" + bodies := map[string]string{ + "unknown base": read, + "numeric base": "MOVL $1,AX\n" + read, + "lost identity": load + "MOVL $1,AX\n" + read, + "unknown index": load + row.name + " (AX)(DI*1),X1\n", + "pointer index": load + row.name + " (AX)(AX*1),X1\n", + "native store": load + read + row.name + " X1,(AX)\n", + "unknown static": row.name + " ·foreign(SB),X1\n", + "unknown path": load + "MOVL $1,CX\nCMPL CX,$0\nJE done\n" + read + "done:\n", + } + if !row.vexOnly { + bodies["undefined mask"] = load + row.name + ".Z (AX),K1,X1\n" + bodies["undefined merge"] = load + "MOVL $1,CX\nKMOVD CX,K1\n" + row.name + " (AX),K1,X1\n" + } + for name, body := range bodies { + source := fmt.Sprintf("TEXT ·view(SB),4,$0-%d\n%sPEXTRD $0,X1,CX\nMOVL CX,ret+%d(FP)\nRET\n", word+4, body, word) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, "func view(p *[64]byte) uint32\n", arch, triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("source closure or incoming identity bypass", arch, row.name, name, api, err) + } + } + } + } + } +} + +func TestX86PackedMoveResultRawOriginBoundaries(t *testing.T) { + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[64]byte) uint32", + "MOVQ p+0(FP),AX\nVMOVDQU64 (AX),Z1\nPEXTRD $0,X1,CX\nMOVL CX,ret0+8(FP)\nRET") + for _, mutation := range []string{"decoded", "address size", "raw text", "source stack", "fixed local", "segment", "symbol"} { + fn := file.Funcs[0] + fn.Instrs = append([]Instr(nil), fn.Instrs...) + fn.Instrs[2].Args = append([]Operand(nil), fn.Instrs[2].Args...) + switch mutation { + case "decoded": + fn.Instrs[2].x86Encoded = true + case "address size": + fn.Instrs[2].x86AddressBits = 32 + case "raw text": + fn.X86RawText = []byte{0xc3} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + case "segment": + fn.Instrs[2].Args[0].Mem.Segment = "FS" + case "symbol": + fn.Instrs[2].Args[0].Mem.Sym = "table<>(SB)" + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("raw/source role acquired ordinary conditional byte availability", mutation, err) + } + } +} + +func TestX86PackedMoveResultStaticSourceThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + var source, declarations strings.Builder + source.WriteString("DATA table<>+0(SB)/8,$0x76543210abcdef01\nGLOBL table<>(SB),8,$64\n") + count := 0 + for _, row := range x86PackedMoveResultOracle { + for _, bank := range []string{"X", "Y", "Z"} { + if row.vexOnly && bank == "Z" { + continue + } + addresses := []string{"table<>(SB)"} + if target.arch == "386" { + for _, annotation := range []string{"AX", "BX", "CX", "DX", "SI", "DI", "SP", "BP"} { + addresses = append(addresses, "table<>(SB)("+annotation+")") + } + } + for _, address := range addresses { + fmt.Fprintf(&declarations, "func view%d() uint32\n", count) + fmt.Fprintf(&source, "TEXT ·view%d(SB),4,$0-4\n%s %s,%s1\nPEXTRD $0,X1,AX\nMOVL AX,ret+0(FP)\nRET\n", count, row.name, address, bank) + count++ + } + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("complete DATA directory did not compose with physical reads", target, api, err) + } + compileLLVMToObject(t, llc, target.triple, "packed-data.ll", "packed-data.o", ir) + } + } +} diff --git a/amd64_packed_vector_move_runtime_partition_test.go b/amd64_packed_vector_move_runtime_partition_test.go new file mode 100644 index 000000000..bcbab4fe5 --- /dev/null +++ b/amd64_packed_vector_move_runtime_partition_test.go @@ -0,0 +1,91 @@ +package plan9asm + +import ( + "os" + "strings" + "testing" +) + +var x86PackedMoveRuntimeRows = []struct { + op string + bits int +}{ + {"VMOVAPS", 32}, {"VMOVAPD", 64}, {"VMOVUPS", 32}, {"VMOVUPD", 64}, + {"VMOVDQA32", 32}, {"VMOVDQA64", 64}, {"VMOVDQU8", 8}, {"VMOVDQU16", 16}, {"VMOVDQU32", 32}, {"VMOVDQU64", 64}, +} + +// Test-only ISA partition. An aligned register copy stays portable; only its +// original physical memory instruction requires the native counterpart. +func x86PackedMoveRuntimeNativeRow(op string, memory bool) bool { + if !memory { + return false + } + switch op { + case "VMOVDQA", "VMOVAPS", "VMOVAPD", "VMOVDQA32", "VMOVDQA64": + return true + } + return false +} + +func TestX86PackedMoveRuntimePartitionIsComplete(t *testing.T) { + // These are the exact historical rows, independent of the implementation + // registry. Every old row must have one owner, never an execution skip. + oldRows := map[string]int{"VMOVAPS": 32, "VMOVAPD": 64, "VMOVUPS": 32, "VMOVUPD": 64, "VMOVDQA32": 32, "VMOVDQA64": 64, "VMOVDQU8": 8, "VMOVDQU16": 16, "VMOVDQU32": 32, "VMOVDQU64": 64} + var masked, views, frame [2]int + seen := make(map[string]bool) + for _, row := range x86PackedMoveRuntimeRows { + name := row.op + if seen[name] || oldRows[name] != row.bits { + t.Fatal("unknown or duplicate historical runtime row", row) + } + seen[name] = true + aligned := name == "VMOVAPS" || name == "VMOVAPD" || name == "VMOVDQA32" || name == "VMOVDQA64" + if aligned { + frame[1]++ // Actual-Go accepted; virtual FP must report Context. + } else { + frame[0]++ // Numeric portable FP oracle. + } + for width := 0; width < 3; width++ { + owner := 0 + if x86PackedMoveRuntimeNativeRow(name, true) { + owner = 1 + } + if (owner == 1) != aligned { + t.Fatal("incorrect historical masked-memory owner", name, width) + } + masked[owner]++ + for form := 0; form < 6; form++ { + owner = 0 + if x86PackedMoveRuntimeNativeRow(name, form < 3) { + owner = 1 + } + if (owner == 1) != (aligned && form < 3) { + t.Fatal("incorrect historical view owner", name, width, form) + } + views[owner]++ + } + } + } + if len(seen) != len(oldRows) || masked != [2]int{18, 12} || views != [2]int{144, 36} || frame != [2]int{6, 4} { + t.Fatal("lost old rows", masked, views, frame) + } + for file, roots := range map[string][]string{ + "amd64_lower_packed_move_memory_test.go": {"TestX86PackedMoveMaskedMemoryRuntime", "TestX86PackedMoveRegisterViewsRuntime"}, + "amd64_packed_vector_move_aligned_runtime_linux_amd64_test.go": { + "TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedMaskedMemory", "TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedRegisterViews", "TestCrossLinuxRuntimeMatrixX86PackedMoveAlignedOriginalGo"}, + } { + source, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(source), "t.Skip") { + t.Fatal("runtime partition silently skips rows", file) + } + for _, root := range roots { + if !strings.Contains(string(source), "func "+root+"(t *testing.T)") { + t.Fatal("missing required runtime counterpart", root) + } + } + } + t.Log("historical memory rows: portable=18 native=12; alias rows: portable=144 native=36; FP rows: portable=6 explicit Context=4; all native roots are required cross-runtime tests") +} diff --git a/amd64_packed_word_multiply_forms.go b/amd64_packed_word_multiply_forms.go new file mode 100644 index 000000000..b361c2d71 --- /dev/null +++ b/amd64_packed_word_multiply_forms.go @@ -0,0 +1,86 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedWordMultiplySpec struct { + mode amd64PackedWordMultiplyMode + encoding x86PackedLogicalEncoding +} + +var x86PackedWordMultiplySpecs = map[Op]x86PackedWordMultiplySpec{ + "PMULLW": {amd64PackedWordMultiplyLow, x86PackedLogicalLegacyInteger}, + "PMULHW": {amd64PackedWordMultiplySignedHigh, x86PackedLogicalLegacyInteger}, + "PMULHUW": {amd64PackedWordMultiplyUnsignedHigh, x86PackedLogicalLegacyInteger}, + "PMULHRSW": {amd64PackedWordMultiplyRoundedSignedHigh, x86PackedLogicalLegacyFP}, + "VPMULLW": {amd64PackedWordMultiplyLow, x86PackedLogicalVEXOrEVEX}, + "VPMULHW": {amd64PackedWordMultiplySignedHigh, x86PackedLogicalVEXOrEVEX}, + "VPMULHUW": {amd64PackedWordMultiplyUnsignedHigh, x86PackedLogicalVEXOrEVEX}, + "VPMULHRSW": {amd64PackedWordMultiplyRoundedSignedHigh, + x86PackedLogicalVEXOrEVEX}, +} + +type x86PackedWordMultiplyForm struct { + x86PackedBinaryForm + spec x86PackedWordMultiplySpec + source Operand // Actual encoded bank, not a new initialized input. +} + +// This shared grammar records the real native read/modify/write shape only. +// It grants neither input initialization nor native backing/caller authority. +func x86PackedWordMultiplyProperties(op Op, ins Instr, arch string) (x86PackedWordMultiplyForm, bool, error) { + base, _, _ := strings.Cut(strings.ToUpper(string(op)), ".") + spec, recognized := x86PackedWordMultiplySpecs[Op(base)] + if !recognized { + return x86PackedWordMultiplyForm{}, false, nil + } + if ins.Op != "" && !strings.EqualFold(string(ins.Op), string(op)) { + return x86PackedWordMultiplyForm{}, true, fmt.Errorf("packed word multiply dispatch/source opcode mismatch: %s/%s", op, ins.Op) + } + local := ins + local.Op = op + local.Args = append([]Operand(nil), ins.Args...) + if spec.encoding == x86PackedLogicalLegacyInteger { + var err error + local, err = x86LegacyMMXReadInstruction(local, arch) + if err != nil { + return x86PackedWordMultiplyForm{}, true, err + } + } + grammarArch := arch + // Only the actual physical decoder sets this private marker. A physical + // mode32 EVEX instruction is not subject to Go's named 3-operand limit. + if arch == "386" && local.x86Encoded && spec.encoding == x86PackedLogicalVEXOrEVEX { + grammarArch = "amd64" + } + binary, err := x86PackedBinaryProperties(local, grammarArch, x86PackedBinarySpec{ + encoding: spec.encoding, zeroing: spec.encoding == x86PackedLogicalVEXOrEVEX, + }) + form := x86PackedWordMultiplyForm{x86PackedBinaryForm: binary, spec: spec} + if err == nil { + form.source = local.Args[binary.first] + if arch == "386" && grammarArch != arch && binary.memory >= 0 && !x86PackedShiftMemoryValid(form.source, arch) { + err = fmt.Errorf("386 %s physical source has an unbound address", op) + } + if err == nil { + err = x86Mode32VectorRegisterContext(local, arch) + } + if err == nil && binary.memory >= 0 && (ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride) { + err = fmt.Errorf("%w: packed word multiply has no bound address-size override", ErrProbeNeedsContext) + } + } + return form, true, err +} + +func x86TypedPackedWordMultiplyByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + form, recognized, err := x86PackedWordMultiplyProperties(op, ins, arch) + if !recognized || err != nil || !form.legacy { + return x86TypedByteEffect{}, recognized && (err != nil || form.legacy), err + } + return x86TypedByteEffect{operands: []x86TypedByteOperand{ + {index: form.first, operand: form.source, width: int64(form.byteWidth), read: true}, + {index: form.destination, operand: ins.Args[form.destination], width: int64(form.byteWidth), read: true, write: true}, + }}, true, nil +} diff --git a/amd64_packed_word_multiply_forms_test.go b/amd64_packed_word_multiply_forms_test.go new file mode 100644 index 000000000..6dc8dcd9e --- /dev/null +++ b/amd64_packed_word_multiply_forms_test.go @@ -0,0 +1,537 @@ +package plan9asm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go/types" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +var x86WordMultiplyNames = []string{ + "PMULLW", "PMULHW", "PMULHUW", "PMULHRSW", + "VPMULLW", "VPMULHW", "VPMULHUW", "VPMULHRSW", +} + +// This catalog is derived from the actual Go ymm, yxm_q4 and _yvandnpd +// rows, independently of the production specification. It includes valid +// syntax which requires a different memory or physical input contract. +func x86WordMultiplyForms(arch string) []string { + var rows []string + for _, name := range x86WordMultiplyNames[:4] { + for _, source := range []string{"X0", "X7", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-16(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, name+" "+source+",X1") + } + if arch == "amd64" { + rows = append(rows, name+" X15,X0") + if name != "PMULHRSW" { + for _, source := range []string{"M0", "M7", "(AX)", "value+0(FP)", "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} { + rows = append(rows, name+" "+source+",M1") + } + } + } + } + for _, name := range x86WordMultiplyNames[4:] { + for _, bank := range []string{"X", "Y", "Z"} { + for _, source := range []string{bank + "0", bank + "7", "(AX)", "16(AX)(CX*4)", "value+0(FP)", "local-64(SP)", "·data(SB)", "(FS)", "(GS)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s1,%s2", name, source, bank, bank)) + if arch == "amd64" { + rows = append(rows, fmt.Sprintf("%s %s,%s31,K1,%s30", name, source, bank, bank), + fmt.Sprintf("%s.Z %s,%s31,K7,%s30", name, source, bank, bank)) + } + } + if arch == "amd64" || bank != "Z" { + rows = append(rows, fmt.Sprintf("%s %s20,%s21,%s22", name, bank, bank, bank)) + } + } + } + return rows +} + +func TestX86PackedWordMultiplyCompleteActualGoAttributes(t *testing.T) { + if len(x86PackedWordMultiplySpecs) != 8 { + t.Fatal("complete named packed-word multiply family must contain eight names") + } + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, row := range []string{ + "{APMULLW, ymm, Py1, opBytes{0xd5, Pe, 0xd5}}", + "{APMULHW, ymm, Py1, opBytes{0xe5, Pe, 0xe5}}", + "{APMULHUW, ymm, Py1, opBytes{0xe4, Pe, 0xe4}}", + "{APMULHRSW, yxm_q4, Pq4, opBytes{0x0b}}", + "{Zm_r_xm, 1, argList{Ymm, Ymr}}", "{Zm_r_xm, 2, argList{Yxm, Yxr}}", + "{Zm_r, 1, argList{Yxm, Yxr}}", + } { + if !strings.Contains(string(legacy), row) { + t.Fatalf("actual Go complete legacy row changed: %s", row) + } + } + for _, class := range []string{"Yax", "Ycx", "Yrx", "Yrl", "Yrl32", "Ym", "Ymr"} { + if !strings.Contains(string(legacy), "ycover["+class+"*Ymax+Ymm] = 1") { + t.Fatal("actual Go MMX source class changed", class) + } + } + for _, classes := range []string{ + "Yxm, Yxr, Yxr", "Yym, Yyr, Yyr", + "YxmEvex, YxrEvex, YxrEvex", "YxmEvex, YxrEvex, Yknot0, YxrEvex", + "YymEvex, YyrEvex, YyrEvex", "YymEvex, YyrEvex, Yknot0, YyrEvex", + "Yzm, Yzr, Yzr", "Yzm, Yzr, Yknot0, Yzr", + } { + start := strings.Index(string(avx), "var _yvandnpd =") + table := strings.SplitN(string(avx)[start:], "\n}", 2)[0] + if !strings.Contains(table, "argList{"+classes+"}") || strings.Count(table, "argList{") != 8 { + t.Fatal("actual Go complete VEX/EVEX class rows changed", classes) + } + } + for _, row := range x86PackedWordMultiplyGoRows { + start := strings.Index(string(avx), "{as: A"+string(row.op)+",") + if start < 0 { + t.Fatal("actual Go omitted opcode", row.op) + } + record := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + if !strings.Contains(record, "ytab: _yvandnpd, prefix: Pavx") || + strings.Contains(record, "Bcst") || strings.Contains(record, "Sae") { + t.Fatal("actual Go opcode-specific attribute changed", record) + } + mapName := "0F" + if row.mapID == 2 { + mapName = "0F38" + } + for _, width := range []string{"128", "256"} { + want := fmt.Sprintf("avxEscape | vex%s | vex66 | vex%s | vexW0, 0x%02X", width, mapName, row.opcode) + if !strings.Contains(record, want) { + t.Fatal("actual Go VEX encoding changed", want) + } + } + for index, width := range []string{"128", "256", "512"} { + want := fmt.Sprintf("avxEscape | evex%s | evex66 | evex%s | evexW0, evexN%d | evexZeroingEnabled, 0x%02X", width, mapName, 16<= 3: + registerIndex = (index - 3) & 7 + } + if len(code) == 5 && code[0] == 0x41 { + code = code[1:] // Actual Go's ignored MMX REX.B annotation. + } + want := []byte{0x0f, row.opcode, byte(0xc8 | registerIndex), 0xc3} + if !bytes.Equal(code, want) { + t.Fatalf("%s actual encoded source bank=%x want=%x", source, code, want) + } + } + } +} + +func TestX86PackedWordMultiplyMalformedIdentityAndBanksBeforeIR(t *testing.T) { + for _, name := range x86WordMultiplyNames { + vector := strings.HasPrefix(name, "V") + operands := "X0,X1" + if vector { + operands = "X0,X1,X2" + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+name+" "+operands+"\nRET\n") + if err != nil { + t.Fatal(err) + } + original := file.Funcs[0].Instrs[1] + for _, identity := range []Op{"", original.Op, Op(strings.ToLower(name)), "MOVQ", Op(name + ".BAD")} { + ins := original + ins.Op = identity + copy := ins + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before, temporary := out.String(), ctx.tmp + known, term, err := ctx.lowerPackedWordMultiply(original.Op, ins) + bad := identity != "" && !strings.EqualFold(string(identity), name) + if !known || term || (err != nil) != bad || bad && (out.String() != before || ctx.tmp != temporary) || !reflect.DeepEqual(ins, copy) { + t.Fatal("dispatch mismatch/clone contract", name, identity, known, term, err) + } + } + bad := []string{"$1,X1", "Y0,Y1", "X16,X0", "X0,(AX)", "(AX),(BX)", "(AX)(SP*2),X1"} + if vector { + bad = []string{"$1,X1,X2", "X0,Y1,Y2", "X0,X1,(AX)", "X0,X1,K0,X2", "(AX)(SP*2),X1,X2", "2147483648(AX),X1,X2"} + } + for _, operands := range bad { + source := "TEXT f(SB),4,$0-0\n" + name + " " + operands + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + ctx, out := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := out.String() + if known, _, err := ctx.lowerPackedWordMultiply(ins.Op, ins); !known || err == nil || out.String() != before { + t.Fatal("malformed form emitted IR", ins, err) + } + } + if vector { + for _, suffix := range []string{".BCST", ".SAE", ".RN_SAE", ".Z"} { + source := "TEXT f(SB),4,$0-0\n" + name + suffix + " (AX),X1,X2\nRET\n" + if suffix == ".Z" { + x86WordMultiplyReservedZeroMaskGoObservation(t, source, name) + } else { + goAssemblerOracleOutput(t, "amd64", source, "", false) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, _, err := x86PackedWordMultiplyProperties(file.Funcs[0].Instrs[1].Op, file.Funcs[0].Instrs[1], "amd64"); err == nil { + t.Fatal("opcode attributes leaked from shared ytab", source) + } + } + } + } +} + +func TestX86PackedWordMultiplyThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 required") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + arch := "amd64" + if strings.HasPrefix(triple, "i") { + arch = "386" + } + t.Run(triple, func(t *testing.T) { + var source, declarations strings.Builder + declarations.WriteString("package oracle\n") + for i, row := range x86WordMultiplyForms(arch) { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$64-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, arch) + if errors.Is(err, ErrProbeNeedsContext) || form.mmx { + continue + } + if err != nil { + t.Fatal(err) + } + if form.legacy && form.memory >= 0 && (form.source.Kind == OpFP || + form.source.Kind == OpMem && form.source.Mem.Base == SP) { + continue + } + fmt.Fprintf(&source, "TEXT ·F%d(SB),4,$64-64\n%s\nRET\n", i, row) + fmt.Fprintf(&declarations, "func F%d(value [64]byte)\n", i) + } + goAssemblerOracleOutput(t, arch, source.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), arch, types.SizesFor("gc", arch)) + if err != nil { + t.Fatal(err) + } + sigs[name] = sig + } + options := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, ir, "word-multiply-text") + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, module.String(), "word-multiply-module") + translated, err := translateGoModuleInContext(ctx, pkg, []byte(source.String()), GoModuleOptions{ + GOARCH: arch, TargetTriple: triple, ResolveSym: options.ResolveSym}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, triple, translated.Module.String(), "word-multiply-go") + x86ResultInitializationObject(t, llc, triple, x86MulDivRoundOptimize(t, ir), "word-multiply-O2") + }) + } +} + +func TestX86PackedWordMultiplyMode32PhysicalAndSourceBoundaries(t *testing.T) { + for _, name := range x86WordMultiplyNames[4:] { + for _, bank := range []string{"X", "Y"} { + for _, index := range []int{8, 15, 16, 31} { + for operand := 0; operand < 3; operand++ { + args := []string{bank + "0", bank + "1", bank + "2"} + args[operand] = fmt.Sprintf("%s%d", bank, index) + source := "TEXT f(SB),4,$0-0\n" + name + " " + strings.Join(args, ",") + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + if _, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, "386"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("named high bank lost physical Context", ins, err) + } + } + } + } + for _, source := range []string{ + name + " Z8,Z1,Z2", name + " Z0,Z8,Z2", name + " Z0,Z1,Z8", + name + " X0,X1,K1,X2", name + ".Z Z0,Z1,K1,Z2", + } { + assembly := "TEXT f(SB),4,$0-0\n" + source + "\nRET\n" + goAssemblerOracleOutput(t, "386", assembly, "", false) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + continue + } + ins := file.Funcs[0].Instrs[1] + if _, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, "386"); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("Go rejected a source format, not merely an unknown physical identity", source, err) + } + } + } + for _, row := range x86PackedWordMultiplyGoRows { + for _, width := range []int{0, 1, 2} { + for _, zero := range []bool{false, true} { + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, width, 1, false, zero, 2, 1, 0) + ins, n, known, err := decodedX86PackedMultiplyInstruction(code, 32) + if !known || err != nil || n != len(code) || !ins.x86Encoded { + t.Fatal("actual low physical mode32 EVEX decode lost its origin", ins, err) + } + if _, _, err := x86PackedWordMultiplyProperties(ins.Op, ins, "386"); err != nil { + t.Fatal("physical grammar borrowed textual operand limit", err) + } + textual := ins + textual.x86Encoded = false + textual.Raw = fmt.Sprintf("BYTE %x genuine-looking text is not a decoder", code) + if _, _, err := x86PackedWordMultiplyProperties(textual.Op, textual, "386"); err == nil { + t.Fatal("raw text manufactured a decoder identity") + } + encoded, err := json.Marshal(ins) + if err != nil { + t.Fatal(err) + } + var restored Instr + if err := json.Unmarshal(encoded, &restored); err != nil { + t.Fatal(err) + } + if restored.x86Encoded { + t.Fatal("public JSON manufactured a decoder marker") + } + if _, _, err := x86PackedWordMultiplyProperties(restored.Op, restored, "386"); err == nil { + t.Fatal("JSON-restored source bypassed Go's named limit") + } + } + } + } +} + +func TestX86PackedWordMultiplyLegacyResultBytesAndUnknownInputs(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, name := range x86WordMultiplyNames[:4] { + for _, first := range []string{"X0", "a+0(FP)", "value-16(SP)"} { + for _, complete := range []bool{true, false} { + move := "MOVOU" + if !complete { + move = "MOVSS" + } + body := move + " a+0(FP),X0\nMOVOU b+16(FP),X1\n" + move + " X0,value-16(SP)\n" + name + " " + first + ",X1\nMOVSD X1,ret0+32(FP)\nRET" + if !complete { + // Memory MOVSS zeros X[127:32]; its register form instead + // preserves the destination's genuinely undefined bytes. + body = strings.Replace(body, "MOVSS a+0(FP),X0", "MOVSS a+0(FP),X2\nMOVSS X2,X0", 1) + } + _, _, sig, source := x86ResultInitializationFixture(t, arch, "func Value(a,b [16]byte) uint64", body) + source = strings.Replace(source, ",$0-", ",$32-", 1) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + goAssemblerOracleOutput(t, arch, source, "", true) + err = CheckX86ABI0ResultInitialization(file.Funcs[0], sig, arch) + wantContext := !complete && first != "a+0(FP)" + if wantContext && !errors.Is(err, ErrProbeNeedsContext) || !wantContext && err != nil { + t.Fatal("legacy result byte proof invented unknown word input", name, first, complete, err) + } + } + } + } + } +} + +func TestX86PackedWordMultiplyVirtualLegacyMemoryRemainsContext(t *testing.T) { + for _, name := range x86WordMultiplyNames[:4] { + for _, address := range []string{"value+0(FP)", "(SP)", "(BP)", "(AX)(BP*2)"} { + source := "TEXT f(SB),4,$64-64\n" + name + " " + address + ",X0\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, arch := range []string{"amd64", "386"} { + _, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"f": x86WordMaddFPType(t)}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("virtual legacy input obtained native storage", source, arch, err) + } + } + } + } +} + +func TestX86PackedWordMultiplyMode32LowZActualGo(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, name := range x86WordMultiplyNames[4:] { + for _, first := range []string{"Z0", "(AX)"} { + row := name + " " + first + ", Z1, Z2" + t.Run(row, func(t *testing.T) { + source := "TEXT wordmul(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "386", source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"wordmul": {Name: "wordmul", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + x86ResultInitializationObject(t, llc, "i386-unknown-linux-gnu", ir, "word-multiply-low-z") + }) + } + } +} + +func TestX86PackedWordMultiplyLegacyByteEffectActualGo(t *testing.T) { + for _, name := range x86WordMultiplyNames[:4] { + for _, arch := range []string{"386", "amd64"} { + row := name + " X0, X1" + t.Run(arch+"/"+name, func(t *testing.T) { + source := "TEXT wordmul(SB),4,$0-0\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + effect, known, err := x86TypedInstructionByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 { + t.Fatalf("accepted legacy format lacks complete typed byte effect: known=%t err=%v effect=%+v", known, err, effect) + } + if !effect.operands[0].read || effect.operands[0].write || !effect.operands[1].read || + !effect.operands[1].write || effect.operands[0].width != 16 || effect.operands[1].width != 16 || + effect.flagsDefined != 0 || effect.flagsKilled != 0 { + t.Fatal(fmt.Sprintf("incorrect read/modify/write/flags effect: %+v", effect)) + } + }) + } + } +} diff --git a/amd64_packed_word_multiply_native_linux_amd64_test.go b/amd64_packed_word_multiply_native_linux_amd64_test.go new file mode 100644 index 000000000..39ff70e3a --- /dev/null +++ b/amd64_packed_word_multiply_native_linux_amd64_test.go @@ -0,0 +1,80 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// This is a required Linux/amd64 runtime axis selected by the cross-runtime +// TestCrossLinuxRuntimeMatrix prefix. Missing AVX/AVX2 is a failure, not a skip. +// Darwin/Rosetta object/software results cannot satisfy this original-Go axis. +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplyOriginalVEX(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + execution.requireAVX(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\nfunc ReadCSR() uint32\nfunc SetCSR(value uint32)\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\nTEXT ·SetCSR(SB),4,$0-4\nLDMXCSR value+0(FP)\nRET\n") + tests.WriteString(x86WordMultiplyOriginalGoTestHeader) + count := 0 + for _, name := range x86WordMultiplyNames[4:] { + for _, width := range []int{16, 32} { + bank := map[int]string{16: "X", 32: "Y"}[width] + for _, memory := range []bool{false, true} { + for destination := 0; destination < 3; destination++ { + fn := fmt.Sprintf("V%d", count) + count++ + source := bank + "0" + if memory { + source = "(DI)" + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed uint64,profile uint32)\n", fn) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-56 +MOVQ out+0(FP),DX +MOVQ a+8(FP),DI +MOVQ b+16(FP),SI +MOVQ flags+24(FP),CX +MOVQ csr+32(FP),BX +VMOVDQU (DI),Y0 +VMOVDQU (SI),Y1 +VMOVDQU (SI),Y2 +LDMXCSR profile+48(FP) +PUSHQ seed+40(FP) +POPFQ +%s %s,%s1,%s%d +VMOVDQU Y%d,(DX) +PUSHFQ +POPQ AX +MOVQ AX,(CX) +STMXCSR (BX) +VZEROUPPER +RET +`, fn, name, source, bank, bank, destination, destination) + fmt.Fprintf(&tests, `{ + for i:=range out {out[i]=0xc3} + flags:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=struct{before,value,after uint32}{0x11223344,0,0x44332211} + %s(op,ap,bp,&flags.value,&csr.value,seed,profile) + if flags.value&0x8d5!=seed||flags.before!=0x1122334455667788||flags.after!=0x8877665544332211||csr.value!=profile||csr.before!=0x11223344||csr.after!=0x44332211||ReadCSR()!=profile||a!=originalA||b!=originalB {t.Fatalf("%s state/canary sample%%d",sample)} + for i,value:=range out { + want:=byte(0xc3) + if i>=offset&&i>uint((index&1)*8)) + } + } + if value!=want {t.Fatalf("%s full-byte/upper-zero/canary sample%%d byte%%d got%%x want%%x",sample,i,value,want)} + } +} +`, fn, fn, width, x86WordMultiplyReferenceMode(name), fn) + } + } + } + } + tests.WriteString("}}\n}\n") + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests.String()) + t.Logf("%d actual original VEX X/Y kernels ×257 samples ×2 six-flag seeds; register/memory and all destination aliases, full Y upper-zero/MXCSR/input/canaries. No original EVEX execution claim.", count) +} diff --git a/amd64_packed_word_multiply_runtime_test.go b/amd64_packed_word_multiply_runtime_test.go new file mode 100644 index 000000000..cf01e70ae --- /dev/null +++ b/amd64_packed_word_multiply_runtime_test.go @@ -0,0 +1,454 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func x86WordMultiplyReservedZeroMaskGoObservation(t *testing.T, source, name string) { + t.Helper() + dir := t.TempDir() + asm, object := filepath.Join(dir, "reserved.s"), filepath.Join(dir, "reserved.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "tool", "asm", "-S", "-p", "oracle", "-o", object, asm) + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(asm, object, output, err, ctx.Err(), err == nil, "amd64"); failure != nil { + t.Fatalf("actual zero-mask source observation: %v\n%s", failure, output) + } + if err != nil { + if !bytes.Contains(output, []byte("mask register must be specified for .Z instructions")) { + t.Fatalf("unknown original-Go rejection: %s", output) + } + return + } + // Go1.20 can emit the reserved EVEX.z=1/aaa=0 pattern. This is an + // accepted object observation, never authority to execute or translate it. + opcode := map[string]byte{"VPMULLW": 0xd5, "VPMULHW": 0xe5, "VPMULHUW": 0xe4, "VPMULHRSW": 0x0b}[name] + mapByte := byte(0xf1) + if name == "VPMULHRSW" { + mapByte = 0xf2 + } + code := []byte{0x62, mapByte, 0x75, 0x88, opcode, 0x10} + listing := make([]string, 0, len(code)+1) + for _, value := range append(append([]byte(nil), code...), 0xc3) { + listing = append(listing, fmt.Sprintf("%02x", value)) + } + if !bytes.Contains(output, []byte(strings.Join(listing, " "))) { + t.Fatalf("actual accepted reserved bytes changed: %s", output) + } + if _, _, recognized, err := decodedX86PackedMultiplyInstruction(code, 64); !recognized || err == nil { + t.Fatal("reserved physical bytes acquired valid semantics") + } + t.Log("actual Go accepts reserved zero-mask bytes; named/raw validation still rejects, and no #UD code is executed") +} + +// Independent integer arithmetic: neither reference reads a production mode, +// emitter, descriptor or LLVM intrinsic. Rounded signed high is deliberately +// wrapping, including (-32768)*(-32768) -> 0x8000, not saturation. +const x86WordMultiplyGoReference = `func product(a,b uint16,mode int) uint16 { + if mode==0 {return uint16(uint32(a)*uint32(b))} + if mode==2 {return uint16((uint32(a)*uint32(b))>>16)} + x,y:=int64(int16(a)),int64(int16(b)) + value,shift:=x*y,uint(16) + if mode==3 {value+=16384;shift=15} + return uint16(value>>shift) +} +` + +const x86WordMultiplyOriginalGoTestHeader = `package oracle +import ("testing";"unsafe";"runtime") +` + x86WordMultiplyGoReference + ` +func aligned(value *[160]byte) (*[64]byte,int) { + offset:=16+int((-uintptr(unsafe.Pointer(&value[16])))&15) + return (*[64]byte)(unsafe.Pointer(&value[offset])),offset +} +func TestOriginal(t *testing.T) { + runtime.LockOSThread();defer runtime.UnlockOSThread() + edges:=[...]uint16{0,1,2,0x3fff,0x4000,0x4001,0x7fff,0x8000,0x8001,0xbfff,0xc000,0xfffe,0xffff} + saved:=ReadCSR();defer SetCSR(saved) + for sample:=0;sample<257;sample++ {for _,seed:=range []uint64{0,0x8d5} { + var a,b,out [160]byte + ap,_:=aligned(&a);bp,_:=aligned(&b);op,offset:=aligned(&out) + for i:=range a {a[i]=0xa7;b[i]=0xb9} + for i:=0;i<64;i+=2 { + x,y:=edges[(sample+i/2)%13],edges[(sample/13+i/2*3)%13] + if sample>169 {x=uint16(sample*71+i*7919);y=uint16(sample*13+i*37)} + ap[i],ap[i+1]=byte(x),byte(x>>8);bp[i],bp[i+1]=byte(y),byte(y>>8) + } + originalA,originalB:=a,b + profile:=uint32(0x1f80|((sample%4)<<13)|(sample%64)) + SetCSR(profile) +` + +const x86WordMultiplyCReference = `#include +#include +#include +#include +static uint16_t word(const uint8_t *p) {return (uint16_t)p[0]|(uint16_t)p[1]<<8;} +static uint16_t product(uint16_t a,uint16_t b,int mode) { + if(mode==0) return (uint16_t)((uint32_t)a*(uint32_t)b); + if(mode==2) return (uint16_t)(((uint32_t)a*(uint32_t)b)>>16); + int64_t x=a<32768?a:(int64_t)a-65536,y=b<32768?b:(int64_t)b-65536; + int64_t value=x*y,divisor=65536; + if(mode==3) {value+=16384;divisor=32768;} + // Mathematical floor division, independent of C's signed-shift policy. + return (uint16_t)(value<0?-((-value+divisor-1)/divisor):value/divisor); +} +static const uint16_t edge[]={0,1,2,0x3fff,0x4000,0x4001,0x7fff,0x8000,0x8001,0xbfff,0xc000,0xfffe,0xffff}; +static const uint64_t masks[]={0,1,UINT64_C(0xaaaaaaaa55555555),UINT64_MAX}; +struct Aligned {_Alignas(64) uint8_t before[64],value[64],after[64];}; +typedef void (*Program)(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t); +static int check(Program program,const char *name,int width,int legacy,int alias,int mode,int masking) { + for(int sample=0;sample<257;sample++) for(int status=0;status<2;status++) for(int m=0;m<4;m++) { + struct Aligned a,b,out; + memset(&a,0xa7,sizeof(a));memset(&b,0xb9,sizeof(b));memset(&out,0xc3,sizeof(out)); + for(int i=0;i<64;i+=2) { + uint16_t x=edge[(sample+i/2)%13],y=edge[(sample/13+i/2*3)%13]; + if(sample>169) {x=(uint16_t)(sample*71+i*7919);y=(uint16_t)(sample*13+i*37);} + a.value[i]=(uint8_t)x;a.value[i+1]=(uint8_t)(x>>8); + b.value[i]=(uint8_t)y;b.value[i+1]=(uint8_t)(y>>8); + } + struct Aligned originalA=a,originalB=b; + struct {uint64_t before,value,after;} flags={UINT64_C(0x1122334455667788),0,UINT64_C(0x8877665544332211)}; + struct {uint32_t before,value,after;} csr={0x11223344,0,0x44332211}; + uint32_t saved=_mm_getcsr(),profile=(uint32_t)(0x1f80|((sample%4)<<13)|(sample%64)); + _mm_setcsr(profile); + uint64_t seed=status?UINT64_C(0x8d5):0; + program(out.value,a.value,b.value,&flags.value,&csr.value,seed,masks[m]); + _mm_setcsr(saved); + if((flags.value&UINT64_C(0x8d5))!=seed||flags.before!=UINT64_C(0x1122334455667788)||flags.after!=UINT64_C(0x8877665544332211)|| + csr.value!=profile||csr.before!=0x11223344||csr.after!=0x44332211||memcmp(&a,&originalA,sizeof(a))||memcmp(&b,&originalB,sizeof(b))) { + fprintf(stderr,"%s state sample%d flags%llx CSR%x/%x\n",name,sample,(unsigned long long)flags.value,csr.value,profile);return 1; + } + for(int i=0;i<64;i++) { + uint16_t value=product(word((alias?b.value:a.value)+(i&~1)),word(b.value+(i&~1)),mode); + int active=!masking||((masks[m]>>(i/2))&1); + uint8_t want=i>((i&1)*8)):(masking==2?0:b.value[i])):(legacy?b.value[i]:0); + if(out.value[i]!=want||out.before[i]!=0xc3||out.after[i]!=0xc3) { + fprintf(stderr,"%s sample%d mask%llx byte%d got%x want%x\n",name,sample,(unsigned long long)masks[m],i,out.value[i],want);return 2; + } + } + } + return 0; +} +` + +func x86WordMultiplyReferenceMode(name string) int { + switch strings.TrimPrefix(name, "V") { + case "PMULLW": + return 0 + case "PMULHW": + return 1 + case "PMULHUW": + return 2 + case "PMULHRSW": + return 3 + default: + panic("unknown independent word multiply reference name") + } +} + +func x86WordMultiplySignature(t *testing.T, pkg GoPackage, name string) FuncSig { + t.Helper() + sig, err := DeriveGoFuncABI0Signature(name, pkg.Types.Scope().Lookup(name).(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + return sig +} + +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplyOriginalGo(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, tests strings.Builder + declarations.WriteString("package oracle\nfunc ReadCSR() uint32\nfunc SetCSR(value uint32)\n") + assembly.WriteString("TEXT ·ReadCSR(SB),4,$0-4\nSTMXCSR ret0+0(FP)\nRET\nTEXT ·SetCSR(SB),4,$0-4\nLDMXCSR value+0(FP)\nRET\n") + tests.WriteString(x86WordMultiplyOriginalGoTestHeader) + count := 0 + for _, name := range x86WordMultiplyNames[:4] { + for _, bank := range []string{"X", "M"} { + if bank == "M" && name == "PMULHRSW" { + continue + } + kinds := []string{"Register", "Alias", "Memory"} + if bank == "M" { + kinds = append(kinds, "AXBank", "R14Bank") + } + for _, kind := range kinds { + fn := fmt.Sprintf("F%d", count) + count++ + move, width, source := "MOVOU", 16, bank+"0" + if bank == "M" { + move, width = "MOVQ", 8 + } + alias := 0 + if kind == "Alias" { + source, alias = bank+"1", 1 + } + if kind == "Memory" { + source = "(DI)" + } + if kind == "AXBank" { + source = "AX" + } + if kind == "R14Bank" { + source = "R14" + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed uint64,profile uint32)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\n%s (DI),%s0\n%s (SI),%s1\n", fn, move, bank, move, bank) + if kind == "R14Bank" { + assembly.WriteString("MOVQ (DI),M6\n") + } + fmt.Fprintf(&assembly, "LDMXCSR profile+48(FP)\nPUSHQ seed+40(FP)\nPOPFQ\n%s %s,%s1\n%s %s1,(DX)\n", name, source, bank, move, bank) + if bank == "M" { + assembly.WriteString("EMMS\n") + } + assembly.WriteString("PUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n") + fmt.Fprintf(&tests, `{ + for i:=range out {out[i]=0xc3} + flags:=struct{before,value,after uint64}{0x1122334455667788,0,0x8877665544332211} + csr:=struct{before,value,after uint32}{0x11223344,0,0x44332211} + %s(op,ap,bp,&flags.value,&csr.value,seed,profile) + if flags.value&0x8d5!=seed||flags.before!=0x1122334455667788||flags.after!=0x8877665544332211||csr.value!=profile||csr.before!=0x11223344||csr.after!=0x44332211||ReadCSR()!=profile||a!=originalA||b!=originalB {t.Fatalf("%s flags/CSR/input/canary sample%%d flags%%x csr%%x/%%x",sample,flags.value,csr.value,profile)} + for i,value:=range out { + want:=byte(0xc3) + if i>=offset&&i>uint((index&1)*8)) + } + if value!=want {t.Fatalf("%s full-byte/canary sample%%d byte%%d got%%x want%%x",sample,i,value,want)} + } +} +`, fn, fn, width, alias, x86WordMultiplyReferenceMode(name), fn) + } + } + } + tests.WriteString("}}\n}\n") + x86WordMultiplyRunOriginalGo(t, execution, declarations.String(), assembly.String(), tests.String()) + t.Logf("%d actual original SSE/MMX kernels ×257 initialized samples ×2 all-six-flag seeds, full MXCSR, bytes/input/canaries; MMX GP spellings observe the physical bank only, not a new LLVM permission. Rosetta is not native AMD64 evidence.", count) +} + +func x86WordMultiplyRunOriginalGo(t *testing.T, execution x86FloatCompareExecution, declarations, assembly, tests string) { + t.Helper() + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + dir := t.TempDir() + for name, contents := range map[string]string{"go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations, "value_amd64.s": assembly, "value_test.go": tests} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + binary := filepath.Join(dir, "original.test") + command := exec.Command("go", "test", "-p=2", "-vet=off", "-c", "-o", binary, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + x86FloatCompareCaptureCommand(t, command) + output := execution.run(t, binary, "-test.run=^TestOriginal$", "-test.count=1", "-test.v") + if !strings.Contains(string(output), "--- PASS: TestOriginal") || !strings.Contains(string(output), "PASS\n") { + t.Fatal("original Go had no successful terminal", string(output)) + } +} + +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplySoftwareAllWidths(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, reference, checks strings.Builder + declarations.WriteString("package oracle\n") + reference.WriteString(x86WordMultiplyCReference) + count := 0 + for _, name := range x86WordMultiplyNames { + widths := []int{16} + if strings.HasPrefix(name, "V") { + widths = append(widths, 32, 64) + } + for _, width := range widths { + for _, memory := range []bool{false, true} { + for _, alias := range []bool{false, true} { + for _, masking := range []int{0, 1, 2} { + if !strings.HasPrefix(name, "V") && masking != 0 { + continue + } + fn := fmt.Sprintf("W%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + source := bank + "0" + if alias { + source = bank + "1" + } + if memory { + source = "(DI)" + } + inputAlias := alias && !memory + op, args := name, source+","+bank+"1" + if strings.HasPrefix(name, "V") { + if masking != 0 { + args += ",K1" + } + if masking == 2 { + op += ".Z" + } + args += "," + bank + "1" + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", fn) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-56\nMOVQ out+0(FP),DX\nMOVQ a+8(FP),DI\nMOVQ b+16(FP),SI\nMOVQ flags+24(FP),CX\nMOVQ csr+32(FP),BX\nVMOVUPS (DI),Z0\nVMOVUPS (SI),Z1\n", fn) + if masking != 0 { + assembly.WriteString("MOVQ mask+48(FP),AX\nKMOVQ AX,K1\n") + } + assembly.WriteString("PUSHQ seed+40(FP)\nPOPFQ\n") + fmt.Fprintf(&assembly, "%s %s\nVMOVUPS Z1,(DX)\nPUSHFQ\nPOPQ AX\nMOVQ AX,(CX)\nSTMXCSR (BX)\nRET\n", op, args) + fmt.Fprintf(&reference, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", fn) + legacy, a := 0, 0 + if !strings.HasPrefix(name, "V") { + legacy = 1 + } + if inputAlias { + a = 1 + } + fmt.Fprintf(&checks, "if(check(%s,%q,%d,%d,%d,%d,%d)) return 1;\n", fn, name+" "+args, width, legacy, a, x86WordMultiplyReferenceMode(name), masking) + } + } + } + } + } + reference.WriteString("int main(void) {\n" + checks.String() + "return 0;}\n") + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sigs[name] = x86WordMultiplySignature(t, pkg, name) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "word-multiply-all-widths", ir, reference.String(), level) + }) + } + t.Logf("%d initialized LLVM software kernels ×257 samples ×2 seeds ×4 masks; full64-byte aliases, six flags, MXCSR/input/canaries. Original-Go accepted source/object is separate; this does NOT execute original AVX/AVX512 instructions or prove native EVEX capability.", count) +} + +func TestCrossLinuxRuntimeMatrixX86PackedWordMultiplySoftwareMaskedMemory(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + count := 0 + for _, name := range x86WordMultiplyNames[4:] { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, zero := range []bool{false, true} { + fn := fmt.Sprintf("M%d", count) + count++ + op, zeroInt := name, 0 + if zero { + op, zeroInt = op+".Z", 1 + } + fmt.Fprintf(&declarations, "func %s(out,a,b *[64]byte,flags *uint64,csr *uint32,seed,mask uint64)\n", fn) + fmt.Fprintf(&assembly, `TEXT ·%s(SB),4,$0-56 +MOVQ out+0(FP),DX +MOVQ a+8(FP),DI +MOVQ b+16(FP),SI +MOVQ flags+24(FP),CX +MOVQ csr+32(FP),BX +MOVQ mask+48(FP),AX +KMOVQ AX,K1 +VMOVUPS (SI),Z1 +PUSHQ seed+40(FP) +POPFQ +%s (DI),%s1,K1,%s1 +VMOVUPS Z1,(DX) +PUSHFQ +POPQ AX +MOVQ AX,(CX) +STMXCSR (BX) +RET +`, fn, op, bank, bank) + fmt.Fprintf(&checks, "if(checkMasked(%s,%q,%d,%d,%d,p)) return 1;\n", fn, op, width, zeroInt, x86WordMultiplyReferenceMode(name)) + } + } + } + goAssemblerOracleOutput(t, "amd64", assembly.String(), "", true) + pkg := mustGoPackage(t, "oracle", declarations.String()) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + sigs := map[string]FuncSig{} + var external strings.Builder + for _, fn := range file.Funcs { + name := strings.TrimPrefix(fn.Sym, "·") + sigs[name] = x86WordMultiplySignature(t, pkg, name) + fmt.Fprintf(&external, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t*,uint32_t*,uint64_t,uint64_t);\n", name) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }}) + if err != nil { + t.Fatal(err) + } + const maskedReference = ` +#include +#include +static int checkMasked(Program program,const char *name,int width,int zero,int mode,const uint8_t *pageTail) { + const uint64_t mask[]={0,1,UINT64_C(1)<<32}; + for(int state=0;state<2;state++) for(int m=0;m<3;m++) { + struct Aligned b,out; + memset(&b,0xb9,sizeof(b));memset(&out,0xc3,sizeof(out)); + struct Aligned before=b; + struct {uint64_t before,value,after;} flags={0x11223344,0,0x44332211}; + struct {uint32_t before,value,after;} csr={0x55667788,0,0x88776655}; + uint64_t seed=state?UINT64_C(0x8d5):0; + uint32_t saved=_mm_getcsr(),profile=0x1fbf; + _mm_setcsr(profile); + program(out.value,m==1?pageTail:NULL,b.value,&flags.value,&csr.value,seed,mask[m]); + _mm_setcsr(saved); + if((flags.value&UINT64_C(0x8d5))!=seed||flags.before!=0x11223344||flags.after!=0x44332211|| + csr.value!=profile||csr.before!=0x55667788||csr.after!=0x88776655||memcmp(&b,&before,sizeof(b))) return 2; + for(int i=0;i<64;i++) { + uint8_t want=0; + if(i>(i*8)); + else if(!zero) want=b.value[i]; + } + if(out.value[i]!=want||out.before[i]!=0xc3||out.after[i]!=0xc3) { + fprintf(stderr,"%s masked read byte%d mask%llx got%x want%x\n",name,i,(unsigned long long)mask[m],out.value[i],want);return 3; + } + } + } + return 0; +} +` + main := x86WordMultiplyCReference + external.String() + maskedReference + ` +int main(void) { + long page=sysconf(_SC_PAGESIZE); + if(page<64) return 4; + uint8_t *base=mmap(NULL,(size_t)page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(base==MAP_FAILED||mprotect(base+page,(size_t)page,PROT_NONE)) return 5; + uint8_t *p=base+page-2;p[0]=0;p[1]=0x80; +` + checks.String() + ` + if(munmap(base,(size_t)page*2)) return 6; + return 0; +} +` + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + execution.compileAndRunLLVM(t, "word-multiply-masked-memory", ir, main, level) + }) + } + t.Logf("%d software EVEX kernels: inactive NULL, guard-page first word, irrelevant high mask bits, two flag seeds/full bytes/MXCSR/canaries. These semantics do not grant source/caller memory permission or native EVEX execution.", count) +} diff --git a/amd64_packed_zero_extend_contract_test.go b/amd64_packed_zero_extend_contract_test.go new file mode 100644 index 000000000..40b06b305 --- /dev/null +++ b/amd64_packed_zero_extend_contract_test.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86PackedZeroExtendDispatchIdentityBeforeEmission(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + opcode := prefix + "MOVZX" + ratio.suffix + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+opcode+" X0,X1\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, identity := range []Op{"", Op(opcode), Op(strings.ToLower(opcode)), "PMOVSXBW", "VPMOVZXBW.Z"} { + ins := file.Funcs[0].Instrs[1] + ins.Op = identity + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + before := output.String() + known, term, err := ctx.lowerPackedExtendMove(Op(opcode), ins) + conflict := identity != "" && !strings.EqualFold(string(identity), opcode) + if !known || term || (err != nil) != conflict || conflict && output.String() != before { + t.Fatalf("dispatch %s/%s: known=%t term=%t err=%v", opcode, identity, known, term, err) + } + } + } + } +} + +func TestX86PackedZeroExtendFPBytesBindActualGoDeclaration(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, bank := range []string{"X", "Y", "Z"} { + if prefix == "P" && bank != "X" { + continue + } + width := map[string]int{"X": 16, "Y": 32, "Z": 64}[bank] * ratio.input / ratio.output + for _, layout := range []string{"complete", "last byte absent", "alignment hole"} { + var parameters []string + for i := 0; i < width; i++ { + if layout == "last byte absent" && i == width-1 { + continue + } + if layout == "alignment hole" && i >= 1 { + parameters = append(parameters, fmt.Sprintf("a%d uint64", i)) + } else { + parameters = append(parameters, fmt.Sprintf("a%d byte", i)) + } + } + fset := token.NewFileSet() + declaration, err := parser.ParseFile(fset, "decl.go", "package bound\nfunc f("+strings.Join(parameters, ",")+")\n", 0) + if err != nil { + t.Fatal(err) + } + typed, err := (&types.Config{Sizes: types.SizesFor("gc", "amd64")}).Check("bound", fset, []*ast.File{declaration}, nil) + if err != nil { + t.Fatal(err) + } + row := prefix + "MOVZX" + ratio.suffix + " a0+0(FP)," + bank + "1" + params := typed.Scope().Lookup("f").Type().(*types.Signature).Params() + var fields []*types.Var + for i := 0; i < params.Len(); i++ { + fields = append(fields, params.At(i)) + } + sizes := types.SizesFor("gc", "amd64") + offsets := sizes.Offsetsof(fields) + end := offsets[len(offsets)-1] + sizes.Sizeof(fields[len(fields)-1].Type()) + allocation := (end + 7) &^ 7 // Actual ABI0 word-aligned scalar allocation. + assembly := fmt.Sprintf("TEXT f(SB),4,$0-%d\n%s\nRET\n", allocation, row) + if layout == "complete" { + requireX86GoAssemblerResult(t, "amd64", assembly, true) + } + ctx := llvm.NewContext() + bound, err := translateGoModuleInContext(ctx, GoPackage{Path: "bound", Types: typed, Syntax: []*ast.File{declaration}}, []byte(assembly), GoModuleOptions{GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu"}) + if bound != nil { + bound.Module.Dispose() + } + ctx.Dispose() + if (err != nil) != (layout != "complete") { + t.Fatalf("%s/%s ignored actual Go field layout: %v", row, layout, err) + } + } + } + } + } +} + +func TestX86PackedZeroExtendEveryDeclaredFPByteBeforeEmission(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, row := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, "amd64") { + if !strings.Contains(row, "8(AX)") { + continue + } + row = strings.Replace(row, "8(AX)", "a+1(FP)", 1) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-64\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + form, _, err := x86PackedZeroExtendProperties(ins, "amd64") + if err != nil { + t.Fatal(err) + } + for missing := -1; missing < form.memoryBytes; missing++ { + sig := FuncSig{Name: "f", Ret: Void} + for i := 0; i < form.memoryBytes; i++ { + if i != missing { + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(i + 1), Type: I8, Index: len(sig.Args), Field: -1}) + sig.Args = append(sig.Args, I8) + } + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], sig, nil) + before := output.String() + known, term, err := ctx.lowerPackedExtendMove(ins.Op, ins) + if !known || term || (err != nil) != (missing >= 0) || missing >= 0 && output.String() != before { + t.Fatalf("%s missing byte %d emitted/accepted undeclared FP bytes: %v", row, missing, err) + } + } + } + } + } +} + +func TestX86GoCallerPackedZeroExtendMemoryCertificatesBindNativeSource(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + for _, original := range x86ZeroExtendAcceptedForms(prefix+"MOVZX"+ratio.suffix, "amd64") { + if !strings.Contains(original, "8(AX)") { + continue + } + row := strings.NewReplacer("8(AX)", "8(DI)", "X15", "X2", "X31", "X2", "Y31", "Y2", "Z31", "Z2").Replace(original) + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", "VMOVUPS (SI),Z2\nKMOVQ AX,K1\nKMOVQ AX,K7\n"+row+"\nVMOVSS X2,(DX)", 1) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64", Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", ResolveSym: resolve, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + found := false + for _, ins := range normalized.Funcs[0].Instrs { + form, recognized, err := x86PackedZeroExtendProperties(ins, "amd64") + if !recognized || err != nil || form.memory < 0 { + continue + } + found = true + index := form.memory + access, exists := ins.x86CallerMemory[index] + if !exists || access.write || access.width != int64(form.memoryBytes) || !x86CallerMemoryCertified(ins, index) { + t.Fatalf("%s lacks its exact shrinking native read certificate", row) + } + for _, change := range []string{"opcode", "width", "address", "index", "destination", "direction", "mask", "address-bits"} { + mutated := ins + mutated.Args = append([]Operand(nil), ins.Args...) + mutated.x86CallerMemory = map[int]x86CallerMemoryAccess{index: access} + switch change { + case "opcode": + mutated.Op = "VPMOVSXBD" + case "width": + access.width++ + mutated.x86CallerMemory[index] = access + case "address": + mutated.Args[index].Mem.Off++ + case "index": + mutated.Args[index].Mem.Index, mutated.Args[index].Mem.Scale = BX, 4 + case "destination": + mutated.Args[form.destination].Reg = "X3" + case "direction": + access.write = true + mutated.x86CallerMemory[index] = access + case "mask": + if form.mask >= 0 { + mutated.Args[form.mask].Reg = "K2" + } else { + mutated.Args = append(mutated.Args, Operand{Kind: OpReg, Reg: "K2"}) + } + case "address-bits": + mutated.x86AddressBits = 32 + } + if x86CallerMemoryCertified(mutated, index) { + t.Fatalf("%s mutation retained %s source certificate", change, row) + } + } + } + if !found { + t.Fatal("no widening source instruction consumed its caller certificate") + } + } + } + } +} + +func TestX86PackedZeroExtendInvalidMemoryAndMaskGrammar(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, row := range []string{ + "VPMOVZXBD (AX)(SP*1),X1", "VPMOVZXBD (AX)(CX*3),X1", + "VPMOVZXBD $glob(SB),X1", "VPMOVZXBD X0,K0,X1", + "VPMOVZXBD.SAE X0,X1", "VPMOVZXBD.RN_SAE X0,X1", + "VPMOVZXBD. X0,X1", "PMOVZXBD X0,K1,X1", + } { + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + continue + } + ctx, output := newAMD64CtxWithFuncForTest(t, file.Funcs[0], FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := output.String() + ins := file.Funcs[0].Instrs[1] + known, term, err := ctx.lowerPackedExtendMove(ins.Op, ins) + if !known || term || err == nil || output.String() != before { + t.Fatalf("%s/%s emitted invalid grammar: %v", arch, row, err) + } + } + } + // A full signed displacement must not silently wrap in native amd64. + for _, off := range []int64{1 << 31, -1<<31 - 1, 1 << 32} { + ins := Instr{Op: "VPMOVZXBD", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX, Off: off}}, {Kind: OpReg, Reg: "X1"}}} + if _, recognized, err := x86PackedZeroExtendProperties(ins, "amd64"); !recognized || err == nil { + t.Fatalf("out-of-range native displacement %d accepted", off) + } + } + for _, scale := range []int64{1, 2, 4, 8} { + row := fmt.Sprintf("VPMOVZXBD 3(AX)(CX*%d),X1", scale) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, _, err := x86PackedZeroExtendProperties(file.Funcs[0].Instrs[1], "amd64"); err != nil { + t.Fatal(row, err) + } + } +} diff --git a/amd64_packed_zero_extend_forms.go b/amd64_packed_zero_extend_forms.go new file mode 100644 index 000000000..dc0f83c13 --- /dev/null +++ b/amd64_packed_zero_extend_forms.go @@ -0,0 +1,115 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86PackedZeroExtendSpec struct { + inputBits int + outputBits int + vector bool + opcode byte +} + +// Legacy yxm_q4 and the two AVX tables have different physical source widths: +// ratio two uses _yvcvtdq2pd (Y/m256 to Z); the other ratios _yvbroadcastss. +// None permits broadcast, SAE or rounding. EVEX masking is per widened lane. +var x86PackedZeroExtendSpecs = map[Op]x86PackedZeroExtendSpec{ + "PMOVZXBW": {8, 16, false, 0x30}, + "PMOVZXBD": {8, 32, false, 0x31}, + "PMOVZXBQ": {8, 64, false, 0x32}, + "PMOVZXWD": {16, 32, false, 0x33}, + "PMOVZXWQ": {16, 64, false, 0x34}, + "PMOVZXDQ": {32, 64, false, 0x35}, + "VPMOVZXBW": {8, 16, true, 0x30}, + "VPMOVZXBD": {8, 32, true, 0x31}, + "VPMOVZXBQ": {8, 64, true, 0x32}, + "VPMOVZXWD": {16, 32, true, 0x33}, + "VPMOVZXWQ": {16, 64, true, 0x34}, + "VPMOVZXDQ": {32, 64, true, 0x35}, +} + +type x86PackedZeroExtendForm struct { + spec x86PackedZeroExtendSpec + byteWidth int + sourceRegisterBytes int + memoryBytes int + destination int + memory int + mask int + zeroing bool +} + +// This specifies encoding and byte effects, never caller bounds or permission. +func x86PackedZeroExtendProperties(ins Instr, goarch string) (x86PackedZeroExtendForm, bool, error) { + base, suffix, dotted := strings.Cut(strings.ToUpper(string(ins.Op)), ".") + spec, recognized := x86PackedZeroExtendSpecs[Op(base)] + if !recognized { + return x86PackedZeroExtendForm{}, false, nil + } + form := x86PackedZeroExtendForm{ + spec: spec, destination: len(ins.Args) - 1, memory: -1, mask: -1, + zeroing: suffix == "Z", sourceRegisterBytes: 16, + } + fail := func() (x86PackedZeroExtendForm, bool, error) { + return form, true, fmt.Errorf("%s %s operands/suffix outside Go packed zero-extension grammar: %q", goarch, ins.Op, ins.Raw) + } + if goarch != "amd64" && goarch != "386" || len(ins.Args) != 2 && len(ins.Args) != 3 || + dotted && (!spec.vector || suffix != "Z") { + return fail() + } + if len(ins.Args) == 3 { + if !spec.vector || !amd64NonzeroKOperand(ins.Args[1]) { + return fail() + } + form.mask = 1 + } else if form.zeroing { + return fail() + } + destination := ins.Args[form.destination] + form.byteWidth = amd64VectorByteWidth(destination.Reg) + if !spec.vector { + index, valid := amd64ParseXReg(destination.Reg) + if destination.Kind != OpReg || !valid || index >= 16 || goarch == "386" && index >= 8 { + return fail() + } + } else if !amd64GoPackedVectorMoveRegister(destination, form.byteWidth, goarch) { + return fail() + } + form.memoryBytes = form.byteWidth * spec.inputBits / spec.outputBits + if form.memoryBytes > 16 { + form.sourceRegisterBytes = 32 + } + source := ins.Args[0] + if source.Kind == OpReg { + if !amd64EVEXVectorRegister(source, form.sourceRegisterBytes) { + return fail() + } + index, _ := amd64VectorRegisterIndex(source.Reg, form.sourceRegisterBytes) + if !spec.vector && (index >= 16 || goarch == "386" && index >= 8) { + return fail() + } + } else if isAMD64MemoryOperand(source) { + if source.Kind == OpSym { + symbol := strings.TrimSpace(source.Sym) + _, absolute := parseInt(symbol) + if strings.HasPrefix(symbol, "$") || !strings.HasSuffix(symbol, "(SB)") && absolute != nil { + return fail() + } + } + if source.Kind == OpMem { + displacement := source.Mem.Off + if displacement != int64(int32(displacement)) && + (goarch != "386" || displacement != int64(uint32(displacement))) || + !x86MemoryRegistersValidForArch(source.Mem, goarch) || source.Mem.Index != "" && + (source.Mem.Index == SP || source.Mem.Scale != 1 && source.Mem.Scale != 2 && source.Mem.Scale != 4 && source.Mem.Scale != 8) { + return fail() + } + } + form.memory = 0 + } else { + return fail() + } + return form, true, x86Mode32VectorRegisterContext(ins, goarch) +} diff --git a/amd64_packed_zero_extend_forms_test.go b/amd64_packed_zero_extend_forms_test.go new file mode 100644 index 000000000..ebcfca1a0 --- /dev/null +++ b/amd64_packed_zero_extend_forms_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "testing" +) + +// Independent byte ratios, not values taken from the production spec. +var x86ZeroExtendTestRatios = []struct { + suffix string + input int + output int +}{ + {"BW", 1, 2}, {"BD", 1, 4}, {"BQ", 1, 8}, + {"WD", 2, 4}, {"WQ", 2, 8}, {"DQ", 4, 8}, +} + +func x86ZeroExtendAcceptedForms(opcode, arch string) []string { + if !strings.HasPrefix(opcode, "V") { + last := 15 + if arch == "386" { + last = 7 + } + return []string{fmt.Sprintf("%s X0,X%d", opcode, last), fmt.Sprintf("%s 8(AX),X%d", opcode, last)} + } + var rows []string + for _, bank := range []string{"X", "Y", "Z"} { + source, last := "X", 31 + if bank == "Z" && (strings.HasSuffix(opcode, "BW") || strings.HasSuffix(opcode, "WD") || strings.HasSuffix(opcode, "DQ")) { + source = "Y" + } + if arch == "386" { + last = 7 + } + for _, input := range []string{source + "0", "8(AX)"} { + rows = append(rows, fmt.Sprintf("%s %s,%s%d", opcode, input, bank, last)) + rows = append(rows, fmt.Sprintf("%s %s,K7,%s%d", opcode, input, bank, last), + fmt.Sprintf("%s.Z %s,K1,%s%d", opcode, input, bank, last)) + } + } + // These select the two VEX optab rows rather than their EVEX aliases. + rows = append(rows, opcode+" X0,X1", opcode+" X0,Y1") + return rows +} + +func TestX86PackedZeroExtendSpecsBindCompleteGoTables(t *testing.T) { + dir := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86") + legacy, err := os.ReadFile(filepath.Join(dir, "asm6.go")) + if err != nil { + t.Fatal(err) + } + avx, err := os.ReadFile(filepath.Join(dir, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + if len(x86PackedZeroExtendSpecs) != 12 { + t.Fatalf("unsigned family has %d names, want 12", len(x86PackedZeroExtendSpecs)) + } + // Bind every operand class row, independently of the lowerer's accepted + // form generator. Memory width attributes belong to each opcode below. + legacyTable := "var yxm_q4 = []ytab{\n\t{Zm_r, 1, argList{Yxm, Yxr}},\n}" + if !strings.Contains(string(legacy), legacyTable) { + t.Fatal("Go changed the complete legacy register/memory operand table") + } + for _, table := range []string{"_yvbroadcastss", "_yvcvtdq2pd"} { + want := "var " + table + " = []ytab{\n" + want += "\t{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yxr}},\n" + want += "\t{zcase: Zvex_rm_v_r, zoffset: 2, args: argList{Yxm, Yyr}},\n" + for _, destination := range []string{"YxrEvex", "YyrEvex", "Yzr"} { + source := "YxmEvex" + if table == "_yvcvtdq2pd" && destination == "Yzr" { + source = "YymEvex" + } + want += fmt.Sprintf("\t{zcase: Zevex_rm_v_r, zoffset: 0, args: argList{%s, %s}},\n", source, destination) + want += fmt.Sprintf("\t{zcase: Zevex_rm_k_r, zoffset: 3, args: argList{%s, Yknot0, %s}},\n", source, destination) + } + want += "}" + if !strings.Contains(string(avx), want) { + t.Fatalf("Go changed %s's complete VEX/EVEX operand table", table) + } + } + for index, ratio := range x86ZeroExtendTestRatios { + opcode := byte(0x30 + index) + for _, prefix := range []string{"P", "VP"} { + name := prefix + "MOVZX" + ratio.suffix + spec, present := x86PackedZeroExtendSpecs[Op(name)] + if !present || spec.inputBits != ratio.input*8 || spec.outputBits != ratio.output*8 || spec.vector != (prefix == "VP") || spec.opcode != opcode { + t.Fatalf("independent unsigned ratio/encoding mismatch: %s %+v", name, spec) + } + if prefix == "P" { + want := fmt.Sprintf("{A%s, yxm_q4, Pq4, opBytes{0x%02x}}", name, opcode) + if !strings.Contains(string(legacy), want) { + t.Fatalf("missing exact Go legacy row: %s", want) + } + continue + } + table := "_yvbroadcastss" + if ratio.output == 2*ratio.input { + table = "_yvcvtdq2pd" + } + marker := "{as: A" + name + ", ytab: " + table + ", prefix: Pavx, op: opBytes{" + start := strings.Index(string(avx), marker) + if start < 0 { + t.Fatalf("missing exact Go AVX family row: %s", marker) + } + row := strings.SplitN(string(avx)[start:], "\n\t}},", 2)[0] + want := marker + for _, width := range []int{128, 256} { + want += fmt.Sprintf("\n\t\tavxEscape | vex%d | vex66 | vex0F38 | vexW0, 0x%02x,", width, opcode) + } + for _, width := range []int{128, 256, 512} { + want += fmt.Sprintf("\n\t\tavxEscape | evex%d | evex66 | evex0F38 | evexW0, evexN%d | evexZeroingEnabled, 0x%02x,", width, width/8*ratio.input/ratio.output, opcode) + } + if row != want { + t.Fatalf("changed complete Go prefix/opcode/width/exception attributes:\n%s\nwant:\n%s", row, want) + } + } + } + for _, source := range []string{string(legacy), string(avx)} { + for _, match := range regexp.MustCompile(`\{(?:as: )?A(V?PMOVZX[A-Z]+),`).FindAllStringSubmatch(source, -1) { + if _, exists := x86PackedZeroExtendSpecs[Op(match[1])]; !exists { + t.Fatalf("Go added an unregistered unsigned family name %s", match[1]) + } + } + } +} + +func TestX86PackedZeroExtendNamedMode32RetainsActualPhysicalContext(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, bank := range []string{"X", "Y"} { + for _, high := range []int{8, 16, 31} { + for _, args := range []string{fmt.Sprintf("X%d,%s0", high, bank), fmt.Sprintf("X0,%s%d", bank, high)} { + row := "VPMOVZX" + ratio.suffix + " " + args + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual mode32 Go source acceptance: %s: %v\n%s", row, err, listing) + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, target := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: target, Sigs: map[string]FuncSig{"f": {Ret: Void}}}); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("named high register acquired physical mode32 binding: %s: %v", row, err) + } + } + } + } + } + } +} + +func TestX86PackedZeroExtendActualGoBytesBindRawWidths(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, row := range []string{"X0,X1", "X0,Y1", "8(AX),Y1", "X20,K3,X21", "X20,K3,Y21"} { + instruction := "VPMOVZX" + ratio.suffix + " " + row + listing, err := x86ExtensionGoListing(t, "amd64", instruction) + if err != nil { + t.Fatal(err, listing) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + break + } + code = append(code, byte(value)) + } + break + } + decoded, length, recognized, err := decodedX86PackedExtendInstruction(code, 64) + if !recognized || err != nil || length <= 0 || decoded.Op != Op("VPMOVZX"+ratio.suffix) { + t.Fatalf("actual Go object did not decode as source family: %s/%x: %+v %v", instruction, code, decoded, err) + } + form, recognized, err := x86PackedZeroExtendProperties(decoded, "amd64") + if !recognized || err != nil || form.memoryBytes != form.byteWidth*ratio.input/ratio.output { + t.Fatalf("actual raw width lost its independent source ratio: %+v %v", form, err) + } + } + } +} diff --git a/amd64_packed_zero_extend_native_runtime_unix_test.go b/amd64_packed_zero_extend_native_runtime_unix_test.go new file mode 100644 index 000000000..9bde099a7 --- /dev/null +++ b/amd64_packed_zero_extend_native_runtime_unix_test.go @@ -0,0 +1,108 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Legacy and VEX-X live/dead kernels use actual original legacy Go bytes as +// their reference. Masked X/Y/Z dead reads use an independent volatile byte +// reference: portable LLVM execution does not certify native AVX512 execution. +// The complete-byte runtime separately checks masks, aliases, FLAGS and MXCSR. +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendNativeResultsAndFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + const ratiosPerBatch = 3 + for batch := 0; batch < 2; batch++ { + t.Run(fmt.Sprint("batch", batch), func(t *testing.T) { + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + for _, ratio := range x86ZeroExtendTestRatios[batch*ratiosPerBatch : (batch+1)*ratiosPerBatch] { + add := func(instruction string, width, discarded int, mask int64, native bool) { + name := fmt.Sprintf("Widen%d", count) + original := fmt.Sprintf("Original%d", count) + count++ + body := "MOVQ p+0(FP),DX\n" + if strings.Contains(instruction, "K1") { + body += fmt.Sprintf("MOVQ $%d,AX\nKMOVQ AX,K1\n", mask) + } + body += instruction + "\nPEXTRQ $0,X1,AX\n" + if discarded != 0 { + body += "MOVQ $0x87654321fedcba98,AX\n" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\n%sMOVQ AX,ret+8(FP)\nRET\n", name, body) + fmt.Fprintf(&declarations, "func %s(p *[32]byte) uint64\n", name) + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\n", name) + if native { + nativeBody := "PMOVZX" + ratio.suffix + " (DI),X0\nPEXTRQ $0,X0,AX\n" + if discarded != 0 { + nativeBody += "MOVQ $0x87654321fedcba98,AX\n" + } + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+nativeBody+"RET\n") + var bytes []string + for _, value := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", value)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\n", original) + } else { + fmt.Fprintf(&originals, "static uint64_t %s(const uint8_t *p) {\n", original) + fmt.Fprintf(&originals, " for (unsigned b = 0; b < %d; b++) {\n", width) + originals.WriteString(" volatile uint8_t value = ((volatile const uint8_t*)p)[b];\n (void)value;\n }\n") + originals.WriteString(" return UINT64_C(0x87654321fedcba98);\n}\n") + } + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d,%d},\n", name, original, width, discarded, ratio.input, ratio.output) + } + for _, prefix := range []string{"P", "VP"} { + for discarded := 0; discarded < 2; discarded++ { + add(prefix+"MOVZX"+ratio.suffix+" (DX),X1", 16*ratio.input/ratio.output, discarded, 0, true) + } + } + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + for _, mask := range []int64{0, 1, -1} { + readWidth := width * ratio.input / ratio.output + if mask == 0 { + readWidth = 0 + } else if mask == 1 { + readWidth = ratio.input + } + add("VPMOVZX"+ratio.suffix+".Z (DX),K1,"+bank+"1", readWidth, 1, mask, false) + } + } + } + if count != 39 || count > 64 { + t.Fatal("incomplete or oversized widening read matrix", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86NativeScalarReadRuntimeC, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprint("O2=", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "widen-native", text, driver, level) + if string(output) != "39 native scalar kernels: numerical 4992, faults 468, tripwires 2; failures 0\n" { + t.Fatal("widening numerical/dead/native/masked reads differ:", string(output)) + } + t.Log("Original legacy Go / portable masked-read reference:", strings.TrimSpace(string(output))) + }) + } + }) + } + }) + } +} diff --git a/amd64_packed_zero_extend_raw_axes_test.go b/amd64_packed_zero_extend_raw_axes_test.go new file mode 100644 index 000000000..4d8a5772c --- /dev/null +++ b/amd64_packed_zero_extend_raw_axes_test.go @@ -0,0 +1,84 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +func TestX86PackedExtendRawWBitMatchesIndependentLLVMDecoder(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM22 llvm-mc required") + } + for _, spec := range rawPackedExtendTestSpecs { + for _, evex := range []bool{false, true} { + for width := 0; width < 3; width++ { + if !evex && width == 2 { + continue + } + code := encodeX86VEXPackedExtend(spec.opcode, width != 0, 1, 0) + code[2] |= 0x80 // All VEX forms are WIG, not W0. + if evex { + code = encodeX86EVEXPackedExtend(spec.opcode, byte(width), 1, true, 1, 0) + code[2] |= 0x80 // EVEX is WIG except signed/unsigned DQ. + } + var bytes strings.Builder + for _, value := range code { + fmt.Fprintf(&bytes, "0x%02x ", value) + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple=x86_64", "--mattr=+avx,+avx2,+avx512f,+avx512bw,+avx512vl") + command.Stdin = strings.NewReader(bytes.String() + "\n") + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + invalid := evex && spec.inputBits == 32 + if invalid { + if !strings.Contains(string(output), "invalid instruction encoding") { + t.Fatalf("LLVM22 did not reject EVEX DQ W1 %x: %v\n%s", code, err, output) + } + } else if err != nil || !strings.Contains(strings.ToUpper(string(output)), string(spec.op)) { + t.Fatalf("LLVM22 independent WIG decode %x: %v\n%s", code, err, output) + } + for _, mode := range []int{32, 64} { + ins, length, recognized, err := decodedX86PackedExtendInstruction(code, mode) + if invalid { + if !recognized || err == nil { + t.Fatalf("mode%d illegal EVEX DQ W1 acquired widening semantics: %x %+v/%v", mode, code, ins, err) + } + } else if !recognized || err != nil || length != len(code) || strings.TrimSuffix(string(ins.Op), ".Z") != string(spec.op) { + t.Fatalf("mode%d legal WIG %x lost exact family semantics: %+v/%v", mode, code, ins, err) + } + } + } + } + } +} + +func TestX86PackedZeroExtendActualGoLowMaskedMode32(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, bank := range []string{"X", "Y", "Z"} { + source := "X0" + if bank == "Z" && ratio.output == 2*ratio.input { + source = "Y0" + } + for _, input := range []string{source, "(AX)"} { + for _, suffix := range []string{"", ".Z"} { + row := fmt.Sprintf("VPMOVZX%s%s %s,K7,%s7", ratio.suffix, suffix, input, bank) + listing, err := x86ExtensionGoListing(t, "386", row) + if err != nil { + t.Fatalf("actual mode32 Go acceptance: %s: %v\n%s", row, err, listing) + } + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: map[string]FuncSig{"f": {Ret: Void}}}); err != nil { + t.Fatalf("actual Go low-bank mask form incorrectly rejected: %s: %v", row, err) + } + } + } + } + } +} diff --git a/amd64_packed_zero_extend_result.go b/amd64_packed_zero_extend_result.go new file mode 100644 index 000000000..23a1af72f --- /dev/null +++ b/amd64_packed_zero_extend_result.go @@ -0,0 +1,31 @@ +package plan9asm + +func x86ResultPackedZeroExtendStep(ins Instr, state *x86ResultState, owned []bool, arch string, conditionalNativeRead bool) (bool, bool) { + form, known, err := x86PackedZeroExtendProperties(ins, arch) + if !known || err != nil { + return false, known + } + if form.mask >= 0 { + index, valid := amd64ParseKReg(ins.Args[form.mask].Reg) + if !valid || !state.masks[index] { + return false, true + } + } + var input uint64 + if form.memory >= 0 { + // Only the lowering observer may use the paired native E5 read. + // The full potential tuple remains required for FP/AUTO/caller proofs; + // a numerical mask cannot manufacture a smaller owned envelope. + if !x86ResultReadWithNativeContinuation(ins, ins.Args[0], int64(form.memoryBytes), state, owned, arch, conditionalNativeRead) { + return false, true + } + input = x86CallerVectorByteMask(int64(form.memoryBytes)) + } else { + index, _ := amd64VectorRegisterIndex(ins.Args[0].Reg, form.sourceRegisterBytes) + input = state.vectorBytes(index) + } + destination, _ := amd64VectorRegisterIndex(ins.Args[form.destination].Reg, form.byteWidth) + initialized := x86PackedZeroExtendDefinedBytes(form, input, state.vectorBytes(destination)) + state.writeVector(destination, form.byteWidth, initialized, !form.spec.vector) + return true, true +} diff --git a/amd64_packed_zero_extend_result_test.go b/amd64_packed_zero_extend_result_test.go new file mode 100644 index 000000000..0a76dae9e --- /dev/null +++ b/amd64_packed_zero_extend_result_test.go @@ -0,0 +1,232 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86PackedZeroExtendResultConsumers(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + for _, native := range []bool{false, true} { + t.Run(fmt.Sprint(target.triple, "/native=", native), func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, vector := range []bool{false, true} { + for _, width := range []int{16, 32, 64} { + if !vector && width != 16 { + continue + } + for masking := 0; masking < 3; masking++ { + if !vector && masking != 0 { + continue + } + name := fmt.Sprint("Widen", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + input := "X0" + if width*ratio.input/ratio.output > 16 { + input = "Y0" + } + offset, load, declaration := 8, "MOVQ value+0(FP),X0", "value uint64" + if native { + input, load, declaration = "(DX)", "MOVQ value+0(FP),DX", "value *[32]byte" + if target.arch == "386" { + offset, load = 4, "MOVL value+0(FP),DX" + } + } + row := "PMOVZX" + ratio.suffix + if vector { + row = "V" + row + } + if masking == 2 { + row += ".Z" + } + row += " " + input + "," + if masking != 0 { + row += "K1," + } + row += bank + "1" + fmt.Fprintf(&declarations, "func %s(%s) uint32\n", name, declaration) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\nVZEROALL\n%s\nMOVL $1,AX\nKMOVW AX,K1\n%s\nPEXTRD $0,X1,AX\nMOVL AX,ret+%d(FP)\nRET\n", name, offset+4, load, row, offset) + } + } + } + } + if count != 60 || count > 64 { + t.Fatal("incomplete or oversized result matrix", count) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("widening result lacks its numerical consumer", api, err) + } + x86ResultInitializationObject(t, llc, target.triple, ir, "widen-result-"+api) + x86ResultInitializationObject(t, llc, target.triple, x86MulDivRoundOptimize(t, ir), "widen-result-"+api+"-O2") + } + }) + } + } +} + +func TestX86PackedZeroExtendNativeResultBoundaries(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + name := prefix + "MOVZX" + ratio.suffix + t.Run(name, func(t *testing.T) { + body := "MOVQ p+0(FP),DX\n" + name + " (DX),X1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+8(FP)\nRET" + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[8]byte) uint32", body) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("standalone result observer acquired native memory", err) + } + } + ins := x86ScalarLaneInstruction(t, name+" -1(DX)(BX*1),X1") + state := x86ResultState{flags: x86ByteFlagsAll} + base, _, _ := x86ResultGP(DX, 8, "amd64") + index, _, _ := x86ResultGP(BX, 8, "amd64") + state.pointerGP[base], state.gp[index] = true, 0xff + state.writeVector(1, 64, 0xa5a5000000000000, false) + before := state.clone() + if accepted, known := x86ResultPackedZeroExtendStep(ins, &state, nil, "amd64", true); !accepted || !known { + t.Fatal("paired native read has no conditional numerical bytes") + } + want := uint64(0xa5a500000000ffff) + if prefix == "VP" { + want = ^uint64(0) + } + if state.vectorBytes(1) != want || state.flags != before.flags || state.gp != before.gp || state.pointerGP != before.pointerGP { + t.Fatal("native widening changed another state domain") + } + for _, mutation := range []string{"decoded", "raw", "raw return", "address", "unbound address", "segment", "unknown base", "unknown index", "pointer index"} { + changed, rejected := ins, before.clone() + switch mutation { + case "decoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "raw return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + case "address": + changed.x86AddressBits = 32 + case "unbound address": + changed.x86UnboundAddressOverride = true + case "segment": + changed.x86SegmentPrefix = 0x64 + case "unknown base": + rejected.pointerGP[base] = false + case "unknown index": + rejected.gp[index] = 0 + case "pointer index": + // A reaching pointer import revokes numerical bytes. + // Only the source-wide shape pass carries their union. + rejected.pointerGP[index], rejected.gp[index] = true, 0 + } + expected := rejected.clone() + if accepted, _ := x86ResultPackedZeroExtendStep(changed, &rejected, nil, "amd64", true); accepted || !reflect.DeepEqual(rejected, expected) { + t.Fatal("unbound origin/address acquired native result bytes", mutation) + } + } + }) + } + } +} + +func TestX86PackedZeroExtendResultTupleAndMaskBoundaries(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, width := range []int{16, 32, 64} { + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + ins := x86ScalarLaneInstruction(t, "VPMOVZX"+ratio.suffix+".Z value+0(FP),K1,"+bank+"1") + for missing := -1; missing < 32; missing++ { + for _, maskKnown := range []bool{false, true} { + state := x86ResultState{fp: make([]bool, 32)} + owned := make([]bool, 32) + for index := range owned { + owned[index], state.fp[index] = true, index != missing + } + state.masks[1] = maskKnown + accepted := x86ResultTransfer(ins, &state, owned, 32, "amd64") + want := maskKnown && (missing < 0 || missing >= width*ratio.input/ratio.output) + if accepted != want || accepted && state.vectorBytes(1) != ^uint64(0) || len(ins.x86CallerMemory) != 0 { + t.Fatal("mask/tuple invented source bytes or caller permission", ratio.suffix, width, missing, maskKnown) + } + } + } + } + } +} + +func TestX86PackedZeroExtendNativeResultsKeepWholeSourceBoundaries(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, prefix := range []string{"P", "VP"} { + name := prefix + "MOVZX" + ratio.suffix + t.Run(name, func(t *testing.T) { + load := name + " (DX),X1\n" + for mutation, body := range map[string]string{ + "pointer index": name + " (DX)(DX*1),X1\n", + "lost pointer": "MOVL $1,DX\n" + load, + "missing path": "MOVL $0,BX\nCMPL BX,$0\nJE after\n" + load + "after:\n", + "native store": load + "MOVL $1,(DX)\n", + "input write": load + "MOVQ $1,p+0(FP)\n", + } { + source := "TEXT ·Value(SB),4,$0-12\nMOVQ p+0(FP),DX\n" + body + "PEXTRD $0,X1,AX\nMOVL AX,ret+8(FP)\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[32]byte) uint32", "amd64", "x86_64-unknown-linux-gnu", api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("widening result borrowed another source contract", mutation, api, err) + } + } + } + body := "MOVQ p+0(FP),DX\n" + load + "PEXTRD $0,X1,AX\nMOVL AX,ret0+8(FP)\nRET" + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[32]byte) uint32", body) + for _, origin := range []string{"raw TEXT", "source stack", "fixed local"} { + fn := file.Funcs[0] + switch origin { + case "raw TEXT": + fn.X86RawText = []byte{0xc3} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("widening result bypassed original entry boundary", origin, err) + } + } + }) + } + } +} + +func TestX86PackedZeroExtendDeadNativeReads(t *testing.T) { + for _, ratio := range x86ZeroExtendTestRatios { + for _, row := range []string{ + "PMOVZX" + ratio.suffix + " (DX),X1", + "VPMOVZX" + ratio.suffix + " (DX),Z1", + "VPMOVZX" + ratio.suffix + ".Z (DX),K1,Z1", + } { + t.Run(row, func(t *testing.T) { + source := "TEXT ·Read(SB),4,$0-16\nMOVQ p+0(FP),DX\nMOVQ mask+8(FP),AX\nKMOVQ AX,K1\n" + row + "\nRET\n" + goAssemblerOracleOutput(t, "amd64", source, "", true) + ir, err := x86ScalarFPBitViewAPI(t, source, "func Read(p *[32]byte,mask uint64)\n", "amd64", "x86_64-unknown-linux-gnu", "Go") + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if !strings.Contains(text, "load volatile") || strings.Contains(row, ".Z") && !strings.Contains(text, "br i1") { + t.Fatal("dead widening discarded native reads or masked fault suppression") + } + } + }) + } + } +} diff --git a/amd64_packed_zero_extend_runtime_test.go b/amd64_packed_zero_extend_runtime_test.go new file mode 100644 index 000000000..625860d14 --- /dev/null +++ b/amd64_packed_zero_extend_runtime_test.go @@ -0,0 +1,321 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// The byte reference uses the source spelling's mathematical ratio, not a +// production spec or emitted instruction. Wider stores observe every alias. +const x86ZeroExtendByteOracle = ` +static int checkExtend( + void (*fn)(uint8_t *, const uint8_t *, const uint8_t *, uint64_t), + const char *name, unsigned width, unsigned input, unsigned output, + int mode, int legacy, int alias, int memory +) { + const uint64_t masks[] = {0, 1, UINT64_C(0x55aa55aa55aa55aa), UINT64_MAX}; + for (unsigned round = 0; round < 4; round++) { + for (unsigned m = 0; m < 4; m++) { + uint8_t aStorage[96], bStorage[96], result[74]; + uint8_t *a = aStorage + 1, *b = bStorage + 3; + memset(aStorage, 0xa7, sizeof(aStorage)); + memset(bStorage, 0xb9, sizeof(bStorage)); + memset(result, 0xc3, sizeof(result)); + for (unsigned i = 0; i < 64; i++) { + a[i] = (uint8_t)(i * 19 + round * 83 + 129); + b[i] = (uint8_t)(i * 31 + round * 59 + 13); + } + uint32_t saved, before = 0x1f80 | (round << 13) | round, after; + __asm__ volatile("stmxcsr %0" : "=m"(saved)); + __asm__ volatile("ldmxcsr %0" : : "m"(before)); + fn(result + 1, a, b, masks[m]); + __asm__ volatile("stmxcsr %0" : "=m"(after)); + __asm__ volatile("ldmxcsr %0" : : "m"(saved)); + if (after != before || result[0] != 0xc3 || result[73] != 0xc3 || + result[65] != 1 || result[66] != 0) { + fprintf(stderr, "%s CSR/flags/canary failure\n", name); + return 1; + } + const uint8_t *old = alias ? a : b; + for (unsigned i = 0; i < 64; i++) { + const unsigned part = i % output; + const unsigned source = i / output * input + part; + const int active = mode == 0 || ((masks[m] >> (i / output)) & 1); + uint8_t want; + if (i >= width) { + want = legacy ? old[i] : 0; + } else if (!active) { + want = mode == 2 ? 0 : old[i]; + } else { + want = part < input ? a[source] : 0; + } + if (result[i + 1] != want || a[i] != (uint8_t)(i * 19 + round * 83 + 129) || + b[i] != (uint8_t)(i * 31 + round * 59 + 13)) { + fprintf(stderr, "%s byte=%u got=%x want=%x\n", name, i, result[i + 1], want); + return 1; + } + } + for (unsigned i = 0; i < 96; i++) { + if ((i < 1 || i >= 65) && aStorage[i] != 0xa7) return 1; + if ((i < 3 || i >= 67) && bStorage[i] != 0xb9) return 1; + } + } + } + if (memory) { + uint8_t b[64], result[74]; + memset(b, 0x59, sizeof(b)); + memset(result, 0xc3, sizeof(result)); + if (mode != 0) { + // Irrelevant high mask bits cannot access an entirely inactive NULL. + fn(result + 1, NULL, b, UINT64_C(1) << 63); + for (unsigned i = 0; i < 64; i++) { + uint8_t want = i < width && mode == 1 ? 0x59 : 0; + if (result[i + 1] != want) return 1; + } + if (result[0] != 0xc3 || result[73] != 0xc3 || result[65] != 1 || result[66] != 0) return 1; + } + const long pageSize = sysconf(_SC_PAGESIZE); + if (pageSize <= 0) return 1; + uint8_t *pages = mmap(NULL, (size_t)pageSize * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (pages == MAP_FAILED) return 1; + if (mprotect(pages + pageSize, (size_t)pageSize, PROT_NONE)) { + munmap(pages, (size_t)pageSize * 2); + return 1; + } + // An unmasked source uses its shrinking native tuple width, not a + // complete X/Y allocation. A masked source needs one active element. + unsigned sourceBytes = mode == 0 ? width * input / output : input; + uint8_t *last = pages + pageSize - sourceBytes; + for (unsigned i = 0; i < sourceBytes; i++) ((volatile uint8_t *)last)[i] = (uint8_t)(0xd0 + i); + fn(result + 1, last, b, 1); + int failed = result[0] != 0xc3 || result[73] != 0xc3 || result[65] != 1 || result[66] != 0; + for (unsigned i = 0; i < 64; i++) { + unsigned part = i % output; + int active = mode == 0 || i < output; + uint8_t want = i >= width ? (legacy ? 0x59 : 0) : + active ? (part < input ? (uint8_t)(0xd0 + i / output * input + part) : 0) : + mode == 1 ? 0x59 : 0; + if (result[i + 1] != want || b[i] != 0x59) failed = 1; + } + if (munmap(pages, (size_t)pageSize * 2)) return 1; + if (failed) return 1; + } + return 0; +} +` + +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendCompleteBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, vector := range []bool{false, true} { + for _, width := range []int{16, 32, 64} { + if !vector && width != 16 { + continue + } + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + sourceBank := "X" + if width*ratio.input/ratio.output > 16 { + sourceBank = "Y" + } + for _, memory := range []bool{false, true} { + for mode := 0; mode < 3; mode++ { + if !vector && mode != 0 { + continue + } + for _, destination := range []int{0, 1} { + if memory && destination == 0 { + continue // Memory has no register-source alias. + } + name := fmt.Sprintf("extend%d", count) + count++ + opcode := "PMOVZX" + ratio.suffix + if vector { + opcode = "V" + opcode + } + input, mask := sourceBank+"0", "" + if memory { + input = "(BX)" + } + if mode != 0 { + mask = "K1," + if mode == 2 { + opcode += ".Z" + } + } + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),AX\nMOVQ a+8(FP),BX\nMOVQ b+16(FP),CX\nMOVQ mask+24(FP),DX\nVMOVUPS (CX),Z1\n", name) + if !memory { + assembly.WriteString("VMOVUPS (BX),Z0\n") + } + fmt.Fprintf(&assembly, "KMOVQ DX,K1\nCMPQ AX,AX\nSTC\n%s %s,%s%s%d\nVMOVUPS Z%d,(AX)\nSETCS 64(AX)\nSETNE 65(AX)\nRET\n", opcode, input, mask, bank, destination, destination) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(uint8_t*,const uint8_t*,const uint8_t*,uint64_t);\n", name) + fmt.Fprintf(&checks, " if (checkExtend(%s,\"%s\",%d,%d,%d,%d,%d,%d,%d)) return 1;\n", name, name, width, ratio.input, ratio.output, mode, x86FloatAddSubBoolInt(!vector), x86FloatAddSubBoolInt(destination == 0), x86FloatAddSubBoolInt(memory)) + } + } + } + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := "#include \n#include \n#include \n#include \n#include \n" + declarations.String() + x86ZeroExtendByteOracle + "\nint main(void) {\n" + checks.String() + " return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "zero-extend-bytes", triple, ir, oracle, prefix) + t.Logf("%d ratio/width/register-memory/mask/self-alias programs, 16 complete-byte flags/MXCSR/canary inputs each; inactive NULL and page-boundary reads", count) +} + +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendOriginalGoBytes(t *testing.T) { + _, _, _, _ = x86FloatAddSubRuntimeTools(t) + var declarations, assembly, checks strings.Builder + declarations.WriteString("package oracle\n") + checks.WriteString(`package oracle +import "testing" +func TestOriginalGoUnsignedBits(t *testing.T) { + for round := 0; round < 4; round++ { + var input, old [64]byte + for i := range input { + input[i] = byte(i*19 + round*83 + 129) + old[i] = byte(i*31 + round*59 + 13) + } +`) + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, memory := range []bool{false, true} { + for _, alias := range []bool{false, true} { + if memory && alias { + continue + } + name := fmt.Sprintf("original%d", count) + count++ + fmt.Fprintf(&declarations, "func %s(a,b,out *byte)\n", name) + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),CX\nMOVOU (BX),X1\n", name) + source, destination := "(AX)", "X1" + if !memory { + assembly.WriteString("MOVOU (AX),X0\n") + source = "X0" + if alias { + destination = "X0" + } + } + fmt.Fprintf(&assembly, "CMPQ AX,AX\nSTC\nPMOVZX%s %s,%s\nMOVOU %s,(CX)\nSETCS 16(CX)\nSETNE 17(CX)\nRET\n", ratio.suffix, source, destination, destination) + fmt.Fprintf(&checks, ` + { + var result [20]byte + for i := range result { result[i] = 0xc3 } + %[1]s(&input[0], &old[0], &result[1]) + if result[0] != 0xc3 || result[19] != 0xc3 || result[17] != 1 || result[18] != 0 { + t.Fatalf("%[1]s flags/canary failure: %%x", result) + } + for i := 0; i < 16; i++ { + want := byte(0) + if i%%%[2]d < %[3]d { want = input[i/%[2]d*%[3]d+i%%%[2]d] } + if result[i+1] != want { t.Fatalf("%[1]s byte %%d got %%x want %%x", i, result[i+1], want) } + } + } +`, name, ratio.output, ratio.input) + } + } + } + checks.WriteString(" }\n}\n") + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module oracle\n\ngo 1.20\n", "value.go": declarations.String(), + "value_amd64.s": assembly.String(), "value_test.go": checks.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", "test", "-p=1", "-vet=off", "-count=1", ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH=amd64", "CGO_ENABLED=0", "GOMAXPROCS=2") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual original Go unsigned byte oracle: %v\n%s", err, output) + } + t.Logf("%d original legacy SSE4.1 register/memory/self-alias functions, 4 complete-byte/flags/canary inputs each; not native VEX/EVEX execution", count) +} + +func TestCrossLinuxRuntimeMatrixX86PackedZeroExtendIndependentFPBytes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + var assembly, declarations, checks strings.Builder + sigs := map[string]FuncSig{} + count := 0 + for _, ratio := range x86ZeroExtendTestRatios { + for _, width := range []int{16, 32, 64} { + for _, legacy := range []bool{false, true} { + if legacy && width != 16 { + continue + } + name := fmt.Sprintf("fpExtend%d", count) + count++ + bank := map[int]string{16: "X", 32: "Y", 64: "Z"}[width] + opcode := "VPMOVZX" + ratio.suffix + if legacy { + opcode = "PMOVZX" + ratio.suffix + } + var cArgs, cInputs []string + sig := FuncSig{Name: name, Ret: Void} + for i := 0; i < 32; i++ { + cArgs = append(cArgs, "uint8_t") + cInputs = append(cInputs, fmt.Sprintf("input[%d]", i)) + sig.Args = append(sig.Args, I8) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(i), Type: I8, Index: i, Field: -1}) + } + sig.Args = append(sig.Args, Ptr) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: 32, Type: Ptr, Index: 32, Field: -1}) + sigs[name] = sig + fmt.Fprintf(&assembly, "TEXT %s(SB),4,$0-40\nMOVQ out+32(FP),AX\nMOVL $89,BX\nMOVD BX,X1\nVPBROADCASTB X1,Z1\n%s a+0(FP),%s1\nVMOVUPS Z1,(AX)\nRET\n", name, opcode, bank) + fmt.Fprintf(&declarations, "extern void %s(%s, uint8_t *);\n", name, strings.Join(cArgs, ", ")) + fmt.Fprintf(&checks, " memset(result, 0xc3, sizeof(result));\n %s(%s, result + 1);\n", name, strings.Join(cInputs, ", ")) + fmt.Fprintf(&checks, ` for (unsigned i = 0; i < 64; i++) { + unsigned part = i %% %d; + uint8_t want = i >= %d ? %d : part < %d ? input[i / %d * %d + part] : 0; + if (result[i + 1] != want) { + fprintf(stderr, "%s independent FP byte %%u: %%x != %%x\n", i, result[i + 1], want); + return 1; + } + } + if (result[0] != 0xc3 || result[65] != 0xc3) return 1; +`, ratio.output, width, map[bool]int{false: 0, true: 89}[legacy], ratio.input, ratio.output, ratio.input, name) + } + } + } + requireX86GoAssemblerResult(t, "amd64", assembly.String(), true) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + oracle := "#include \n#include \n#include \n" + declarations.String() + ` +int main(void) { + for (unsigned round = 0; round < 4; round++) { + uint8_t input[32], result[66]; + for (unsigned i = 0; i < 32; i++) input[i] = (uint8_t)(round * 83 + i * 19 + 129); +` + checks.String() + " }\n return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "zero-extend-fp-bytes", triple, ir, oracle, prefix) + t.Logf("%d widening programs, 32 independently passed byte fields, 4 full 64-byte alias/canary inputs each", count) +} diff --git a/amd64_parallel_bits_cross_test.go b/amd64_parallel_bits_cross_test.go new file mode 100644 index 000000000..48b7a6f25 --- /dev/null +++ b/amd64_parallel_bits_cross_test.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "testing" +) + +// This name is intentionally included by the required cross-runtime CI regex. +func TestCrossLinuxRuntimeMatrixParallelBits386(t *testing.T) { + if os.Getenv("PLAN9ASM_CROSS_EXEC") != "1" { + t.Skip("set PLAN9ASM_CROSS_EXEC=1 for the required Linux cross-runtime matrix") + } + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("cross-runtime driver requires linux/amd64") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, tool := range []string{"i686-linux-gnu-gcc", "qemu-i386"} { + if _, err := exec.LookPath(tool); err != nil { + t.Fatalf("required cross-runtime tool %s: %v", tool, err) + } + } + var source, decl, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("parallel386_%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-12\nMOVL value+0(FP),AX\nMOVL mask+4(FP),BX\n", name) + line := op + " (BX),AX,CX\n" + if raw { + code := assembleX87ControlBytes(t, "386", "TEXT probe(SB),4,$0-0\n"+line+"RET\n") + for _, b := range code[:len(code)-1] { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + } else { + source.WriteString(line) + } + source.WriteString("MOVL CX,ret+8(FP)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{I32, Ptr}, Ret: I32, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I32, Index: 0, Field: -1}, {Offset: 4, Type: Ptr, Index: 1, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I32, Index: 0, Field: -1}}}} + fmt.Fprintf(&decl, "extern uint32_t %s(uint32_t,const uint32_t *);\n", name) + deposit := 0 + if strings.HasPrefix(op, "PDEP") { + deposit = 1 + } + fmt.Fprintf(&checks, "if(check(%s,mask,%d)) return %d;\n", name, deposit, index) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +#include +#include +` + decl.String() + ` +static void memory_fault(int signal_number) { (void)signal_number; _exit(98); } +static uint32_t reference(uint32_t value,uint32_t mask,int deposit) { + uint32_t result=0,cursor=1; + while(mask) {uint32_t low=mask&(-mask);if(deposit ? (value&cursor)!=0 : (value&low)!=0) result|=deposit?low:cursor;mask&=mask-1;cursor<<=1;} + return result; +} +static int check(uint32_t(*fn)(uint32_t,const uint32_t *),uint32_t *mask,int deposit) { + uint32_t state=0x9e3779b9; + for(int i=0;i<256;i++) { + state^=state<<13;state^=state>>17;state^=state<<5;uint32_t value=state; + state^=state<<13;state^=state>>17;state^=state<<5;*mask=state; + if(i==0)*mask=0;if(i==1)*mask=0xffffffffU;if(i>=2&&i<34)*mask=1U<<(i-2); + if(fn(value,mask)!=reference(value,*mask,deposit)) {fprintf(stderr,"386 parallel bits mismatch case=%d deposit=%d\n",i,deposit);return 1;} + } + return 0; +} +int main(void) { + if(signal(SIGSEGV,memory_fault)==SIG_ERR)return 92; + long page=sysconf(_SC_PAGESIZE);if(page<4)return 90; + uint8_t *memory=mmap(0,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if(memory==MAP_FAILED||mprotect(memory+page,page,PROT_NONE))return 91; + uint32_t *mask=(uint32_t *)(memory+page-4); +` + checks.String() + "return munmap(memory,page*2)!=0;\n}\n" + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc"}, "parallel_bits_386", "i386-unknown-linux-gnu", ir, mainC, []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) +} diff --git a/amd64_polynomial_fold_composition_test.go b/amd64_polynomial_fold_composition_test.go new file mode 100644 index 000000000..3ef4f3539 --- /dev/null +++ b/amd64_polynomial_fold_composition_test.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +const x86PolynomialFoldDeclaration = "func fold(p *[128]byte, n, seed uint32) uint32\n" + +// A first-party polynomial fold, not a copied CRC implementation. Its complete +// result path crosses native pointer reads, private aligned DATA, both CLMUL +// qword selectors, byte/lane shifts, a loop backedge and a two-arm value join. +func x86PolynomialFoldSource(arch string) string { + word, move, add := 8, "MOVQ", "ADDQ" + if arch == "386" { + word, move, add = 4, "MOVL", "ADDL" + } + return fmt.Sprintf(`DATA foldPoly<>+0(SB)/8,$0x1f123bb5a7c9d001 +DATA foldPoly<>+8(SB)/8,$0xa7c1e305f1b692d3 +GLOBL foldPoly<>(SB),8,$16 + +TEXT ·fold(SB),4,$0-%d + %s p+0(FP),SI + MOVL n+%d(FP),CX + MOVL seed+%d(FP),AX + PXOR X1,X1 + PINSRD $0,AX,X1 + MOVOA foldPoly<>(SB),X3 + CMPL CX,$0 + JE finish +loop: + MOVOU (SI),X0 + PXOR X0,X1 + MOVOU X1,X2 + PCLMULQDQ $0,X3,X1 + PCLMULQDQ $17,X3,X2 + PSRLDQ $8,X2 + PXOR X2,X1 + TESTL $1,CX + JE even + MOVOU X1,X4 + PSRLQ $7,X4 + PXOR X4,X1 + JMP joined +even: + MOVOU X1,X4 + PSLLQ $3,X4 + PXOR X4,X1 +joined: + %s $16,SI + SUBL $1,CX + JNE loop +finish: + MOVOU X1,X2 + PSRLO $8,X2 + PXOR X2,X1 + PSRLQ $11,X1 + PEXTRD $0,X1,AX + MOVL AX,ret+%d(FP) + RET +`, word+12, move, word, word+4, add, word+8) +} + +func TestX86PolynomialFoldCompositionThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + source := x86PolynomialFoldSource(target.arch) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, x86PolynomialFoldDeclaration, target.arch, target.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, native := range []string{"movdqu", "movdqa"} { + if !strings.Contains(ir, "asm sideeffect \""+native+" ($1), $0\"") { + t.Fatal(api, "conditional result lost its original native load", native) + } + } + compileLLVMToObject(t, llc, target.triple, "polynomial-fold.ll", "polynomial-fold.o", ir) + } + }) + } +} + +func TestX86PolynomialFoldCompositionDoesNotCreateStandalonePermissions(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + source := x86PolynomialFoldSource(arch) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + pkg := mustGoPackage(t, "fold", "package fold\n"+x86PolynomialFoldDeclaration) + resolve := func(symbol string) string { return strings.TrimPrefix(symbol, "·") } + sigs, err := goSigsForAsmFile(pkg, file, resolve, arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + fn := file.Funcs[0] + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(fn, sigs["fold"], arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("linked-data/native-load composition leaked declaration-only authority", arch, err) + } + } + } +} + +func TestX86PolynomialFoldCompositionRejectsOneMissingProof(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for name, change := range map[string][2]string{ + "external data": {"MOVOA foldPoly<>(SB),X3", "MOVOA otherPoly<>(SB),X3"}, + "short data": {"DATA foldPoly<>+8(SB)/8,$0xa7c1e305f1b692d3\nGLOBL foldPoly<>(SB),8,$16", "GLOBL foldPoly<>(SB),8,$8"}, + "relocated data": {"$0xa7c1e305f1b692d3", "$·external(SB)"}, + "missing pointer": {"MOVOU (SI),X0", "MOVOU (DI),X0"}, + "missing joined value": {"even:\n\tMOVOU X1,X4", "even:\n\tMOVOU X5,X4"}, + "unknown data input": {"PCLMULQDQ $0,X3,X1", "PCLMULQDQ $0,X5,X1"}, + } { + t.Run(arch+"/"+name, func(t *testing.T) { + source := strings.Replace(x86PolynomialFoldSource(arch), change[0], change[1], 1) + if arch == "386" && name == "relocated data" { + source = strings.Replace(source, "+8(SB)/8,$·external(SB)", "+8(SB)/4,$·external(SB)", 1) + } + goAssemblerOracleOutput(t, arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, x86PolynomialFoldDeclaration, arch, testTargetTriple("linux", arch), api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal(api, "incomplete composition acquired a result", err) + } + } + }) + } + } +} diff --git a/amd64_polynomial_fold_runtime_unix_test.go b/amd64_polynomial_fold_runtime_unix_test.go new file mode 100644 index 000000000..403324540 --- /dev/null +++ b/amd64_polynomial_fold_runtime_unix_test.go @@ -0,0 +1,148 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +const x86PolynomialFoldOriginalGo = `package main +import ("encoding/binary"; "fmt") + +func multiply(a,b uint64) (lo,hi uint64) { + for bit:=uint(0);bit<64;bit++ { + if b>>bit&1==0 { continue } + lo^=a<>(64-bit) } + } + return +} +func reference(p *[128]byte,n,seed uint32) uint32 { + low,high:=uint64(seed),uint64(0) + for block:=uint32(0);block>7 + high^=high>>7 + } else { + low^=low<<3 + high^=high<<3 + } + } + return uint32((low^high)>>11) +} +func main() { + seeds:=[]uint32{0,1,0xffffffff,0x89abcdef} + for sample:=0;sample<8;sample++ { + var storage [144]byte + for i:=range storage { storage[i]=byte(i*71+sample*97+(i*sample)%251) } + before:=storage + for offset:=0;offset<4;offset++ { + p:=(*[128]byte)(storage[offset:offset+128]) + for n:=uint32(0);n<=8;n++ { + for _,seed:=range seeds { + got,want:=fold(p,n,seed),reference(p,n,seed) + if got!=want || storage!=before { + panic(fmt.Sprintf("original Go sample=%d offset=%d n=%d seed=%x got=%x want=%x",sample,offset,n,seed,got,want)) + } + fmt.Println(got) + } + } + } + } + for _,seed:=range seeds { + got,want:=fold(nil,0,seed),reference(nil,0,seed) + if got!=want { panic("nil/zero-length control") } + fmt.Println(got) + } +} +` + +func TestCrossLinuxRuntimeMatrixX86PolynomialFoldOriginalGoThreeAPIs(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + goos := "linux" + if strings.Contains(execution.triple, "apple") { + goos = "darwin" + } + source := x86PolynomialFoldSource("amd64") + directory := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module polynomial_fold_oracle\n\ngo 1.20\n", + "main.go": x86PolynomialFoldOriginalGo + x86PolynomialFoldDeclaration, + "fold.s": source, + } { + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + original := filepath.Join(directory, "original") + environment := []string{"GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local", "CGO_ENABLED=0", "GOARCH=amd64", "GOOS=" + goos} + dataGlobalAlignmentCommand(t, directory, environment, "go", "build", "-p=2", "-o", original, ".") + results := strings.Fields(string(execution.run(t, original))) + const samples = 8*4*9*4 + 4 + if len(results) != samples { + t.Fatalf("actual linked Go executed %d results, want %d", len(results), samples) + } + var values strings.Builder + for _, result := range results { + value, err := strconv.ParseUint(result, 10, 32) + if err != nil { + t.Fatal("original Go result is not uint32", err) + } + fmt.Fprintf(&values, "UINT32_C(%d),\n", value) + } + main := `#include +#include +#include +extern uint32_t fold(const uint8_t*,uint32_t,uint32_t); +static const uint32_t original[]={ +` + values.String() + `}; +int main(void) { + const uint32_t seeds[]={0,1,0xffffffff,0x89abcdef}; + unsigned index=0; + for(unsigned sample=0;sample<8;sample++) { + uint8_t storage[144],before[144]; + for(unsigned i=0;i<144;i++) storage[i]=(uint8_t)(i*71+sample*97+(i*sample)%251); + memcpy(before,storage,sizeof(storage)); + for(unsigned offset=0;offset<4;offset++) + for(unsigned n=0;n<=8;n++) + for(unsigned seed=0;seed<4;seed++) { + uint32_t got=fold(storage+offset,n,seeds[seed]); + if(got!=original[index] || memcmp(storage,before,sizeof(storage))) { + fprintf(stderr,"LLVM sample=%u offset=%u n=%u seed=%x got=%x original=%x\n", + sample,offset,n,seeds[seed],got,original[index]); + return 1; + } + index++; + } + } + for(unsigned seed=0;seed<4;seed++) if(fold(0,0,seeds[seed])!=original[index++]) return 2; + return index!=sizeof(original)/sizeof(original[0]); +} +` + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, x86PolynomialFoldDeclaration, "amd64", execution.triple, api) + if err != nil { + t.Fatal(api, err) + } + for _, level := range []string{"-O0", "-O2"} { + t.Run(api+"/"+level, func(t *testing.T) { + text := ir + if level == "-O2" { + text = x86MulDivRoundOptimize(t, text) + } + execution.compileAndRunLLVM(t, "polynomial-fold", text, main, level) + }) + } + } + t.Logf("%d actual Go linked loop/join results, independently checked polynomial reference, three LLVM APIs O0/O2; source identical, inputs unchanged", samples) +} diff --git a/amd64_private_aligned_frame_audit_test.go b/amd64_private_aligned_frame_audit_test.go new file mode 100644 index 000000000..8c1fe0527 --- /dev/null +++ b/amd64_private_aligned_frame_audit_test.go @@ -0,0 +1,291 @@ +package plan9asm + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// These are source/translation observations, not execution tests. In +// particular, neither missing initialization nor a missing SP restoration +// fixture is executed. A typed ABI0 declaration is not a private-frame proof. +func x86PrivateAlignedFrameAuditFixture(t *testing.T, operation string) (string, FuncSig, GoPackage) { + t.Helper() + const goSource = `package frameaudit +func Value(a, b, c, d uint64) uint64 +` + return x86PrivateAlignedFrameAuditTypedFixture(t, operation, goSource) +} + +func x86PrivateAlignedFrameAuditTypedFixture(t *testing.T, operation, goSource string) (string, FuncSig, GoPackage) { + t.Helper() + fset := token.NewFileSet() + syntax, err := parser.ParseFile(fset, "frameaudit.go", goSource, 0) + if err != nil { + t.Fatal(err) + } + sizes := types.SizesFor("gc", "amd64") + typed, err := (&types.Config{Sizes: sizes}).Check("frameaudit", fset, []*ast.File{syntax}, nil) + if err != nil { + t.Fatal(err) + } + declaration, ok := typed.Scope().Lookup("Value").(*types.Func) + if !ok { + t.Fatal("missing actual type-checked first-party declaration") + } + sig, err := DeriveGoFuncABI0Signature("frameaudit.Value", declaration, "amd64", sizes) + if err != nil { + t.Fatal(err) + } + source := fmt.Sprintf(`TEXT ·Value(SB),4,$160-40 +MOVQ a+0(FP),AX +MOVQ b+8(FP),CX +MOVQ c+16(FP),DX +MOVQ d+24(FP),BX +MOVQ SP,R11 +MOVQ $31,R9 +NOTQ R9 +ANDQ R9,SP +ADDQ $32,SP +MOVQ R11,96(SP) +MOVQ AX,16(SP) +MOVQ CX,24(SP) +MOVQ DX,48(SP) +MOVQ BX,56(SP) +MOVOU 48(SP),X0 +%s +MOVQ X0,AX +MOVQ 96(SP),R11 +MOVQ R11,SP +MOVQ AX,ret+32(FP) +RET +`, operation) + return source, sig, GoPackage{Path: "frameaudit", Types: typed, Syntax: []*ast.File{syntax}} +} + +func x86PrivateAlignedFrameAuditAPIs(t *testing.T, source string, sig FuncSig, pkg GoPackage, check func(*testing.T, error)) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + resolve := func(symbol string) string { + return strings.Replace(symbol, "·", "frameaudit.", 1) + } + sigs := map[string]FuncSig{sig.Name: sig} + if declaration, ok := pkg.Types.Scope().Lookup("callee").(*types.Func); ok { + callee, err := DeriveGoFuncABI0Signature("frameaudit.callee", declaration, "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + sigs[callee.Name] = callee + } + opt := Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: resolve, Sigs: sigs, + } + t.Run("Translate", func(t *testing.T) { + _, err := Translate(file, opt) + check(t, err) + }) + t.Run("TranslateModuleInContext", func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + check(t, err) + }) + t.Run("TranslateGoModule", func(t *testing.T) { + translation, err := TranslateGoModule(pkg, []byte(source), GoModuleOptions{ + GOOS: "linux", GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + ResolveSym: resolve, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + check(t, err) + }) +} + +func TestX86PrivateAlignedFrameAuditOriginalGoM128Source(t *testing.T) { + // Full legacy opcode membership of the bridges inspected in this audit. + // The count instructions have their own exact-opcode fault contract; this + // list does not impose an alignment requirement on them. + operations := []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", + "ADDPS 16(SP),X0", "ADDPD 16(SP),X0", "SUBPS 16(SP),X0", "SUBPD 16(SP),X0", + "MULPS 16(SP),X0", "MULPD 16(SP),X0", "DIVPS 16(SP),X0", "DIVPD 16(SP),X0", + "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + "PSLLW 16(SP),X0", "PSLLL 16(SP),X0", "PSLLQ 16(SP),X0", + "PSRLW 16(SP),X0", "PSRLL 16(SP),X0", "PSRLQ 16(SP),X0", + } + if len(operations) != 21 { + t.Fatal("incomplete audited legacy bridge membership") + } + for _, operation := range operations { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + source, _, _ := x86PrivateAlignedFrameAuditFixture(t, operation) + listing := string(x86InsertExtractGoListing(t, "amd64", source)) + for _, required := range []string{ + "locals=0xa8", + "MOVQ\ta+176(FP), AX", "MOVQ\tb+184(FP), CX", + "MOVQ\tc+192(FP), DX", "MOVQ\td+200(FP), BX", "MOVQ\tAX, ret+208(FP)", + "ANDQ\tR9, SP", "ADDQ\t$32, SP", "MOVQ\tR11, SP", + "RET", + } { + if !strings.Contains(listing, required) { + t.Fatalf("actual Go source frame/prologue/restoration missing %q:\n%s", required, listing) + } + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + code = append(code, x86InsertExtractListedBytes(line)...) + } + if len(code) == 0 { + t.Fatal("original Go listing lacks physical instruction bytes") + } + // Identify complete observed machine sequences, never a version + // string or autosize alone. Older Go saves BP in the 160(SP) + // slot; current Go pushes BP. Both reserve 168 bytes, but their + // producer and restoration instructions are not interchangeable. + physicalForm := "" + for _, form := range []struct { + name string + prefix, suffix []byte + instructions []string + }{ + { + name: "pushed_BP", + prefix: []byte{0x55, 0x48, 0x89, 0xe5, 0x48, 0x81, 0xec, 0xa0, 0, 0, 0}, + suffix: []byte{0x48, 0x81, 0xc4, 0xa0, 0, 0, 0, 0x5d, 0xc3}, + instructions: []string{ + "PUSHQ\tBP", "MOVQ\tSP, BP", "SUBQ\t$160, SP", "ADDQ\t$160, SP", "POPQ\tBP", + }, + }, + { + name: "slot_saved_BP", + prefix: []byte{0x48, 0x81, 0xec, 0xa8, 0, 0, 0, 0x48, 0x89, 0xac, 0x24, 0xa0, 0, 0, 0, 0x48, 0x8d, 0xac, 0x24, 0xa0, 0, 0, 0}, + suffix: []byte{0x48, 0x8b, 0xac, 0x24, 0xa0, 0, 0, 0, 0x48, 0x81, 0xc4, 0xa8, 0, 0, 0, 0xc3}, + instructions: []string{ + "SUBQ\t$168, SP", "MOVQ\tBP, 160(SP)", "LEAQ\t160(SP), BP", "MOVQ\t160(SP), BP", "ADDQ\t$168, SP", + }, + }, + } { + if !bytes.HasPrefix(code, form.prefix) || !bytes.HasSuffix(code, form.suffix) { + continue + } + for _, instruction := range form.instructions { + if !strings.Contains(listing, instruction) { + t.Fatalf("actual %s object lacks its paired listing instruction %q", form.name, instruction) + } + } + physicalForm = form.name + } + if physicalForm == "" { + t.Fatalf("actual Go physical prologue/epilogue lacks an observed complete pairing: %x", code) + } + t.Logf("actual Go original accepted; TEXT160 + savedBP8; physicalForm=%s operation=%s sourceSHA=%x codeSHA=%x code=%x", physicalForm, operation, sha256.Sum256([]byte(source)), sha256.Sum256(code), code) + if strings.HasPrefix(operation, "PADDB ") { + t.Logf("independent original Go frame/slot listing:\n%s", listing) + } + }) + } +} + +func TestX86PrivateAlignedFrameAuditBoundedSourceStillNeedsContract(t *testing.T) { + for _, operation := range []string{ + "PADDB 16(SP),X0", "PADDW 16(SP),X0", "PADDL 16(SP),X0", "PADDQ 16(SP),X0", + "PMADDWL 16(SP),X0", "CMPPS 16(SP),X0,$0", "CMPPD 16(SP),X0,$0", + } { + t.Run(strings.Fields(operation)[0], func(t *testing.T) { + source, sig, pkg := x86PrivateAlignedFrameAuditFixture(t, operation) + x86PrivateAlignedFrameAuditAPIs(t, source, sig, pkg, func(t *testing.T, err error) { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("typed scalar declaration alone granted native private-frame memory: %v", err) + } + t.Logf("bounded initialized source remains Context pending its private-frame producer: %v", err) + }) + }) + } +} + +// RED checkpoint: the legacy arithmetic bridge currently admits these +// unproved virtual-SP inputs. This is a missing source contract observation, +// not a claim that any third-party function has been executed incorrectly. +func TestX86PrivateAlignedFrameAuditUnprovedNativeFloatingMemory(t *testing.T) { + mutations := []struct { + name string + edit func(string) string + goSource string + }{ + {"unaligned source residue", func(source string) string { + return strings.Replace(source, "ADDQ $32,SP", "ADDQ $33,SP", 1) + }, ""}, + {"uninitialized source bytes", func(source string) string { + return strings.Replace(source, "MOVQ CX,24(SP)\n", "", 1) + }, ""}, + {"source TEXT extent", func(source string) string { + return strings.Replace(source, "16(SP),X0", "152(SP),X0", 1) + }, ""}, + {"missing source SP restoration", func(source string) string { + return strings.Replace(source, "MOVQ R11,SP\n", "", 1) + }, ""}, + {"unknown alignment mask", func(source string) string { + return strings.Replace(source, "MOVQ $31,R9\nNOTQ R9", "MOVQ AX,R9", 1) + }, ""}, + {"private pointer result escape", func(source string) string { + return strings.Replace(source, "MOVQ AX,ret+32(FP)", "MOVQ R11,ret+32(FP)", 1) + }, ""}, + {"overwritten saved SP", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ $0,96(SP)", 1) + }, ""}, + {"inconsistent restore join", func(source string) string { + return strings.Replace(source, "MOVQ R11,SP", "CMPQ AX,$0\nJE afterRestore\nMOVQ R11,SP\nafterRestore:", 1) + }, ""}, + {"private pointer global escape", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nMOVQ SP,framePointer(SB)", 1) + "GLOBL framePointer(SB),8,$8\n" + }, ""}, + {"unknown incoming pointer alias", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nMOVQ SP,(BX)", 1) + }, "package frameaudit\nfunc Value(a, b, c uint64, d *uint64) uint64\n"}, + {"unproved call", func(source string) string { + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nCALL ·callee(SB)", 1) + }, "package frameaudit\nfunc Value(a, b, c, d uint64) uint64\nfunc callee()\n"}, + {"raw SP overwrite", func(source string) string { + // MOVQ AX,SP is encoded explicitly, not described by a forged + // decoder marker or an assumed neutral raw effect. + return strings.Replace(source, "MOVQ R11,96(SP)", "MOVQ R11,96(SP)\nBYTE $0x48\nBYTE $0x89\nBYTE $0xc4", 1) + }, ""}, + } + for _, op := range []string{"ADDPS", "ADDPD", "SUBPS", "SUBPD", "MULPS", "MULPD", "DIVPS", "DIVPD"} { + for _, mutation := range mutations { + t.Run(op+"/"+mutation.name, func(t *testing.T) { + source, sig, pkg := x86PrivateAlignedFrameAuditFixture(t, op+" 16(SP),X0") + if mutation.goSource != "" { + source, sig, pkg = x86PrivateAlignedFrameAuditTypedFixture(t, op+" 16(SP),X0", mutation.goSource) + } + source = mutation.edit(source) + // Object acceptance cannot authorize execution of these negative + // fixtures or stand in for source-SP/result/no-escape semantics. + x86InsertExtractGoListing(t, "amd64", source) + x86PrivateAlignedFrameAuditAPIs(t, source, sig, pkg, func(t *testing.T, err error) { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source private-frame backing admitted by native bridge: %v", err) + } + }) + }) + } + } +} diff --git a/amd64_private_legacy_read_family_test.go b/amd64_private_legacy_read_family_test.go new file mode 100644 index 000000000..cc5f8091f --- /dev/null +++ b/amd64_private_legacy_read_family_test.go @@ -0,0 +1,322 @@ +package plan9asm + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// Independent yxmov whole128 directory; MOVOA is cmd/asm's MOVO alias. +// The complete original opcode/prefix/zcase oracle is separately retained by +// TestX86GoCallerLegacyMoveCompleteGoRows, not derived from production's map. +var x86PrivateLegacyReadNames = []string{ + "MOVOU", "MOVOA", "MOVO", "MOVUPS", "MOVAPS", "MOVUPD", "MOVAPD", +} + +func x86PrivateLegacyReadOriginal(t *testing.T, body string) (*File, FuncSig, Options) { + t.Helper() + assembly := "TEXT ·kernel(SB),4,$0-32\nMOVQ a+0(FP),DI\n" + body + "\nBYTE $0xc3\n" + goAssemblerOracleOutput(t, "amd64", assembly, "", true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + sig := sigs["caller.kernel"] + if sig.Ret != Void || len(sig.Frame.Results) != 0 || len(sig.Args) != 4 || + sig.x86CallerRegions == nil || !x86CallerRegionSourceMatches(sig, file.Funcs[0], "amd64") { + t.Fatal("actual source and same pointer-only Void signature prerequisite is missing") + } + for _, argument := range sig.Args { + if argument != Ptr { + t.Fatal("fixture changed the existing four-pointer caller domain") + } + } + return file, sig, Options{Goarch: "amd64", ResolveSym: resolve, Sigs: sigs} +} + +func x86PrivateLegacyReadProved(t *testing.T, body string, width int64) (Func, FuncSig) { + t.Helper() + file, sig, options := x86PrivateLegacyReadOriginal(t, body) + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + if len(normalized.Funcs) != 1 || !x86CallerNormalizedSourceMatches(normalized.Funcs[0], sig) { + t.Fatal("whole normalized source does not match its exact caller signature") + } + fn := normalized.Funcs[0] + found := 0 + for ordinal, ins := range fn.Instrs { + // Native pointer imports have their own exact FP/physical-stack + // certificate. Select the actual DI-to-X numerical source role first. + if len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.Args[0].Mem.Base != DI || + ins.Args[1].Kind != OpReg || !isAMD64XReg(ins.Args[1].Reg) { + continue + } + access, present := ins.x86CallerMemory[0] + if !present || access.write || access.width != width || !x86CallerMemoryCertified(ins, 0) { + t.Fatalf("instruction %d lacks its exact numerical-source read%d certificate", ordinal, width) + } + found++ + } + if found != 1 { + t.Fatalf("source has %d certified reads, want exactly one", found) + } + return fn, sig +} + +func TestX86PrivateLegacyReadObserverCompleteSourceForms(t *testing.T) { + // Input is the existing numerical slice-origin/nonalias contract, not an + // allocation or native-pointer extent. The independent producer must pass + // before these leaves can diagnose the private observer. + for _, op := range x86PrivateLegacyReadNames { + for _, destination := range []string{"X1", "X0"} { + t.Run(op+"/register_"+destination, func(t *testing.T) { + body := "MOVOU (DI),X0\n" + op + " X0," + destination + "\nPEXTRD $3," + destination + ",AX" + fn, sig := x86PrivateLegacyReadProved(t, body, 16) + if err := CheckX86ABI0ResultInitialization(fn, sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("public declaration-only Void query acquired caller memory: %v", err) + } + if err := checkX86ScalarFPResultInitialization(fn, sig, "amd64"); err != nil { + t.Errorf("private observer lost a same-source exact read16 before %s register transfer: %v", op, err) + } + }) + } + } + for _, op := range []string{"MOVOU", "MOVUPS", "MOVUPD"} { + for _, address := range []string{"(DI)", "4(DI)", "4(DI)(BX*2)"} { + t.Run(op+"/read_"+address, func(t *testing.T) { + body := "MOVQ $1,BX\n" + op + " " + address + ",X0\nPEXTRD $3,X0,AX" + fn, sig := x86PrivateLegacyReadProved(t, body, 16) + if err := checkX86ScalarFPResultInitialization(fn, sig, "amd64"); err != nil { + t.Errorf("private observer discarded a proved full-width source read: %v", err) + } + }) + } + } +} + +func TestX86PrivateLegacyReadObserverExistingByteCounterparts(t *testing.T) { + // These are ordinary logical byte-effect models, never caller producers. + // In particular, logical FP/AUTO bytes do not prove native alignment/backing. + forms := []string{ + "X0,X1", "X0,X0", "a+0(FP),X1", "X0,ret0+16(FP)", + "value-32(SP),X1", "X0,value-16(SP)", + } + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86PrivateLegacyReadNames { + for _, operands := range forms { + t.Run(arch+"/"+op+"/"+operands, func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, op+" "+operands) + effect, known, err := x86TypedLegacyPackedMoveByteEffect(ins.Op, ins, arch) + if !known || err != nil || len(effect.operands) != 2 || effect.flagsRead != 0 || + effect.flagsDefined != 0 || effect.flagsKilled != 0 || effect.dynamicMemory { + t.Fatalf("whole128 descriptor differs from the independent move rows: %+v/%v", effect, err) + } + for index, operand := range effect.operands { + if operand.index != index || operand.width != 16 || operand.read != (index == 0) || + operand.write != (index == 1) || !reflect.DeepEqual(operand.operand, ins.Args[index]) { + t.Fatal("typed effect changed source/destination, direction or exact width") + } + } + owned := make([]bool, 32) + state := x86ResultState{fp: make([]bool, 32), auto: make([]bool, 32), flags: x86ByteFlagCF | x86ByteFlagOF} + for offset := range owned { + owned[offset] = true + state.fp[offset], state.auto[offset] = offset < 16, offset < 16 + } + state.xmm[0] = 0xffff + for _, transfer := range []func(Instr, *x86ResultState, []bool, int64, string) bool{x86ResultTransfer, x86ScalarFPResultTransfer} { + copy := state.clone() + if !transfer(ins, ©, owned, 16, arch) || copy.flags != state.flags { + t.Fatal("existing full-byte transition failed or changed numerical flags") + } + if !x86ResultRead(ins.Args[1], 16, ©, owned, arch) { + t.Fatal("complete move failed to define exactly its observed destination") + } + } + }) + } + for missing := 0; missing < 16; missing++ { + t.Run(fmt.Sprintf("%s/%s/missing-byte%d", arch, op, missing), func(t *testing.T) { + ins := x86CallerLegacyMoveInstruction(t, op+" X0,X1") + state := x86ResultState{flags: x86ByteFlagCF | x86ByteFlagOF} + state.xmm[0], state.xmm[1] = 0xffff&^(uint16(1)< 0 && isX86RawDirective(fn.Instrs[start-1]) { + start-- + } + return start > 0 && start < len(fn.Instrs) && fn.Instrs[start-1].Op == OpRET +} + +func x86RawTailLayoutObserved(file *File, symbol string) bool { + target := strings.TrimSuffix(symbol, "<>") + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + for _, operand := range ins.Args { + if fn.Sym == symbol && operand.Kind == OpMem && operand.Mem.Base == PC { + return true + } + + ref := "" + switch operand.Kind { + case OpSym: + ref = operand.Sym + case OpMem: + ref = operand.Mem.Sym + } + if ref == "" { + continue + } + ref = strings.TrimSpace(strings.TrimPrefix(ref, "*")) + base, offset, ok := parseSBRef(ref) + if !ok || strings.TrimSuffix(strings.TrimPrefix(base, "$"), "<>") != target { + continue + } + if offset != 0 || !isX86DirectControlTransfer(ins.Op) { + return true + } + } + } + } + for _, datum := range file.Data { + base, _, ok := parseSBRef(datum.Addr) + if ok && strings.TrimSuffix(base, "<>") == target { + return true + } + } + return false +} + +func isX86DirectControlTransfer(op Op) bool { + return op == OpCALL || op == OpJMP || op == OpRET || isAMD64ConditionalBranch(op) +} + +// NormalizeRawFileForTranslation performs the whole-file raw-byte analysis +// before a compile-only caller partitions functions into bounded modules. +// Address-sensitive TEXT bodies must be identified while all references are +// still visible. TranslateModuleInContext may safely normalize the result +// again after partitioning. This preparation API has no function signatures: +// final typed FP return-contract validation runs when translation consumes its +// existing Options.Sigs. Preparation does not prove a callable return ABI. +func NormalizeRawFileForTranslation(file *File, goarch string) (*File, error) { + return normalizeX86RawFile(file, goarch) +} + +// preserveAddressSensitiveX86RawText identifies raw-only TEXT bodies whose +// exact byte layout is observed elsewhere in the same source file. Typical +// examples are boot trampolines patched through symbol+offset references and +// TEXT-backed descriptor/data tables. Decoding such a body into LLVM +// operations would make the referenced byte offsets false even when the +// instruction semantics happened to match. +// +// Raw-only bodies that are merely called or jumped to are deliberately not +// selected: those remain on the normal semantic decoder path. +func preserveAddressSensitiveX86RawText(file *File) error { + if file == nil || file.Arch != ArchAMD64 { + return nil + } + candidates := make(map[string]int) + encoded := make(map[int][]byte) + alignments := make(map[int]int64) + for index := range file.Funcs { + fn := &file.Funcs[index] + candidateName := strings.TrimSuffix(fn.Sym, "<>") + if fn.X86RawText != nil { + candidates[candidateName] = index + encoded[index] = append([]byte(nil), fn.X86RawText...) + alignments[index] = fn.X86RawAlign + continue + } + var code []byte + hasRaw := false + rawOnly := true + var alignment int64 + for _, ins := range fn.Instrs { + if ins.Op == OpTEXT { + continue + } + if form, known, err := x86DirectiveProperties(ins); known && form.effect == x86DirectiveLayout && !hasRaw { + if err != nil { + rawOnly = false + break + } + if form.alignment > alignment { + alignment = form.alignment + } + continue + } + if !isX86RawDirective(ins) { + rawOnly = false + break + } + part, err := x86RawDirectiveBytes(ins) + if err != nil { + return fmt.Errorf("%s: %w", fn.Sym, err) + } + code = append(code, part...) + hasRaw = true + } + if !rawOnly || !hasRaw { + continue + } + candidates[candidateName] = index + encoded[index] = code + alignments[index] = alignment + } + if len(candidates) == 0 { + return nil + } + + addressed := make(map[int]bool) + markSymbol := func(symbol string) { + base, _, ok := parseSBRef(strings.TrimSpace(strings.TrimPrefix(symbol, "*"))) + if !ok { + return + } + base = strings.TrimSuffix(base, "<>") + if index, exists := candidates[base]; exists { + addressed[index] = true + } + } + for _, fn := range file.Funcs { + for _, ins := range fn.Instrs { + op := Op(strings.ToUpper(string(ins.Op))) + if op == OpCALL || op == OpJMP || op == OpRET || isAMD64ConditionalBranch(op) { + continue + } + for _, operand := range ins.Args { + switch operand.Kind { + case OpSym: + markSymbol(operand.Sym) + case OpMem: + if operand.Mem.Sym != "" { + markSymbol(operand.Mem.Sym) + } + } + } + } + } + for index := range addressed { + file.Funcs[index].X86RawText = append([]byte(nil), encoded[index]...) + file.Funcs[index].X86RawAlign = alignments[index] + } + return nil +} + +// decodeX86RawDirectives replaces consecutive BYTE/WORD/LONG/QUAD directives +// with the Plan 9 instructions represented by those bytes. The decoder and +// syntax printer come from Go's architecture tables; the resulting instruction +// still has to pass the ordinary plan9asm operand-form and semantic lowerers. +// This is deliberately not an opaque inline-assembly escape hatch. +func decodeX86RawDirectives(fn Func, goarch string) (Func, error) { + mode := 64 + if goarch == "386" { + mode = 32 + } else if goarch != "" && goarch != "amd64" { + return fn, fmt.Errorf("raw x86 directives require GOARCH 386 or amd64, got %q", goarch) + } + + knownLabels := make(map[string]bool) + for _, ins := range fn.Instrs { + if ins.Op == OpLABEL && len(ins.Args) == 1 && ins.Args[0].Kind == OpLabel { + knownLabels[ins.Args[0].Sym] = true + } + } + + var normalized []Instr + for i := 0; i < len(fn.Instrs); { + if !isX86RawDirective(fn.Instrs[i]) { + normalized = append(normalized, fn.Instrs[i]) + i++ + continue + } + + start := i + var code []byte + var rawLines []string + opaque := make(map[int]x86RawDecodedInstruction) + for i < len(fn.Instrs) { + if isX86RawDirective(fn.Instrs[i]) { + encoded, err := x86RawDirectiveBytes(fn.Instrs[i]) + if err != nil { + return fn, err + } + code = append(code, encoded...) + rawLines = append(rawLines, fn.Instrs[i].Raw) + i++ + continue + } + // Keep a run of named SSE constant reads in the same physical + // layout only when raw bytes resume after it. Go's encoder lengths + // are checked against the official assembler in tests. + end := i + for end < len(fn.Instrs) { + if _, ok := x86RawMixedSSESymbolReadSize(fn.Instrs[end]); !ok { + break + } + end++ + } + if end == i || end == len(fn.Instrs) || !isX86RawDirective(fn.Instrs[end]) { + break + } + for i < end { + ins := fn.Instrs[i] + length, _ := x86RawMixedSSESymbolReadSize(ins) + opaque[len(code)] = x86RawDecodedInstruction{length: length, instrs: []Instr{ins}} + code = append(code, make([]byte, length)...) + rawLines = append(rawLines, ins.Raw) + i++ + } + } + // Go appends BYTE/WORD/LONG/QUAD after a final RET to the previous + // TEXT, even when GLOBL declarations intervene. Without a following + // source label, the tail has no control-flow entry and is not an + // executable instruction stream. Generated tables can therefore + // contain arbitrary bytes, including apparent branches. Keep labeled + // tails on the normal decoder path so reachable code never disappears. + if i == len(fn.Instrs) && start > 0 && fn.Instrs[start-1].Op == OpRET { + continue + } + rawGroup := strings.Join(rawLines, "; ") + if mode == 64 { + if target, ok := x86RawRetpolineTarget(code); ok { + normalized = append(normalized, Instr{ + Op: OpJMP, + Args: []Operand{{Kind: OpReg, Reg: target}}, + Raw: fmt.Sprintf("JMP %s /* decoded retpoline from %s */", target, rawGroup), + }) + continue + } + } + decoded, err := decodeX86RawDirectiveGroupWithOpaque(code, mode, start, rawGroup, knownLabels, opaque) + if err != nil { + return fn, err + } + for index := range decoded { + decoded[index].x86SourceRaw = &x86SourceRawInstruction{ + op: decoded[index].Op, + args: append([]Operand(nil), decoded[index].Args...), + } + } + normalized = append(normalized, decoded...) + } + fn.Instrs = normalized + return fn, nil +} + +type x86RawDecodedInstruction struct { + length int + instrs []Instr +} + +// decodeX86RawDirectiveGroup follows every reachable path through one raw +// directive group. Internal relative JMP/Jcc targets become synthetic source +// labels, so backward edges and both short and near conditional branches keep +// their control-flow semantics. Bytes skipped by all reachable paths remain +// uninterpreted, which permits the embedded signatures and constant data used +// by generated assembly. Any edge outside the group or into an instruction is +// still rejected rather than being guessed from final linked layout. +func decodeX86RawDirectiveGroup(code []byte, mode, start int, rawGroup string, knownLabels map[string]bool) ([]Instr, error) { + return decodeX86RawDirectiveGroupWithOpaque(code, mode, start, rawGroup, knownLabels, nil) +} + +func decodeX86RawDirectiveGroupWithOpaque( + code []byte, mode, start int, rawGroup string, + knownLabels map[string]bool, opaque map[int]x86RawDecodedInstruction, +) ([]Instr, error) { + decodedByOffset := make(map[int]x86RawDecodedInstruction) + labels := make(map[int]string) + addressTargets := make(map[int]int) + var literalRanges []x86RawLiteralRange + recordLiteral := func(offset int, literal x86RawLiteralRange) { + literal.source = offset + literalRanges = append(literalRanges, literal) + } + owners := make([]int, len(code)) + for i := range owners { + owners[i] = -1 + } + for offset, named := range opaque { + if named.length <= 0 || offset < 0 || offset+named.length > len(code) { + return nil, fmt.Errorf("invalid named x86 source span at byte %d: %q", offset, rawGroup) + } + for current := offset; current < offset+named.length; current++ { + if owners[current] != -1 { + return nil, fmt.Errorf("overlapping named x86 source spans at byte %d: %q", current, rawGroup) + } + owners[current] = offset + } + } + queue := []int{0} + queued := map[int]bool{0: true} + + labelFor := func(target int) string { + if label := labels[target]; label != "" { + return label + } + base := fmt.Sprintf("raw_jcc_%d_%d", start, target) + label := base + for suffix := 2; knownLabels[label]; suffix++ { + label = fmt.Sprintf("%s_%d", base, suffix) + } + knownLabels[label] = true + labels[target] = label + return label + } + enqueue := func(target int) error { + if target < 0 || target > len(code) { + return fmt.Errorf("raw x86 PC-relative target byte %d at instruction %d is outside directive group: %q", target, start, rawGroup) + } + if target < len(code) && owners[target] >= 0 && owners[target] != target { + return fmt.Errorf("raw x86 PC-relative target byte %d at instruction %d is not an instruction boundary: %q", target, start, rawGroup) + } + labelFor(target) + if !queued[target] { + queued[target] = true + queue = append(queue, target) + } + return nil + } + markInstruction := func(offset, length int) error { + if length <= 0 || offset+length > len(code) { + return fmt.Errorf("raw x86 instruction at instruction %d byte %d has invalid length %d: %q", start, offset, length, rawGroup) + } + for current := offset; current < offset+length; current++ { + if owners[current] >= 0 && owners[current] != offset { + return fmt.Errorf("raw x86 instruction at instruction %d byte %d overlaps instruction at byte %d: %q", start, offset, owners[current], rawGroup) + } + } + for current := offset; current < offset+length; current++ { + owners[current] = offset + } + return nil + } + annotate := func(instrs []Instr) []Instr { + for i := range instrs { + instrs[i].Raw = fmt.Sprintf("%s /* decoded from %s */", instrs[i].Raw, rawGroup) + } + return instrs + } + + for len(queue) != 0 { + blockStart := queue[0] + queue = queue[1:] + if blockStart == len(code) { + continue + } + if owners[blockStart] >= 0 && owners[blockStart] != blockStart { + return nil, fmt.Errorf("raw x86 PC-relative target byte %d at instruction %d is not an instruction boundary: %q", blockStart, start, rawGroup) + } + for offset := blockStart; offset < len(code); { + if _, exists := decodedByOffset[offset]; exists { + break + } + if owners[offset] >= 0 && owners[offset] != offset { + return nil, fmt.Errorf("raw x86 reachable byte %d at instruction %d is inside instruction at byte %d: %q", offset, start, owners[offset], rawGroup) + } + if named, ok := opaque[offset]; ok { + decodedByOffset[offset] = named + offset += named.length + continue + } + if instruction, length, ok, err := decodedX86VectorZeroInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector zero at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawADX(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ADX at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + if literal.last > literal.first { + recordLiteral(offset, literal) + } + offset += length + continue + } + if instruction, length, ok, err := decodedX86ExtendedPrefetchInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 extended prefetch at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EnqueueInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 enqueue at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86WaitPackageInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 WAITPKG at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok := decodedX86EndBranchInstruction(code[offset:]); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok := decodedX86SystemManagementInstruction(code[offset:]); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + + if instruction, length, ok, err := decodedX86SegmentMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 segment-register move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if syntax, length, ok := decodedX86RandomInstruction(code[offset:], mode); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instrs, err := parseDecodedX86Instruction(syntax) + if err != nil { + return nil, fmt.Errorf("parse decoded raw x86 instruction %q at instruction %d byte %d: %w", syntax, start, offset, err) + } + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: annotate(instrs)} + offset += length + continue + } + if instruction, length, ok := decodedX86OrderingFenceInstruction(code[offset:], mode); ok { + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Op, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86CacheLineWritebackInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 cache-line writeback at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawBlockBroadcastRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX block broadcast literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawBlockBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX block broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVBROADCAST128RIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VBROADCAST128 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VBROADCAST128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VBROADCAST128 at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVMOVIntegerLiteral(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VMOVD/VMOVQ literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VMOVQInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VMOVQ at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MaskMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 KMOV at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MaskVectorMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask/vector move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawEVEXPackedIntegerMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed integer move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedIntegerMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed integer move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawGatherInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 gather at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawScatterInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scatter at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawQwordPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 qword permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedSignInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed sign at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ExplicitMaskMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 explicit mask move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawDuplicateMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 duplicate move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86DuplicateMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 duplicate move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawFloatLogicalRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 floating logical literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawEVEXFloatLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX floating logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedFloatLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed floating logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86FloatingUnpackInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 floating unpack at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 in-lane floating permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawMaskBlendRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask blend literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawMaskBlendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask blend at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVariableBlendRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable blend literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VariableBlendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable blend at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawImmediatePackedBlendRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 immediate packed blend literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawCLMULRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 carryless multiply literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawCLMULInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 carryless multiply at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ImmediatePackedBlendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 immediate packed blend at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawBinaryFloatRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 binary floating literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXBinaryFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX binary floating instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedWidenConversionRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed widening conversion literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedWidenConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed widening conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarFloatToIntegerInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar float-to-integer instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarIntegerFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar integer-to-float instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar precision conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawQQToFloatRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 QWORD-to-float literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawQQToFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 QWORD-to-float conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedNumericConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw packed numeric conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86SameWidthConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 same-width packed conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedDoubleDwordInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed double/dword conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedPrecisionConvertInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed precision conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarHalfConversionRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar FP16 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarHalfConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar FP16 conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawFP16ConversionRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 FP16 conversion literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedHalfConversionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed half conversion at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawBF16RIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BF16 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawBF16Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BF16 at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MinimumPositionInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 minimum-position instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + // F3/0F38/3A is K-data MW2D, not 66/0F38/3A VPMINUW. + // Match the complete prefix key before the older min/max matcher. + if instruction, length, ok, err := decodedX86EVEXMaskDataBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask-data broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawMinMaxRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer min/max literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedIntegerMinMaxInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer min/max at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86SingleSourceNarrowInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 single-source narrowing at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVariableShiftRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 per-lane variable shift literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PerLaneVariableShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 per-lane shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawMaskCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawMaskCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask compare at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedIntegerCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer compare at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorBitCountInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector bit count at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorHighLowMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector high/low move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorHalfMemoryInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector half memory move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawKLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 K-register operation at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawMaskShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 mask shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedTestMaskInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed test mask at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedSADInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed SAD at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarFlagCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar flag compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarFlagCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar flag compare at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXVectorTestInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX vector test at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXMoveMaskInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX move mask at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVectorFloatCompareRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector floating compare literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VectorFloatCompareInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector floating compare instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXHorizontalIntegerInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 horizontal integer at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawHorizontalFloatRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 horizontal floating literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXHorizontalFloatInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX horizontal floating instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedLogicalRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed logical literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedIntegerLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed integer logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedLogicalInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed logical instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedMADDRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed multiply-add literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedMADDInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed multiply-add at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedMADDInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX packed multiply-add instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawFMA3RIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 FMA3 literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXFMA3Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX FMA3 instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXTRACT128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEXTRACT128 instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXNonTemporalMoveInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VEX non-temporal move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86InsertPSInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 INSERTPS/VINSERTPS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedScalarExtractInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed scalar extract at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPERMQInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPERMQ at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VINSERT128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VINSERT128 at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedInsertInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed insert at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScalarFloatBroadcastLiteral(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar float broadcast literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86ScalarBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedBroadcastLiteral(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed broadcast literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed broadcast at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedAlignRightInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed align-right at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86Permute128Instruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 128-bit lane permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VSHUFPSInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VSHUFPS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedShuffleInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed shuffle at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86Shuffle128BitBlocksInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VSHUFF/VSHUFI block shuffle at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVPSHUFBRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPSHUFB literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPSHUFBInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPSHUFB at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedExtendInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed extending move at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedArithmeticRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer arithmetic literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed integer arithmetic at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVariableRotateRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable rotate literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVariableRotateInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 variable rotate at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedImmediateRotateInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed immediate rotate at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawImmediateThreeVectorRIPData( + code, offset, mode, 0x03, decodedX86VectorAlignInstruction, "vector align", + ); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector align literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86VectorAlignInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VALIGN at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawImmediateThreeVectorRIPData( + code, offset, mode, 0x25, decodedX86TernaryLogicInstruction, "ternary logic", + ); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ternary logic literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86TernaryLogicInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ternary logic at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86FunnelShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 funnel shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedUniformShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed uniform shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawIndexedPermuteRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 indexed permute literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86IndexedPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 indexed permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedCompressInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed compress at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedExpandInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed expand at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86IFMAInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 IFMA at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86GFNIInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 GFNI at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPINSRInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPINSR at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXVPINSRInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX VPINSR at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawEVEXUnpackRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed unpack literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXPackedUnpackInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX packed unpack at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VPUNPCKInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPUNPCK at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VNNIInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VNNI at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VariableDwordPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPERMD/VPERMPS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawEVEXVariableDwordPermuteRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX variable dword permute literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86EVEXVariableDwordPermuteInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 EVEX variable dword permute at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86SHAInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 SHA at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawBZHIInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BZHI at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86BMI2ShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BMI2 variable shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ParallelBitsInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 parallel bits at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86BLSInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 BLS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86ANDNInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 ANDN at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86MULXInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 MULX at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RORXInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 RORX at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86AESInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 AES at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 dword-to-qword multiply at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedAbsRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPABS literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawScaledRoundRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scaled round literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVEXRoundRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VROUND literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawVectorByteShiftRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector byte shift literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVectorByteShiftInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 vector byte shift at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawScaledRoundInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scaled round at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawVEXRoundInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VROUND at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86RawPackedAbsInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 VPABS at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86VectorPackedSaturatingNarrowInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed saturating narrow at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, literal, ok, err := decodeX86RawPackedMultiplyRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed multiply literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedAverageInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed average at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + if instruction, length, ok, err := decodedX86PackedMultiplyInstruction(code[offset:], mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 packed multiply at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + + if instruction, length, ok, err := decodedX86ScalarPortIOInstruction(code[offset:]); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 scalar port I/O at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + offset += length + continue + } + + if instruction, length, literal, ok, err := decodeX86RawLegacySIMDMoveRIPData(code, offset, mode); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 legacy SIMD move literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if err := markInstruction(offset, length); err != nil { + return nil, err + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: length, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += length + continue + } + + inst, err := x86asm.Decode(code[offset:], mode) + if err != nil && mode == 64 { + if nop, ok := decodedX86REXWNOP(code[offset:]); ok { + inst, err = nop, nil + } + } + if err != nil || inst.Len <= 0 || inst.Op == 0 { + if err == nil { + if inst.Len <= 0 { + err = fmt.Errorf("decoder consumed zero bytes") + } else { + err = fmt.Errorf("incomplete or unknown instruction") + } + } + end := offset + 16 + if end > len(code) { + end = len(code) + } + return nil, fmt.Errorf("decode raw x86 directive group at instruction %d byte %d (%x): %w: %q", start, offset, code[offset:end], err, rawGroup) + } + if err := markInstruction(offset, inst.Len); err != nil { + return nil, err + } + syntax, err := decodedX86GoSyntax(inst, code[offset:offset+inst.Len]) + if err != nil { + return nil, fmt.Errorf("convert decoded raw x86 instruction at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + if inst.Op == x86asm.NOP { + // Prefix/encoding checks above still apply. A multi-byte NOP's + // ModRM/SIB fields encode its length, not a memory access or + // PC-relative reference. Keep that length for branch boundaries, + // but do not send inert fields to the physical memory handoff. + instruction := Instr{Op: "NOP", Raw: fmt.Sprintf("NOP /* decoded from %s */", rawGroup)} + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: []Instr{instruction}} + offset += inst.Len + continue + } + opName := strings.Fields(syntax)[0] + op := Op(strings.ToUpper(opName)) + if inst.PCRel != 0 { + if instruction, literal, ok, err := decodeX86RawLegacyScalarRIPData(code, offset, inst, syntax); ok { + if err != nil { + return nil, fmt.Errorf("decode raw x86 legacy scalar literal at instruction %d byte %d: %w: %q", start, offset, err, rawGroup) + } + instruction.Raw = fmt.Sprintf("%s /* decoded from %s */", instruction.Raw, rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: []Instr{instruction}} + recordLiteral(offset, literal) + offset += inst.Len + continue + } + if mode == 64 && inst.Op == x86asm.LEA { + mem, ok := inst.Args[1].(x86asm.Mem) + if ok && mem.Base == x86asm.RIP { + // x86asm exposes disp32 as an unsigned magnitude here. + // RIP-relative LEA always sign-extends that 32-bit field. + target := offset + inst.Len + int(int32(mem.Disp)) + if target < 0 || target >= len(code) { + return nil, fmt.Errorf("raw x86 local address target byte %d at instruction %d is outside directive group: %q", target, start, rawGroup) + } + instrs, err := parseDecodedX86PhysicalInstruction(inst, syntax) + if err != nil || len(instrs) != 1 || len(instrs[0].Args) != 2 { + return nil, fmt.Errorf("parse decoded raw x86 local address %q at instruction %d byte %d: %v", syntax, start, offset, err) + } + instrs[0].Args[0] = Operand{Kind: OpSym, Sym: "·__plan9asm_raw_address_pending(SB)"} + instrs[0].Raw = fmt.Sprintf("%s ·__plan9asm_raw_address_pending(SB), %s /* decoded from %s */", op, instrs[0].Args[1], rawGroup) + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: instrs} + addressTargets[offset] = target + offset += inst.Len + continue + } + } + target, ok := x86RawRelativeTarget(inst, offset) + isJump := op == OpJMP + isConditional := isAMD64ConditionalBranch(op) + if !ok || (!isJump && !isConditional) { + return nil, fmt.Errorf("raw x86 PC-relative instruction %q at instruction %d byte %d cannot be mapped safely to source labels: %q", syntax, start, offset, rawGroup) + } + if err := enqueue(target); err != nil { + return nil, fmt.Errorf("raw x86 branch at byte %d: %w", offset, err) + } + label := labelFor(target) + branch := Instr{Op: op, Args: []Operand{{Kind: OpIdent, Ident: label}}, Raw: fmt.Sprintf("%s %s /* decoded from %s */", op, label, rawGroup)} + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: []Instr{branch}} + if isConditional { + if err := enqueue(offset + inst.Len); err != nil { + return nil, fmt.Errorf("raw x86 conditional fallthrough at byte %d: %w", offset, err) + } + } + break + } + if inst.Op == x86asm.JMP { + return nil, fmt.Errorf("raw x86 indirect jump at instruction %d byte %d cannot be mapped safely: %q", start, offset, rawGroup) + } + instrs, err := parseDecodedX86PhysicalInstruction(inst, syntax) + if err != nil { + return nil, fmt.Errorf("parse decoded raw x86 instruction %q at instruction %d byte %d: %w", syntax, start, offset, err) + } + if inst.Op == x86asm.LEA && len(instrs) == 1 && len(instrs[0].Args) == 2 { + // Bare absolute SIB/no-base addresses print as numbers, but + // the Go parser treats those as symbols. Retain the actual + // typed decoder memory operand, not a fabricated SB name. + physical := normalizeDecodedX86LEASegments(normalizeDecodedX86MemoryDisplacements(inst)) + if memory, ok := physical.Args[1].(x86asm.Mem); ok && memory.Base == 0 && memory.Index == 0 && memory.Segment == 0 { + instrs[0].Args[0] = Operand{Kind: OpMem, Mem: MemRef{Off: memory.Disp}} + } + } + if inst.AddrSize != mode { + physicalMemory := false + for _, operand := range inst.Args { + if _, ok := operand.(x86asm.Mem); ok { + physicalMemory = true + } + } + switch inst.Op { + case x86asm.MASKMOVQ, x86asm.MASKMOVDQU, x86asm.MONITOR: + // These decoded forms address DI/AX implicitly. x/arch + // exposes only register (or no) operands, not a Mem. + physicalMemory = true + } + for i := range instrs { + if physicalMemory { + instrs[i].x86AddressBits = inst.AddrSize + // Generic Go syntax erases address-size prefixes. Only + // LEA's complete typed form consumes that width here; + // ordinary memory must not borrow a full-width address. + instrs[i].x86UnboundAddressOverride = inst.Op != x86asm.LEA + } + } + } + if inst.Op == x86asm.MOVSXD && inst.DataSize != 64 { + for i := range instrs { + instrs[i].x86RawMOVSXDNeedsCaller = true + } + } + if inst.Op == x86asm.CVTPI2PS || inst.Op == x86asm.CVTPI2PD { + for i := range instrs { + instrs[i].x86MMXSourceConversion = true + } + } + if inst.Op == x86asm.RET { + form := x86RawNearReturnEncoding(code[offset : offset+inst.Len]) + for i := range instrs { + if instrs[i].Op != OpRET { + continue + } + instrs[i].x86RawNearReturn = &form + if form.nativeWidth && form.cleanup == 0 { + // Keep the encoded semantics in private metadata. Removing the + // zero operand does not validate the containing source frame. + instrs[i].Args = nil + } + } + } + decodedByOffset[offset] = x86RawDecodedInstruction{length: inst.Len, instrs: annotate(instrs)} + offset += inst.Len + if inst.Op == x86asm.RET { + break + } + } + } + + for _, literal := range literalRanges { + for offset := literal.first; offset < literal.last; offset++ { + if owners[offset] >= 0 { + return nil, fmt.Errorf("RIP-relative literal byte %d overlaps reachable instruction at byte %d: %q", offset, owners[offset], rawGroup) + } + } + } + if len(addressTargets) != 0 { + first := len(code) + for _, target := range addressTargets { + if target < first { + first = target + } + } + // Only a fully unreachable suffix after a decoded RET can be moved to + // a shared global. Addressing executable bytes or a separately folded + // literal would lose source-relative pointer identity. + retBeforeSuffix := false + for offset, decoded := range decodedByOffset { + if offset >= first { + continue + } + for _, ins := range decoded.instrs { + if ins.Op == OpRET { + retBeforeSuffix = true + } + } + } + if !retBeforeSuffix { + return nil, fmt.Errorf("raw x86 local address data at byte %d has no preceding RET: %q", first, rawGroup) + } + for offset := first; offset < len(code); offset++ { + if owners[offset] >= 0 { + return nil, fmt.Errorf("raw x86 local address byte %d overlaps reachable instruction: %q", offset, rawGroup) + } + } + for _, literal := range literalRanges { + if literal.last <= first { + continue + } + if literal.first < first { + return nil, fmt.Errorf("raw x86 local address overlaps the middle of a folded literal: %q", rawGroup) + } + decoded := decodedByOffset[literal.source] + for index := range decoded.instrs { + ins := &decoded.instrs[index] + for arg := range ins.Args { + if ins.Args[arg].Kind != OpSym || ins.Args[arg].Sym != "·__plan9asm_raw_literal_pending(SB)" { + continue + } + if err := rebindX86RawStaticRead(ins, arg, "·__plan9asm_raw_address_pending(SB)"); err != nil { + return nil, err + } + ins.Raw = strings.Replace(ins.Raw, "·__plan9asm_raw_literal_pending(SB)", ins.Args[arg].Sym, 1) + // An address escape may permit writes to this pool. Force a + // runtime load rather than specializing from its initial bytes. + ins.x86RIPLiteralData = nil + // Address escape forbids constant specialization, not a + // read of this exact allocated static object. Retain the + // physical read range and bind the shared whole suffix. + ins.x86RIPMemoryRead.offset = int64(literal.first - first) + ins.x86RIPMemoryRead.object = append([]byte(nil), code[first:]...) + ins.x86RIPAddressData = code[first:] + ins.x86RIPAddressOff = literal.first - first + ins.x86RIPAddressGroup = start + } + } + decodedByOffset[literal.source] = decoded + } + for offset, target := range addressTargets { + decoded := decodedByOffset[offset] + decoded.instrs[0].x86RIPLiteral = true + decoded.instrs[0].x86RIPAddressData = code[first:] + decoded.instrs[0].x86RIPAddressOff = target - first + decoded.instrs[0].x86RIPAddressGroup = start + decodedByOffset[offset] = decoded + } + } + + offsets := make([]int, 0, len(decodedByOffset)) + for offset := range decodedByOffset { + offsets = append(offsets, offset) + } + sort.Ints(offsets) + result := make([]Instr, 0, len(offsets)+len(labels)) + for _, offset := range offsets { + if label := labels[offset]; label != "" { + result = append(result, Instr{Op: OpLABEL, Args: []Operand{{Kind: OpLabel, Sym: label}}, Raw: label + ":"}) + } + result = append(result, decodedByOffset[offset].instrs...) + } + if label := labels[len(code)]; label != "" { + result = append(result, Instr{Op: OpLABEL, Args: []Operand{{Kind: OpLabel, Sym: label}}, Raw: label + ":"}) + } + return result, nil +} + +// decodedX86SegmentMoveInstruction recognizes the complete 8C/8E segment +// register move family represented by Go 1.27's ymovtab. Go spells these +// operations MOVW even though x86 disassemblers often print register forms as +// MOVL or MOVQ according to the encoded general-register destination width. +func decodedX86SegmentMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segmentOverride := Reg("") + addressOverride := false + rex := byte(0) + for i < len(code) { + switch code[i] { + case 0x64: + rex = 0 + segmentOverride = FS + i++ + case 0x65: + rex = 0 + segmentOverride = GS + i++ + case 0x66: + rex = 0 + i++ + case 0x67: + rex = 0 + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + } + +opcode: + if i >= len(code) || (code[i] != 0x8c && code[i] != 0x8e) || len(code) < i+2 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRM := code[i+1] + segmentNumber := int(modRM>>3&7) + int(rex>>2&1)*8 + segmentNames := [...]Reg{ES, CS, SS, DS, FS, GS} + if segmentNumber >= len(segmentNames) { + return Instr{}, 0, true, fmt.Errorf("reserved segment-register encoding %d", segmentNumber) + } + bExt := int(rex & 1) + xExt := int(rex>>1) & 1 + other, consumed, decodeErr := decodedX86RMSource(code[i+1:], mode, bExt, xExt, segmentOverride, 1) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + segment := Operand{Kind: OpReg, Reg: segmentNames[segmentNumber]} + args := []Operand{segment, other} + if code[i] == 0x8e { + args[0], args[1] = args[1], args[0] + } + return Instr{ + Op: "MOVW", + Args: args, + Raw: fmt.Sprintf("MOVW %s, %s", args[0].String(), args[1].String()), + }, i + 1 + consumed, true, nil +} + +// decodedX86MaskMoveInstruction recognizes Go 1.27's complete VEX-encoded +// KMOVB/W/D/Q family. x/arch v0.14 does not decode opmask registers and can +// split these bytes into unrelated legacy instructions, so recover all four +// _ykmovb operand rows before consulting the generic decoder. +func decodedX86MaskMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86MaskMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86MaskMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex || code[opcodeIndex] < 0x90 || code[opcodeIndex] > 0x93 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be zero") + } + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111") + } + + opcode := code[opcodeIndex] + pp := vexBits & 3 + width64 := vexLength == 3 && vexBits&0x80 != 0 + width := "" + if opcode == 0x90 || opcode == 0x91 { + switch { + case !width64 && pp == 1: + width = "B" + case !width64 && pp == 0: + width = "W" + case width64 && pp == 1: + width = "D" + case width64 && pp == 0: + width = "Q" + } + } else { + switch { + case !width64 && pp == 1: + width = "B" + case !width64 && pp == 0: + width = "W" + case !width64 && pp == 3: + width = "D" + case width64 && pp == 3: + width = "Q" + } + } + if width == "" { + return Instr{}, 0, true, fmt.Errorf("invalid VEX.W/pp combination") + } + + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + mod := modRM >> 6 + regNumber := int(modRM>>3&7) + rExt*8 + rmNumber := int(modRM&7) + bExt*8 + kReg := func(number int) (Operand, error) { + if number < 0 || number > 7 { + return Operand{}, fmt.Errorf("invalid opmask register K%d", number) + } + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", number))}, nil + } + gpReg := func(number int) (Operand, error) { + reg, valid := decodedX86GeneralRegister(number) + if !valid || mode == 32 && number >= 8 { + return Operand{}, fmt.Errorf("invalid general register %d", number) + } + return Operand{Kind: OpReg, Reg: reg}, nil + } + + var src, dst Operand + consumed := 1 + switch opcode { + case 0x90: // K-or-memory -> K + dst, err = kReg(regNumber) + if err != nil { + return Instr{}, 0, true, err + } + if mod == 3 { + src, err = kReg(rmNumber) + } else { + src, consumed, err = memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + } + case 0x91: // K -> memory + if mod == 3 { + return Instr{}, 0, true, fmt.Errorf("opcode 91 requires a memory destination") + } + src, err = kReg(regNumber) + if err == nil { + dst, consumed, err = memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + } + case 0x92: // GP -> K + if mod != 3 { + return Instr{}, 0, true, fmt.Errorf("opcode 92 requires a general-register source") + } + dst, err = kReg(regNumber) + if err == nil { + src, err = gpReg(rmNumber) + } + case 0x93: // K -> GP + if mod != 3 { + return Instr{}, 0, true, fmt.Errorf("opcode 93 requires a general-register destination") + } + src, err = kReg(rmNumber) + if err == nil { + dst, err = gpReg(regNumber) + } + } + if err != nil { + return Instr{}, 0, true, err + } + op := Op("KMOV" + width) + instruction = Instr{ + Op: op, + Args: []Operand{src, dst}, + Raw: fmt.Sprintf("%s %s, %s", op, src.String(), dst.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86EVEXPackedIntegerMoveInstruction recognizes the complete EVEX +// family behind Go 1.27's shared _yvmovdqa32 table. x/arch v0.14 cannot +// decode these encodings, including the raw VMOVDQU32 loads emitted by +// github.com/minio/sha256-simd. Decode all six opcodes and all twelve table +// rows before consulting the generic decoder. +func decodedX86EVEXPackedIntegerMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedIntegerMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedIntegerMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedMoveInstruction(code, mode, memory, "packed integer move", func(pp byte, width64 bool, opcode byte) (Op, bool, bool) { + store := opcode == 0x7f + if opcode != 0x6f && !store { + return "", false, false + } + switch { + case pp == 1 && !width64: + return "VMOVDQA32", store, true + case pp == 1 && width64: + return "VMOVDQA64", store, true + case pp == 3 && !width64: + return "VMOVDQU8", store, true + case pp == 3 && width64: + return "VMOVDQU16", store, true + case pp == 2 && !width64: + return "VMOVDQU32", store, true + case pp == 2 && width64: + return "VMOVDQU64", store, true + default: + return "", false, false + } + }) +} + +type decodedX86EVEXMoveMatcher func(pp byte, width64 bool, opcode byte) (op Op, store bool, ok bool) + +func decodedX86EVEXPackedMoveInstruction(code []byte, mode int, memory x86RawMemoryDecoder, family string, match decodedX86EVEXMoveMatcher) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 1 || p1&0x04 == 0 || p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, false, nil + } + pp := p1 & 3 + width64 := p1&0x80 != 0 + op, store, matched := match(pp, width64, opcode) + if !matched { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is not valid for %s", family) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + disp8Scale := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if zeroing && store && code[i+5]>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing is invalid for a memory store") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + regNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, regNumber))} + rm, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + args := []Operand{rm, reg} + if store { + args[0], args[1] = reg, rm + } + if maskNumber != 0 { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))} + args = []Operand{args[0], mask, args[1]} + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + instruction = Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))} + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXRMOperand(code []byte, mode, bExt, xExt int, segment Reg, vectorPrefix string, disp8Scale int) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 == 3 { + number := int(code[0]&7) + bExt*8 + xExt*16 + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, number))}, 1, nil + } + return decodedX86RMSource(code, mode, bExt, xExt, segment, disp8Scale) +} + +func x86RawRelativeTarget(inst x86asm.Inst, offset int) (int, bool) { + for _, arg := range inst.Args { + if relative, ok := arg.(x86asm.Rel); ok { + return offset + inst.Len + int(relative), true + } + } + return 0, false +} + +// decodedX86VMOVQInstruction recognizes the complete Go 1.27 _yvmovd and +// _yvmovq families. VMOVD has the X-to-GP/memory and reverse forms; VMOVQ +// additionally has two X-to-X encodings. Recover every VEX and EVEX form +// before consulting the generic decoder, which can split the raw VMOVD bytes +// emitted by Weaviate into a relative branch. Raw AVX512-FP16 VMOVW uses the +// same operand grammar with a two-byte transfer and a distinct opcode map. +func decodedX86VMOVQInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXVMOVDQInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + + var rExt, xExt, bExt, vexLength int + var vexBits byte + switch code[i] { + case 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case 0xc4: + if len(code) < i+4 || code[i+1]&0x1f != 1 { + return Instr{}, 0, false, nil + } + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex { + return Instr{}, 0, false, nil + } + + opcode := code[opcodeIndex] + pp := vexBits & 3 + width64 := vexLength == 3 && vexBits&0x80 != 0 + type vmovqDirection int + const ( + xToGeneral vmovqDirection = iota + xToVector + generalToX + vectorToX + ) + op := Op("") + direction := xToGeneral + switch { + case opcode == 0x7e && pp == 1 && !width64: + op = "VMOVD" + direction = xToGeneral + case opcode == 0x6e && pp == 1 && !width64: + op = "VMOVD" + direction = generalToX + case opcode == 0x7e && pp == 1 && width64: + op = "VMOVQ" + direction = xToGeneral + case opcode == 0xd6 && pp == 1 && !width64: + op = "VMOVQ" + direction = xToVector + case opcode == 0x6e && pp == 1 && width64: + op = "VMOVQ" + direction = generalToX + case opcode == 0x7e && pp == 2 && !width64: + op = "VMOVQ" + direction = vectorToX + default: + return Instr{}, 0, false, nil + } + ok = true + if vexBits&0x7c != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111 and VEX.L must be zero") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + vectorNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || vectorNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", vectorNumber))} + rmVector := direction == xToVector || direction == vectorToX + rm, consumed, decodeErr := decodedX86VEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, rmVector) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + if mode == 32 && width64 && rm.Kind == OpReg { + return Instr{}, 0, true, fmt.Errorf("64-bit general register is unavailable in 32-bit mode") + } + + args := []Operand{vector, rm} + if direction == generalToX || direction == vectorToX { + args[0], args[1] = rm, vector + } + instruction = Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s", op, args[0].String(), args[1].String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXVMOVDQInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + wordMove := p0&0x0f == 5 && p1&3 == 1 && (opcode == 0x6e || opcode == 0x7e) + if p0&0x0f != 1 && !wordMove { + return Instr{}, 0, false, nil + } + if p1&0x04 == 0 || p1&0x78 != 0x78 || p2 != 0x08 { + if wordMove { + return Instr{}, 0, true, fmt.Errorf("VMOVW requires reserved vvvv, fixed 128-bit length, and no mask or broadcast") + } + return Instr{}, 0, false, nil + } + pp := p1 & 3 + width64 := p1&0x80 != 0 + type vmovdqDirection int + const ( + xToGeneral vmovdqDirection = iota + xToVector + generalToX + vectorToX + ) + op := Op("") + direction := xToGeneral + switch { + case wordMove: + op = "VMOVW" + if opcode == 0x6e { + direction = generalToX + } + case opcode == 0x7e && pp == 1 && !width64: + op = "VMOVD" + direction = xToGeneral + case opcode == 0x6e && pp == 1 && !width64: + op = "VMOVD" + direction = generalToX + case opcode == 0x7e && pp == 1 && width64: + op = "VMOVQ" + direction = xToGeneral + case opcode == 0xd6 && pp == 1 && width64: + op = "VMOVQ" + direction = xToVector + case opcode == 0x6e && pp == 1 && width64: + op = "VMOVQ" + direction = generalToX + case opcode == 0x7e && pp == 2 && width64: + op = "VMOVQ" + direction = vectorToX + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + vectorNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + rmVector := direction == xToVector || direction == vectorToX + if mode == 32 && (vectorNumber >= 8 || rmVector && modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + if modRM>>6 == 3 && !rmVector && xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.X cannot extend a general register") + } + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", vectorNumber))} + transferBytes := 4 + if wordMove { + transferBytes = 2 + } else if op == "VMOVQ" { + transferBytes = 8 + } + var rm Operand + var consumed int + var decodeErr error + if rmVector { + rm, consumed, decodeErr = decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X", transferBytes) + } else { + rm, consumed, decodeErr = decodedX86RMSource(code[modRMIndex:], mode, bExt, xExt, segment, transferBytes) + } + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + if mode == 32 && op == "VMOVQ" && rm.Kind == OpReg && !rmVector { + return Instr{}, 0, true, fmt.Errorf("64-bit general register is unavailable in 32-bit mode") + } + args := []Operand{vector, rm} + if direction == generalToX || direction == vectorToX { + args[0], args[1] = rm, vector + } + instruction = Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s", op, args[0].String(), args[1].String()), + x86Encoded: wordMove, + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86VEXRMOperand(code []byte, mode, bExt, xExt int, segment Reg, vectorRegister bool) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 != 3 || !vectorRegister { + return decodedX86VEXRMSource(code, mode, bExt, xExt, segment) + } + number := int(code[0]&7) + bExt*8 + if mode == 32 && number >= 8 { + return Operand{}, 0, fmt.Errorf("extended vector register in 32-bit mode") + } + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", number))}, 1, nil +} + +// decodedX86VEXNonTemporalMoveInstruction recognizes the complete VEX.128 and +// VEX.256 non-temporal vector-move family before x/arch's generic decoder. +// x/arch v0.14 loses the VEX.L register width for VMOVNTDQ/VMOVNTDQA and can +// decode VMOVNTPD/VMOVNTPS as legacy integer subtraction. EVEX forms are not +// handled here because the generic decoder preserves their X/Y/Z widths. +func decodedX86VEXNonTemporalMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXNonTemporalMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXNonTemporalMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + + var rExt, xExt, bExt, vexLength, opcodeMap int + var vexBits byte + switch code[i] { + case 0xc5: + vexLength = 2 + opcodeMap = 1 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case 0xc4: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + vexLength = 3 + opcodeMap = int(code[i+1] & 0x1f) + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex { + return Instr{}, 0, false, nil + } + + opcode := code[opcodeIndex] + prefix := vexBits & 3 + load := false + op := Op("") + switch { + case opcodeMap == 1 && opcode == 0xe7 && prefix == 1: + op = "VMOVNTDQ" + case opcodeMap == 1 && opcode == 0x2b && prefix == 1: + op = "VMOVNTPD" + case opcodeMap == 1 && opcode == 0x2b && prefix == 0: + op = "VMOVNTPS" + case opcodeMap == 2 && opcode == 0x2a && prefix == 1: + op = "VMOVNTDQA" + load = true + default: + return Instr{}, 0, false, nil + } + ok = true + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + if modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("register r/m operand is absent from Go 1.27's %s table", op) + } + vectorNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || vectorNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, vectorNumber))} + operand, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + args := []Operand{vector, operand} + if load { + args[0], args[1] = operand, vector + } + instruction = Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s", op, args[0].String(), args[1].String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86VINSERT128Instruction recognizes the complete VEX +// VINSERTF128/VINSERTI128 family. x/arch v0.14 can misclassify the bytes that +// follow these instructions as PC-relative control flow. +func decodedX86VINSERT128Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VINSERT128InstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VINSERT128InstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 3 { + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{0x18: "VINSERTF128", 0x38: "VINSERTI128"}[code[i+3]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 %s exceeds the Go assembler frontend's operand limit", op) + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vex0, vexBits := code[i+1], code[i+2] + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("%s requires VEX.256", op) + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + firstSource, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86EVEXPackedInsertInstruction recognizes every EVEX row in Go +// 1.27's _yvinsertf32x4 and _yvinsertf32x8 tables. The floating and integer +// spellings share their encodings apart from the opcode, while EVEX.W selects +// 32- or 64-bit lane names. This includes Y/Z output widths, register and +// memory insert sources, and optional merge/zero masks. +func decodedX86EVEXPackedInsertInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedInsertInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedInsertInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if len(code) < i+5 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 3 || p1&3 != 1 { + return Instr{}, 0, false, nil + } + integer := opcode == 0x38 || opcode == 0x3a + wideInsert := opcode == 0x1a || opcode == 0x3a + if opcode != 0x18 && opcode != 0x1a && !integer { + return Instr{}, 0, false, nil + } + ok = true + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed vector inserts are unavailable in 32-bit mode") + } + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for packed vector inserts") + } + vectorBits := p2 >> 5 & 3 + if wideInsert { + if vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("wide packed vector insert requires a Z output") + } + } else if vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("packed vector insert requires a Y or Z output") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + insertBytes := 16 + sourceName := "X" + laneBits := 32 + if p1&0x80 != 0 { + laneBits = 64 + } + if wideInsert { + insertBytes = 32 + sourceName = "Y" + } + stem := "VINSERTF" + if integer { + stem = "VINSERTI" + } + op := Op(fmt.Sprintf("%s%dX%d", stem, laneBits, insertBytes*8/laneBits)) + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + first, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, sourceName, insertBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86VEXPackedBroadcastInstruction recognizes the complete VEX and EVEX +// VPBROADCASTB/W/D/Q and K-data broadcast families represented by Go 1.27's +// _yvpbroadcastb and _yvpbroadcastmb2q tables. +// x/arch v0.14 can misclassify the VEX bytes as PC-relative branches, as +// observed in github.com/vivint/infectious, and cannot decode the EVEX +// general-register form used by github.com/JohanLindvall/haste. +func decodedX86VEXPackedBroadcastInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedBroadcastInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0x78: "VPBROADCASTB", + 0x79: "VPBROADCASTW", + 0x58: "VPBROADCASTD", + 0x59: "VPBROADCASTQ", + }[code[i+3]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vex0, vexBits := code[i+1], code[i+2] + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if int(^vexBits>>3)&15 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXPackedBroadcastInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+5 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f == 2 && p1&3 == 2 && (opcode == 0x2a || opcode == 0x3a) { + return decodedX86EVEXMaskDataBroadcastInstruction(code, mode) + } + if p0&0x0f != 2 || p1&3 != 1 { + return Instr{}, 0, false, nil + } + width64 := p1&0x80 != 0 + vectorSource := false + laneBytes := 0 + op := Op("") + switch opcode { + case 0x7a: + op, laneBytes = "VPBROADCASTB", 1 + case 0x7b: + op, laneBytes = "VPBROADCASTW", 2 + case 0x7c: + if width64 { + op, laneBytes = "VPBROADCASTQ", 8 + } else { + op, laneBytes = "VPBROADCASTD", 4 + } + case 0x78: + op, laneBytes, vectorSource = "VPBROADCASTB", 1, true + case 0x79: + op, laneBytes, vectorSource = "VPBROADCASTW", 2, true + case 0x58: + op, laneBytes, vectorSource = "VPBROADCASTD", 4, true + case 0x59: + op, laneBytes, vectorSource = "VPBROADCASTQ", 8, true + default: + return Instr{}, 0, false, nil + } + ok = true + if (!vectorSource || opcode != 0x59) && opcode != 0x7c && width64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be zero for %s", op) + } + if vectorSource && opcode == 0x59 && !width64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be one for %s", op) + } + if p1&0x7c != 0x7c || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for packed scalar broadcasts") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + + var source Operand + consumed := 0 + if vectorSource { + if mode == 32 && modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, err = decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X", laneBytes) + } else { + if modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX opcode %#x requires a general-register source", opcode) + } + if xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.X cannot extend a general register") + } + if mode == 32 && opcode == 0x7c && width64 { + // GP-only opcode 7C ignores EVEX.W in non-64-bit mode. + // Retain the original Q/W1 identity, but decode its physical + // r32 input; the shared broadcast grammar uses dword lanes. + laneBytes = 4 + } + source, consumed, err = decodedX86RMSource(code[modRMIndex:], mode, bExt, 0, segment, laneBytes) + } + if err != nil { + return Instr{}, 0, true, err + } + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86PackedAlignRightInstruction recognizes every raw PALIGNR and +// VPALIGNR encoding represented by Go 1.27's ypalignr and +// _yvgf2p8affineinvqb tables: legacy X, VEX X/Y, and EVEX X/Y/Z with optional +// merge/zero masks. x/arch v0.14 rejects the VEX bytes used by the historical +// sha256-simd copy in github.com/ipfs/fs-repo-migrations. +func decodedX86PackedAlignRightInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + hasDataSizePrefix := false + rex := byte(0) + hasREX := false + for i < len(code) { + switch code[i] { + case 0x64: + rex = 0 + hasREX = false + segment = FS + i++ + case 0x65: + rex = 0 + hasREX = false + segment = GS + i++ + case 0x66: + rex = 0 + hasREX = false + hasDataSizePrefix = true + i++ + case 0x67: + rex = 0 + hasREX = false + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + hasREX = true + i++ + continue + } + goto opcode + } + } + +opcode: + if i >= len(code) { + return Instr{}, 0, false, nil + } + if code[i] == 0xc4 { + if hasDataSizePrefix || hasREX || len(code) < i+5 || code[i+1]&0x1f != 3 || code[i+3] != 0x0f { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 VPALIGNR exceeds the Go assembler frontend's operand limit") + } + vex0, vexBits := code[i+1], code[i+2] + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immediateIndex])}, + firstSource, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))}, + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: "VPALIGNR", Args: args, Raw: "VPALIGNR " + strings.Join(rawArgs, ", ")}, immediateIndex + 1, true, nil + } + if code[i] == 0x62 { + if hasDataSizePrefix || hasREX || len(code) < i+6 || code[i+1]&0x0f != 3 || code[i+2]&0x07 != 5 || code[i+4] != 0x0f { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 VPALIGNR exceeds the Go assembler frontend's operand limit") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be zero") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is absent from Go 1.27's VPALIGNR table") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, vectorWidth) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + op := Op("VPALIGNR") + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immediateIndex])}, + firstSource, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))}, + } + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))}) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil + } + + if !hasDataSizePrefix || len(code) < i+5 || code[i] != 0x0f || code[i+1] != 0x3a || code[i+2] != 0x0f { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if hasREX && rex&0x08 != 0 { + return Instr{}, 0, true, fmt.Errorf("REX.W is absent from Go 1.27's PALIGNR encoding") + } + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex & 1) + modRMIndex := i + 3 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (destinationNumber >= 8 || bExt != 0 || xExt != 0) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + instruction = Instr{ + Op: "PALIGNR", + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("PALIGNR %s, %s, %s", immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86Permute128Instruction recognizes every raw VPERM2F128 and +// VPERM2I128 encoding represented by Go 1.27's _yvperm2f128 table: an +// immediate, Y/m256 first source, Y second source, and Y destination. +func decodedX86Permute128Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + hasLegacyPrefix := false + hasREX := false + for i < len(code) { + switch code[i] { + case 0x64: + hasREX = false + segment = FS + i++ + case 0x65: + hasREX = false + segment = GS + i++ + case 0x66: + hasREX = false + hasLegacyPrefix = true + i++ + case 0x67: + hasREX = false + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + hasREX = true + i++ + continue + } + goto opcode + } + } + +opcode: + if i >= len(code) || code[i] != 0xc4 || hasLegacyPrefix || hasREX || len(code) < i+5 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + if vex0&0x1f != 3 || (opcode != 0x06 && opcode != 0x46) { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 VPERM2F128/VPERM2I128 exceeds the Go assembler frontend's operand limit") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must select 256-bit vectors") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "Y") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + op := Op("VPERM2F128") + if opcode == 0x46 { + op = "VPERM2I128" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immediateIndex])}, + firstSource, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", secondSourceNumber))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))}, + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86InsertPSInstruction recognizes all encodings represented by Go +// 1.27's yxshuf and _yvinsertps tables: legacy INSERTPS plus VEX and EVEX +// VINSERTPS, each with an X-register or ModRM/SIB memory source. x/arch v0.14 +// can consume one byte past VINSERTPS and desynchronize the rest of a raw +// directive group, as observed in github.com/racerxdl/segdsp. +func decodedX86InsertPSInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto encoding + } + } + +encoding: + if len(code) <= i { + return Instr{}, 0, false, nil + } + + if code[i] == 0x66 { + j := i + 1 + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) < j+4 || code[j] != 0x0f || code[j+1] != 0x3a || code[j+2] != 0x21 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + modRMIndex := j + 3 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: "INSERTPS", + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("INSERTPS %s, %s, %s", immediate.String(), source.String(), destination.String()), + }, immediateIndex + 1, true, nil + } + + if code[i] == 0xc4 { + if len(code) < i+5 || code[i+1]&0x1f != 3 || code[i+3] != 0x21 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1 := code[i+1], code[i+2] + // The physical VEX form is WIG, independently of Go's W0 encoder. + if p1&0x07 != 0x01 { + return Instr{}, 0, true, fmt.Errorf("VINSERTPS requires VEX.L0.66") + } + modRMIndex := i + 4 + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + baseNumber := int(^p1>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (baseNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VINSERTPS register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + base := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", baseNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: "VINSERTPS", + Args: []Operand{immediate, source, base, destination}, + Raw: fmt.Sprintf("VINSERTPS %s, %s, %s, %s", immediate.String(), source.String(), base.String(), destination.String()), + x86Encoded: true, + }, immediateIndex + 1, true, nil + } + + if code[i] != 0x62 || len(code) < i+6 || code[i+1]&0x0f != 3 || code[i+4] != 0x21 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("EVEX VINSERTPS is unavailable in 32-bit mode") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x87 != 0x05 || p2&0xf7 != 0 { + return Instr{}, 0, true, fmt.Errorf("VINSERTPS requires EVEX.W0.L0.66 without mask, zeroing, or broadcast") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + baseNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X", 4) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + base := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", baseNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: "VINSERTPS", + Args: []Operand{immediate, source, base, destination}, + Raw: fmt.Sprintf("VINSERTPS %s, %s, %s, %s", immediate.String(), source.String(), base.String(), destination.String()), + }, immediateIndex + 1, true, nil +} + +// decodedX86PackedScalarExtractInstruction recognizes every VEX and EVEX +// encoding row shared by Go 1.27's _yvextractps and _yvpextrw tables: +// VEXTRACTPS and VPEXTRB/W/D/Q with GP-register or ModRM/SIB memory +// destinations. x/arch v0.14 does not decode VEXTRACTPS and can interpret +// bytes after other scalar extracts as instructions or PC-relative operands, +// so recover the complete family before consulting the generic decoder. +func decodedX86PackedScalarExtractInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto prefix + } + } + +prefix: + if len(code) <= i { + return Instr{}, 0, false, nil + } + var ( + mapNumber byte + opcode byte + width64 bool + rExt int + rHighExt int + xExt int + bExt int + modRMIndex int + disp8Scale = 1 + evex bool + twoByteVEX bool + prefixValidated bool + ) + switch code[i] { + case 0xc4: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + mapNumber = code[i+1] & 0x1f + opcode = code[i+3] + width64 = code[i+2]&0x80 != 0 + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + modRMIndex = i + 4 + prefixValidated = code[i+2]&0x7f == 0x79 + case 0xc5: + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + mapNumber = 1 + opcode = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + modRMIndex = i + 3 + twoByteVEX = true + prefixValidated = code[i+1]&0x7f == 0x79 + case 0x62: + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + evex = true + mapNumber = code[i+1] & 0x0f + opcode = code[i+4] + width64 = code[i+2]&0x80 != 0 + rExt = int(^code[i+1]>>7) & 1 + rHighExt = int(^code[i+1]>>4) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + modRMIndex = i + 5 + prefixValidated = code[i+2]&0x7f == 0x7d && code[i+3] == 0x08 + default: + return Instr{}, 0, false, nil + } + + var op Op + laneBytes := 0 + expectedWidth64 := false + switch { + case mapNumber == 3 && opcode == 0x14: + op, laneBytes = "VPEXTRB", 1 + case mapNumber == 3 && opcode == 0x15: + op, laneBytes = "VPEXTRW", 2 + case mapNumber == 3 && opcode == 0x16 && !width64: + op, laneBytes = "VPEXTRD", 4 + case mapNumber == 3 && opcode == 0x16 && width64: + op, laneBytes, expectedWidth64 = "VPEXTRQ", 8, true + case mapNumber == 3 && opcode == 0x17: + op, laneBytes = "VEXTRACTPS", 4 + case mapNumber == 1 && opcode == 0xc5: + op, laneBytes = "VPEXTRW", 2 + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if !prefixValidated { + return Instr{}, 0, true, fmt.Errorf("reserved vvvv, vector length, prefix, or EVEX mask bits are invalid") + } + if width64 != expectedWidth64 { + return Instr{}, 0, true, fmt.Errorf("invalid W bit for %s", op) + } + if evex { + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed scalar extracts are unavailable in 32-bit mode") + } + disp8Scale = laneBytes + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + sourceNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && sourceNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended source register in 32-bit mode") + } + destination, consumed, decodeErr := decodedX86RMSource(code[modRMIndex:], mode, bExt, xExt, segment, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + if mapNumber == 1 && destination.Kind != OpReg { + return Instr{}, 0, true, fmt.Errorf("VEX map-1 VPEXTRW requires a GP-register destination") + } + if evex && destination.Kind == OpReg && xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed scalar extract destination is outside the GP-register class") + } + if twoByteVEX && destination.Kind == OpReg && bExt != 0 { + return Instr{}, 0, true, fmt.Errorf("two-byte VEX cannot encode an extended destination register") + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", sourceNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86VPERMQInstruction recognizes the complete VEX.256 immediate +// VPERMQ Y/m256, Y family. x/arch v0.14 can misclassify its trailing immediate +// as a PC-relative operand, so recover register and ModRM/SIB memory forms +// before consulting the generic decoder. EVEX forms are decoded correctly by +// x/arch and continue through the ordinary path. +func decodedX86VPERMQInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 3 || code[i+3] != 0x00 { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if code[i+2] != 0xfd { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be one, VEX.vvvv must be 1111, VEX.L must be one, and VEX.pp must be 66") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + vex1 := code[i+1] + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + source, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "Y") + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))} + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + instruction = Instr{ + Op: "VPERMQ", + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("VPERMQ %s, %s, %s", immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86ScalarPrecisionConvertInstruction recognizes the complete +// CVTSS2SD/CVTSD2SS and VCVTSS2SD/VCVTSD2SS family represented by Go 1.27's +// legacy yxm and EVEX-aware _yvaddsd tables. Besides VEX and legacy forms it +// handles EVEX high X registers, memory, masks, zeroing, SAE, and all four +// embedded rounding modes. x/arch v0.14 can split VCVTSD2SS into POP/ROLL. +func decodedX86ScalarPrecisionConvertInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto encoding + } + } + +encoding: + if len(code) <= i { + return Instr{}, 0, false, nil + } + + // Legacy F2/F3 0F 5A /r forms overwrite their X destination. + j := i + if code[j] == 0xf2 || code[j] == 0xf3 { + prefix := code[j] + j++ + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+3 && code[j] == 0x0f && code[j+1] == 0x5a { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + op := Op("CVTSD2SS") + if prefix == 0xf3 { + op = "CVTSS2SD" + } + modRMIndex := j + 2 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{Op: op, Args: []Operand{source, destination}, Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String())}, modRMIndex + consumed, true, nil + } + } + + if code[i] == 0xc4 || code[i] == 0xc5 { + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch code[i] { + case 0xc5: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case 0xc4: + if len(code) < i+5 || code[i+1]&0x1f != 1 { + return Instr{}, 0, false, nil + } + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + } + if opcode != 0x5a || (vexBits&3 != 2 && vexBits&3 != 3) { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar precision conversion requires VEX.128") + } + op := Op("VCVTSD2SS") + if vexBits&3 == 2 { + op = "VCVTSS2SD" + } + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + modRM := code[modRMIndex] + first, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{Op: op, Args: []Operand{first, second, destination}, Raw: fmt.Sprintf("%s %s, %s, %s", op, first.String(), second.String(), destination.String())}, modRMIndex + consumed, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+4] != 0x5a { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + prefix := p1 & 3 + wBit := p1&0x80 != 0 + op := Op("") + inputBytes := 0 + supportsRounding := false + switch prefix { + case 3: + op = "VCVTSD2SS" + inputBytes = 8 + supportsRounding = true + if !wBit { + return Instr{}, 0, true, fmt.Errorf("VCVTSD2SS requires EVEX.W1") + } + case 2: + op = "VCVTSS2SD" + inputBytes = 4 + if wBit { + return Instr{}, 0, true, fmt.Errorf("VCVTSS2SD requires EVEX.W0") + } + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + embeddedControl := p2&0x10 != 0 + vectorBits := p2 >> 5 & 3 + if !embeddedControl && vectorBits != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar precision conversion requires EVEX.128") + } + if embeddedControl && modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("embedded rounding/SAE requires a register source") + } + if embeddedControl { + if supportsRounding { + op += Op("." + [...]string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"}[vectorBits]) + } else if vectorBits == 0 { + op += ".SAE" + } else { + return Instr{}, 0, true, fmt.Errorf("VCVTSS2SD supports SAE but not explicit rounding") + } + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 scalar precision conversion mask forms exceed the Go assembler frontend's operand limit") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + first, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X", inputBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86DuplicateMoveInstruction recognizes every Go 1.27 yxm and +// _yvmovddup encoding for MOVDDUP/MOVSHDUP/MOVSLDUP and their V-prefixed +// forms. This includes legacy X, VEX X/Y, and EVEX X/Y/Z register or memory +// sources with optional merge/zero masks and compressed displacements. +// x/arch v0.14 rejects the VMOVSHDUP bytes generated by segdsp. +func decodedX86DuplicateMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto encoding + } + } + +encoding: + if len(code) <= i { + return Instr{}, 0, false, nil + } + + // Legacy duplicate moves use F2/F3 0F opcode /r and the ModRM.reg X + // register as the destination. + j := i + if code[j] == 0xf2 || code[j] == 0xf3 { + prefix := code[j] + j++ + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+3 && code[j] == 0x0f { + op := Op("") + switch { + case prefix == 0xf2 && code[j+1] == 0x12: + op = "MOVDDUP" + case prefix == 0xf3 && code[j+1] == 0x12: + op = "MOVSLDUP" + case prefix == 0xf3 && code[j+1] == 0x16: + op = "MOVSHDUP" + } + if op != "" { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + modRMIndex := j + 2 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{Op: op, Args: []Operand{source, destination}, Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String())}, modRMIndex + consumed, true, nil + } + } + } + + if code[i] == 0xc4 || code[i] == 0xc5 { + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch code[i] { + case 0xc5: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case 0xc4: + if len(code) < i+5 || code[i+1]&0x1f != 1 { + return Instr{}, 0, false, nil + } + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + } + prefix := vexBits & 3 + op := Op("") + switch { + case prefix == 3 && opcode == 0x12: + op = "VMOVDDUP" + case prefix == 2 && opcode == 0x12: + op = "VMOVSLDUP" + case prefix == 2 && opcode == 0x16: + op = "VMOVSHDUP" + } + if op == "" { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{Op: op, Args: []Operand{source, destination}, Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String())}, modRMIndex + consumed, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 1 { + return Instr{}, 0, false, nil + } + p0, p1, p2, opcode := code[i+1], code[i+2], code[i+3], code[i+4] + prefix := p1 & 3 + op := Op("") + elementBytes := 4 + switch { + case prefix == 3 && opcode == 0x12: + op = "VMOVDDUP" + elementBytes = 8 + case prefix == 2 && opcode == 0x12: + op = "VMOVSLDUP" + case prefix == 2 && opcode == 0x16: + op = "VMOVSHDUP" + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + double := op == "VMOVDDUP" + if double != (p1&0x80 != 0) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match duplicate-move element width") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvvv must be reserved") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is absent from Go 1.27's duplicate-move table") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + disp8Scale := vectorWidth + if double && vectorBits == 0 { + disp8Scale = elementBytes + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + sourceNumber := int(modRM&7) + bExt*8 + xExt*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorName == "Z" { + if modRM>>6 == 3 && sourceNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86InLaneFloatingPermuteInstruction recognizes every encoding in Go +// 1.27's shared _yvpermilpd table: VPERMILPS/VPERMILPD immediate and +// variable-control VEX X/Y forms plus EVEX X/Y/Z, memory, scalar broadcast, +// merge-mask, and zero-mask forms. x/arch v0.14 can split a VEX immediate form +// into a fake PC-relative branch inside generated segdsp byte streams. +func decodedX86InLaneFloatingPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto encoding + } + } + +encoding: + if len(code) <= i { + return Instr{}, 0, false, nil + } + if code[i] == 0xc4 { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + mapNumber := vex0 & 0x1f + immediate := mapNumber == 3 && (opcode == 0x04 || opcode == 0x05) + variable := mapNumber == 2 && (opcode == 0x0c || opcode == 0x0d) + if !immediate && !variable { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if immediate && vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("immediate VPERMIL requires reserved VEX.vvvv") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + secondNumber := int(^vexBits>>3) & 15 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8 || variable && secondNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + first, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + double := opcode&1 != 0 + op := Op("VPERMILPS") + if double { + op = "VPERMILPD" + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := make([]Operand, 0, 3) + end := modRMIndex + consumed + if immediate { + if len(code) <= end { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[end])}, first, destination) + end++ + } else { + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + args = append(args, first, second, destination) + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2, opcode := code[i+1], code[i+2], code[i+3], code[i+4] + mapNumber := p0 & 0x0f + immediate := mapNumber == 3 && (opcode == 0x04 || opcode == 0x05) + variable := mapNumber == 2 && (opcode == 0x0c || opcode == 0x0d) + if !immediate && !variable { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p1&0x07 != 5 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit and pp must select 66") + } + double := opcode&1 != 0 + if double != (p1&0x80 != 0) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match VPERMILPS/PD") + } + if immediate && (p1&0x78 != 0x78 || p2&0x08 == 0) { + return Instr{}, 0, true, fmt.Errorf("immediate VPERMIL requires reserved EVEX.vvvvv") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + op := Op("VPERMILPS") + if double { + laneBytes = 8 + op = "VPERMILPD" + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 VPERMIL mask forms exceed the Go assembler frontend's operand limit") + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + op += ".BCST" + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + firstNumber := int(modRM&7) + bExt*8 + xExt*16 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorName == "Z" { + if modRM>>6 == 3 && firstNumber >= 8 || variable && secondNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + } + first, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := make([]Operand, 0, 4) + end := modRMIndex + consumed + if immediate { + if len(code) <= end { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[end])}, first) + end++ + } else { + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + args = append(args, first, second) + } + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil +} + +// decodedX86ImmediatePackedBlendInstruction recognizes the complete Go 1.27 +// immediate packed-blend family: legacy PBLENDW/BLENDPS/BLENDPD X forms and +// VEX VPBLENDW/VPBLENDD/VBLENDPS/VBLENDPD X/Y forms. Every register and +// ModRM/SIB memory source is handled before x/arch's generic decoder, which +// can split VBLENDPD into unrelated scalar instructions in generated code. +func decodedX86ImmediatePackedBlendInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto encoding + } + } + +encoding: + if len(code) <= i { + return Instr{}, 0, false, nil + } + + // Legacy 66 0F 3A /r ib encodings use the ModRM.reg X register as both + // their second source and destination. + j := i + if code[j] == 0x66 { + j++ + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+4 && code[j] == 0x0f && code[j+1] == 0x3a { + op, recognized := map[byte]Op{0x0c: "BLENDPS", 0x0d: "BLENDPD", 0x0e: "PBLENDW"}[code[j+2]] + if recognized { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + modRMIndex := j + 3 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + }, immediateIndex + 1, true, nil + } + } + } + + // All vector forms are VEX.128/256.66.0F3A.W0. Go's 386 frontend + // cannot represent their four Plan 9 operands and rejects them. + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 3 { + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0x02: "VPBLENDD", + 0x0c: "VBLENDPS", + 0x0d: "VBLENDPD", + 0x0e: "VPBLENDW", + }[code[i+3]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 %s exceeds the Go assembler frontend's operand limit", op) + } + vex0, vexBits := code[i+1], code[i+2] + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + first, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: op, + Args: []Operand{immediate, first, second, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), first.String(), second.String(), destination.String()), + }, immediateIndex + 1, true, nil +} + +// decodedX86FloatingUnpackInstruction recognizes every legacy yxm and +// VEX/EVEX _yvandnpd row used by UNPCKLPS/UNPCKHPS/UNPCKLPD/UNPCKHPD and +// their V-prefixed forms. This includes X/Y/Z widths, register and ModRM/SIB +// memory sources, EVEX broadcast, merge masks, zero masks, and compressed +// displacements. x/arch v0.14 can overrun VUNPCK by one byte and desynchronize +// a raw directive stream, as observed in github.com/racerxdl/segdsp. +func decodedX86FloatingUnpackInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto encoding + } + } + +encoding: + if len(code) <= i { + return Instr{}, 0, false, nil + } + + legacyPrefix := byte(0) + j := i + if code[j] == 0x66 { + legacyPrefix = 1 + j++ + } + rex := byte(0) + if j < len(code) && code[j]&0xf0 == 0x40 { + rex = code[j] + j++ + } + if len(code) >= j+3 && code[j] == 0x0f && (code[j+1] == 0x14 || code[j+1] == 0x15) { + opcode := code[j+1] + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 && rex != 0 { + return Instr{}, 0, true, fmt.Errorf("REX prefix in 32-bit mode") + } + stem := "UNPCKL" + if opcode == 0x15 { + stem = "UNPCKH" + } + op := Op(stem + map[byte]string{0: "PS", 1: "PD"}[legacyPrefix]) + modRMIndex := j + 2 + modRM := code[modRMIndex] + rExt := int(rex>>2) & 1 + xExt := int(rex>>1) & 1 + bExt := int(rex) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + return Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + } + if vexLength != 0 && (opcode == 0x14 || opcode == 0x15) { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 > 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select PS or PD floating unpack") + } + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("floating unpack requires VEX.W0") + } + stem := "VUNPCKL" + if opcode == 0x15 { + stem = "VUNPCKH" + } + op := Op(stem + map[byte]string{0: "PS", 1: "PD"}[vexBits&3]) + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + secondNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended VEX register in 32-bit mode") + } + first, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: op, + Args: []Operand{first, second, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, first.String(), second.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + + if code[i] != 0x62 || len(code) < i+6 || code[i+1]&0x0f != 1 || (code[i+4] != 0x14 && code[i+4] != 0x15) { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + if p1&3 > 1 { + return Instr{}, 0, true, fmt.Errorf("EVEX.pp does not select PS or PD floating unpack") + } + double := p1&3 == 1 + if double != (p1&0x80 != 0) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W and pp disagree for floating unpack") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorBytes := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + suffix := "PS" + if double { + laneBytes = 8 + suffix = "PD" + } + stem := "VUNPCKL" + if code[i+4] == 0x15 { + stem = "VUNPCKH" + } + op := Op(stem + suffix) + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX masking is unavailable for 386 floating unpack forms") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorBytes + if broadcast { + disp8Scale = laneBytes + op += ".BCST" + } + evexRExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + evexXExt := int(^p0>>6) & 1 + evexBExt := int(^p0>>5) & 1 + firstNumber := int(modRM&7) + evexBExt*8 + evexXExt*16 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + evexRExt*8 + rHighExt*16 + if mode == 32 { + if modRM>>6 == 3 && firstNumber >= 8 || secondNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("%w: EVEX floating unpack high register has no proved physical mode32 binding", ErrProbeNeedsContext) + } + } + first, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, evexBExt, evexXExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86VEXPackedFloatLogicalInstruction recognizes the complete +// VEX.128/VEX.256 VANDPS/PD, VANDNPS/PD, VORPS/PD, and VXORPS/PD family. +// x/arch v0.14 can split or reject valid VEX encodings, including the raw +// VXORPS emitted by Weaviate, so recover every register and ModRM/SIB memory +// form before the generic decoder. VEX.W is ignored by these opcode rows. +func decodedX86VEXPackedFloatLogicalInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXPackedFloatLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXPackedFloatLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{ + 0x54: "VAND", + 0x55: "VANDN", + 0x56: "VOR", + 0x57: "VXOR", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + suffix := "" + switch vexBits & 3 { + case 0: + suffix = "PS" + case 1: + suffix = "PD" + default: + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select a packed floating logical instruction") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := Op(stem + suffix) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// K is a data register here, not an optional writemask. Recognizing these +// physical register shapes grants no initialized-mask or memory authority. +func decodedX86EVEXMaskDataBroadcastInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 2 || p.opcode != 0x2a && p.opcode != 0x3a { + return Instr{}, 0, false, nil + } + op := Op("VPBROADCASTMW2D") + if p.opcode == 0x2a { + op = "VPBROADCASTMB2Q" + } + fail := func(reason string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s %s", op, reason) + } + if mode != 32 && mode != 64 { + return fail("has an unsupported x86 mode") + } + if p.w != (p.opcode == 0x2a) { + return fail("has a reserved EVEX.W value") + } + if !p.fixed || p.upper != 0 { + return fail("requires reserved EVEX.vvvv and the fixed prefix bit") + } + if p.segment != "" || p.addressOverride { + return fail("does not admit segment or address-size overrides") + } + if p.mask != 0 || p.zero || p.broadcast { + return fail("does not admit writemasking, zeroing, or EVEX.b") + } + if p.vectorLength == 3 { + return fail("has a reserved EVEX vector length") + } + if len(code) <= p.modRM { + return fail("is missing its ModRM byte") + } + modRM := code[p.modRM] + if modRM>>6 != 3 || p.b != 0 || p.x != 0 { + return fail("requires an unextended K-register source") + } + destination := int(modRM>>3&7) + p.r*8 + if mode == 32 && destination >= 8 { + return fail("has an extended vector register in 32-bit mode") + } + src := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM&7))} + dst := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", [...]string{"X", "Y", "Z"}[p.vectorLength], destination))} + return Instr{Op: op, Args: []Operand{src, dst}, Raw: fmt.Sprintf("%s %s, %s", op, src.String(), dst.String())}, p.modRM + 1, true, nil +} + +// decodedX86VEXPackedIntegerLogicalInstruction recognizes the complete +// VEX.128/VEX.256 VPAND, VPANDN, VPOR, and VPXOR family described by Go +// 1.27's shared _yvaddsubpd operand table. Recover both VEX.WIG values and +// every register and ModRM/SIB memory form before the generic decoder, which +// can split or reject raw encodings emitted by ecosystem assembly. +func decodedX86VEXPackedIntegerLogicalInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXPackedIntegerLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXPackedIntegerLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0xdb: "VPAND", + 0xdf: "VPANDN", + 0xeb: "VPOR", + 0xef: "VPXOR", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86EVEXPackedLogicalInstruction recognizes every EVEX register, +// memory, broadcast, merge-mask, and zero-mask row shared by Go 1.27's +// VPANDD/Q, VPANDND/Q, VPORD/Q, and VPXORD/Q _yvblendmpd table. +func decodedX86EVEXPackedLogicalInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXPackedLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXPackedLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if len(code) < i+5 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || p1&3 != 1 { + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{0xdb: "VPAND", 0xdf: "VPANDN", 0xeb: "VPOR", 0xef: "VPXOR"}[code[i+4]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX fixed bit must be one") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := p2 >> 5 & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorBytes := [...]int{16, 32, 64}[vectorBits] + width := "D" + laneBytes := 4 + if p1&0x80 != 0 { + width = "Q" + laneBytes = 8 + } + op := Op(stem + width) + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX masking is unavailable for 386 packed logical forms") + } + + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorBytes + if broadcast { + disp8Scale = laneBytes + op += ".BCST" + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + firstNumber := int(modRM&7) + bExt*8 + xExt*16 + secondNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorName == "Z" { + if modRM>>6 == 3 && firstNumber >= 8 || secondNumber >= 8 || destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + } + first, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{first, second} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86VEXPackedMADDInstruction recognizes the complete VEX.128 and +// VEX.256 VPMADDWD and VPMADDUBSW family in Go 1.27's opcode tables. +// VPMADDWD is available through both two- and three-byte VEX map-1 forms; +// VPMADDUBSW uses the three-byte VEX map-2 form. VEX.W is ignored. Recover +// every register and ModRM/SIB memory form before the generic decoder, which +// can split raw VPMADDWD bytes emitted by ecosystem assembly. +func decodedX86VEXPackedMADDInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, opcodeMap, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + opcodeMap = 1 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4: + vexLength = 3 + opcodeMap = int(code[i+1] & 0x1f) + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + op, recognized := map[[2]int]Op{ + {1, 0xf5}: "VPMADDWD", + {2, 0x04}: "VPMADDUBSW", + }[[2]int{opcodeMap, int(opcode)}] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + x86Encoded: true, + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86VEXBinaryFloatInstruction recognizes the complete VEX and EVEX +// VADD, VMUL, VSUB, VMIN, VDIV, and VMAX family for PS/PD/SS/SD, plus +// VSQRTSS/VSQRTSD, plus raw-only packed/scalar FP16 MAP5 binary forms. Packed SQRT +// has a distinct single-source grammar and is decoded by +// decodedX86SameWidthConversionInstruction. Scalar VEX forms require VEX.128; +// packed VEX forms accept both vector widths. EVEX adds X/Y/Z widths, masking, +// zeroing, packed broadcasts, rounding, and SAE. +// Recover every register and ModRM/SIB memory form before the generic decoder. +func decodedX86VEXBinaryFloatInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXBinaryFloatInstruction(code, i, mode, segment, addressOverride) + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{ + 0x51: "VSQRT", + 0x58: "VADD", + 0x59: "VMUL", + 0x5c: "VSUB", + 0x5d: "VMIN", + 0x5e: "VDIV", + 0x5f: "VMAX", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + pp := vexBits & 3 + if opcode == 0x51 && pp < 2 { + return Instr{}, 0, false, nil + } + suffix := [...]string{"PS", "PD", "SS", "SD"}[pp] + scalar := pp >= 2 + if scalar && vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar binary floating instruction requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := Op(stem + suffix) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXBinaryFloatInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + stem, recognized := map[byte]string{ + 0x51: "VSQRT", + 0x58: "VADD", + 0x59: "VMUL", + 0x5c: "VSUB", + 0x5d: "VMIN", + 0x5e: "VDIV", + 0x5f: "VMAX", + }[opcode] + mapNumber := p0 & 0x0f + rawHalf := mapNumber == 5 && (p1&3 == 0 || p1&3 == 2) + rawHalfScalar := rawHalf && p1&3 == 2 + if !recognized || mapNumber != 1 && !rawHalf { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + pp := p1 & 3 + if opcode == 0x51 && pp < 2 { + return Instr{}, 0, false, nil + } + width64 := p1&0x80 != 0 + wantWidth64 := pp == 1 || pp == 3 + if rawHalf { + wantWidth64 = false + } + if width64 != wantWidth64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match floating element type") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + scalar := pp >= 2 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + embeddedControl := evexB && registerSource + broadcast := evexB && !registerSource + if broadcast && scalar { + return Instr{}, 0, true, fmt.Errorf("scalar binary floating instruction does not support broadcast") + } + if !embeddedControl && vectorBits == 3 && !rawHalfScalar { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "X" + vectorWidth := 16 + switch { + case embeddedControl && !scalar: + vectorPrefix = "Z" + vectorWidth = 64 + case !scalar: + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + laneBytes := 4 + if rawHalf { + laneBytes = 2 + } else if pp == 1 || pp == 3 { + laneBytes = 8 + } + disp8Scale := vectorWidth + if scalar || broadcast { + disp8Scale = laneBytes + } + + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (!rawHalf && maskNumber != 0 || rExt != 0 || rHighExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register or mask in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + suffix := [...]string{"PS", "PD", "SS", "SD"}[pp] + if rawHalf { + suffix = "PH" + if rawHalfScalar { + suffix = "SH" + } + } + op := Op(stem + suffix) + if broadcast { + op += ".BCST" + } + if embeddedControl { + if stem == "VMIN" || stem == "VMAX" { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + } + if zeroing { + op += ".Z" + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{ + Op: op, Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: rawHalf, + }, modRMIndex + consumed, true, nil +} + +// decodedX86ScalarIntegerFloatInstruction recognizes Go 1.27's complete +// signed and unsigned scalar integer-to-floating-point family. VEX supports +// the four signed forms; EVEX additionally supports unsigned forms, ignored +// scalar L'L encodings, and register-only embedded rounding where enabled. +func decodedX86ScalarIntegerFloatInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto prefix + } + } + +prefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXScalarIntegerFloatInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case code[i] == 0xc4 && len(code) >= i+5 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex || code[opcodeIndex] != 0x2a { + return Instr{}, 0, false, nil + } + ok = true + pp := vexBits & 3 + if pp != 2 && pp != 3 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be F3 or F2") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar integer-to-float instruction requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + passthroughNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || passthroughNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86RMSource(code[modRMIndex:], mode, bExt, xExt, segment, 1) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + width64 := vexLength == 3 && vexBits&0x80 != 0 + op := scalarIntegerFloatRawOperation(false, pp, width64) + passthrough := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", passthroughNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{source, passthrough, destination} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s, %s, %s", op, source.String(), passthrough.String(), destination.String())}, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXScalarIntegerFloatInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 1 || opcode != 0x2a && opcode != 0x7b { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + pp := p1 & 3 + if pp != 2 && pp != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX.pp must be F3 or F2") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x87 != 0 { + return Instr{}, 0, true, fmt.Errorf("masking and zeroing are absent from the scalar integer-to-float table") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + embeddedRounding := p2&0x10 != 0 + if !embeddedRounding && vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + if embeddedRounding && modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("embedded rounding requires a register source") + } + width64 := p1&0x80 != 0 + roundingEnabled := pp == 2 || width64 + if embeddedRounding && !roundingEnabled { + return Instr{}, 0, true, fmt.Errorf("explicit rounding is absent from VCVTSI2SDL/VCVTUSI2SDL") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + passthroughNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if modRM>>6 == 3 && xExt != 0 { + return Instr{}, 0, true, fmt.Errorf("general register source cannot use the EVEX high vector extension") + } + if mode == 32 && (rExt != 0 || rHighExt != 0 || xExt != 0 || bExt != 0 || passthroughNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + sourceBytes := 4 + if width64 && (mode == 64 || opcode == 0x7b) { + // Signed VCVTSI2SS/SD ignore W1 in mode32, including the + // compressed-displacement tuple. Keep the separate unsigned + // family unchanged until its own complete physical proof. + sourceBytes = 8 + } + source, consumed, decodeErr := decodedX86RMSource(code[modRMIndex:], mode, bExt, xExt, segment, sourceBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + op := scalarIntegerFloatRawOperation(opcode == 0x7b, pp, width64) + if embeddedRounding { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + passthrough := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", passthroughNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{source, passthrough, destination} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s, %s, %s", op, source.String(), passthrough.String(), destination.String())}, modRMIndex + consumed, true, nil +} + +func scalarIntegerFloatRawOperation(unsigned bool, pp byte, width64 bool) Op { + prefix := "VCVTSI2" + if unsigned { + prefix = "VCVTUSI2" + } + result := "SS" + if pp == 3 { + result = "SD" + } + source := "L" + if width64 { + source = "Q" + } + return Op(prefix + result + source) +} + +// decodedX86VectorFloatCompareInstruction recognizes the complete VEX and +// EVEX VCMPPD/PS/SD/SS family. VEX writes X/Y vector results. EVEX writes K +// results and adds an optional K write mask, packed broadcast, and SAE. +func decodedX86VectorFloatCompareInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto prefix + } + } + +prefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXVectorFloatCompareInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case code[i] == 0xc4 && len(code) >= i+6 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + opcodeIndex := i + vexLength + if len(code) <= opcodeIndex || code[opcodeIndex] != 0xc2 { + return Instr{}, 0, false, nil + } + ok = true + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + pp := vexBits & 3 + scalar := pp >= 2 + if scalar && vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar vector floating compare requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: raw mode32 VEX compare high-bank identity", ErrProbeNeedsContext) + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + op := vectorFloatCompareRawOperation(pp) + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource, destination} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), firstSource.String(), secondSource.String(), destination.String()), x86Encoded: true}, immediateIndex + 1, true, nil +} + +func decodedX86EVEXVectorFloatCompareInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+7 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || code[i+4] != 0xc2 { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + pp := p1 & 3 + width64 := p1&0x80 != 0 + wantWidth64 := pp == 1 || pp == 3 + if width64 != wantWidth64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match floating element type") + } + if p2&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing is absent from vector floating compare") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + if rExt != 0 || rHighExt != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX destination is an unextended K register") + } + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + scalar := pp >= 2 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + sae := evexB && registerSource + broadcast := evexB && !registerSource + if broadcast && scalar { + return Instr{}, 0, true, fmt.Errorf("scalar vector floating compare does not support broadcast") + } + if !sae && vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "X" + vectorWidth := 16 + switch { + case sae && !scalar: + vectorPrefix = "Z" + vectorWidth = 64 + case !scalar: + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + laneBytes := 4 + if width64 { + laneBytes = 8 + } + disp8Scale := vectorWidth + if scalar || broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + + op := vectorFloatCompareRawOperation(pp) + if broadcast { + op += ".BCST" + } + if sae { + op += ".SAE" + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + if mode == 32 && (xExt != 0 || bExt != 0 || secondSourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: raw mode32 EVEX compare high-bank identity", ErrProbeNeedsContext) + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM>>3&7))} + args := []Operand{immediate, firstSource, secondSource} + if writeMask := int(p2 & 7); writeMask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", writeMask))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: true}, immediateIndex + 1, true, nil +} + +func vectorFloatCompareRawOperation(pp byte) Op { + return [...]Op{"VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"}[pp] +} + +// decodedX86VEXHorizontalFloatInstruction recognizes the complete VEX.128 and +// VEX.256 VHADD/VHSUB/VADDSUB PS/PD family in Go 1.27's _yvaddsubpd table. +// VEX.W is ignored. Recover every register and ModRM/SIB memory form before +// the generic decoder, which can misdecode these bytes as relative branches. +func decodedX86VEXHorizontalFloatInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + stem, recognized := map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"}[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + suffix := "" + switch vexBits & 3 { + case 1: + suffix = "PD" + case 3: + suffix = "PS" + default: + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select a horizontal floating instruction") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := Op(stem + suffix) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +type decodedX86VEXFMA3Spec struct { + stem string + scalar bool +} + +func (spec decodedX86VEXFMA3Spec) op(width64 bool) Op { + suffix := "PS" + if spec.scalar { + suffix = "SS" + } + if width64 { + if spec.scalar { + suffix = "SD" + } else { + suffix = "PD" + } + } + return Op(spec.stem + suffix) +} + +// decodedX86VEXFMA3Ops covers all 36 packed and 24 scalar FMA3 names in Go +// 1.27's _yvaddpd and _yvaddsd operand tables. VEX.W selects PS/PD or SS/SD. +var decodedX86VEXFMA3Ops = map[byte]decodedX86VEXFMA3Spec{ + 0x96: {stem: "VFMADDSUB132"}, + 0x97: {stem: "VFMSUBADD132"}, + 0x98: {stem: "VFMADD132"}, + 0x99: {stem: "VFMADD132", scalar: true}, + 0x9a: {stem: "VFMSUB132"}, + 0x9b: {stem: "VFMSUB132", scalar: true}, + 0x9c: {stem: "VFNMADD132"}, + 0x9d: {stem: "VFNMADD132", scalar: true}, + 0x9e: {stem: "VFNMSUB132"}, + 0x9f: {stem: "VFNMSUB132", scalar: true}, + 0xa6: {stem: "VFMADDSUB213"}, + 0xa7: {stem: "VFMSUBADD213"}, + 0xa8: {stem: "VFMADD213"}, + 0xa9: {stem: "VFMADD213", scalar: true}, + 0xaa: {stem: "VFMSUB213"}, + 0xab: {stem: "VFMSUB213", scalar: true}, + 0xac: {stem: "VFNMADD213"}, + 0xad: {stem: "VFNMADD213", scalar: true}, + 0xae: {stem: "VFNMSUB213"}, + 0xaf: {stem: "VFNMSUB213", scalar: true}, + 0xb6: {stem: "VFMADDSUB231"}, + 0xb7: {stem: "VFMSUBADD231"}, + 0xb8: {stem: "VFMADD231"}, + 0xb9: {stem: "VFMADD231", scalar: true}, + 0xba: {stem: "VFMSUB231"}, + 0xbb: {stem: "VFMSUB231", scalar: true}, + 0xbc: {stem: "VFNMADD231"}, + 0xbd: {stem: "VFNMADD231", scalar: true}, + 0xbe: {stem: "VFNMSUB231"}, + 0xbf: {stem: "VFNMSUB231", scalar: true}, +} + +// decodedX86VEXFMA3Instruction recognizes every VEX and EVEX FMA3 form behind +// Go 1.27's complete FMA3 table. EVEX adds X/Y/Z widths, masking, zeroing, +// packed broadcasts, and embedded rounding. x/arch v0.14 can split or reject +// raw FMA encodings emitted by generated ecosystem assembly, so recover the +// family before the generic decoder. +func decodedX86VEXFMA3Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXFMA3Instruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + spec, recognized := decodedX86VEXFMA3Ops[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if spec.scalar && vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("scalar FMA3 requires VEX.128") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + op := spec.op(vexBits&0x80 != 0) + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXFMA3Instruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + spec, recognized := decodedX86VEXFMA3Ops[opcode] + half := p0&0x0f == 6 + if p0&0x0f != 2 && !half || !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 || p1&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("EVEX.pp must be 66") + } + if half && p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("FP16 FMA3 requires EVEX.W0") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + embeddedRounding := evexB && registerSource + broadcast := evexB && !registerSource + if broadcast && spec.scalar { + return Instr{}, 0, true, fmt.Errorf("scalar FMA3 does not support broadcast") + } + if !embeddedRounding && vectorBits == 3 && !(half && spec.scalar) { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "X" + vectorWidth := 16 + switch { + case embeddedRounding && !spec.scalar: + vectorPrefix = "Z" + vectorWidth = 64 + case !spec.scalar: + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + laneBytes := 4 + width64 := p1&0x80 != 0 + if width64 { + laneBytes = 8 + } + if half { + laneBytes = 2 + } + disp8Scale := vectorWidth + if spec.scalar || broadcast { + disp8Scale = laneBytes + } + + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (maskNumber != 0 && !half || rExt != 0 || rHighExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register or mask in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + op := spec.op(width64) + if half { + op = Op(spec.stem + "PH") + if spec.scalar { + op = Op(spec.stem + "SH") + } + } + if broadcast { + op += ".BCST" + } + if embeddedRounding { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + if zeroing { + op += ".Z" + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: half}, modRMIndex + consumed, true, nil +} + +// decodedX86VEXTRACT128Instruction recognizes the complete Go 1.27 vector +// lane-extract family: the VEX F128/I128 forms and all EVEX F/I 32/64-bit +// 128/256-bit block forms. x/arch v0.14 can misdecode these raw bytes as +// PC-relative control flow or reject the EVEX encodings outright. +func decodedX86VEXTRACT128Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86VEXTRACT128InstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86VEXTRACT128InstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXTRACTInstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + if len(code) < i+6 || code[i] != 0xc4 || code[i+1]&0x1f != 3 { + return Instr{}, 0, false, nil + } + vex0, vexBits, opcode := code[i+1], code[i+2], code[i+3] + op, recognized := map[byte]Op{0x19: "VEXTRACTF128", 0x39: "VEXTRACTI128"}[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("VEXTRACT128 requires VEX.256") + } + if int(^vexBits>>3)&15 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be 1111") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + modRMIndex := i + 4 + modRM := code[modRMIndex] + sourceNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || sourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + destination, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", sourceNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +func decodedX86EVEXTRACTInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXTRACTInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXTRACTInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 3 || p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + opcode := code[i+4] + width64 := p1&0x80 != 0 + outputBytes := 0 + op := Op("") + switch opcode { + case 0x19: + outputBytes = 16 + op = "VEXTRACTF32X4" + if width64 { + op = "VEXTRACTF64X2" + } + case 0x1b: + outputBytes = 32 + op = "VEXTRACTF32X8" + if width64 { + op = "VEXTRACTF64X4" + } + case 0x39: + outputBytes = 16 + op = "VEXTRACTI32X4" + if width64 { + op = "VEXTRACTI64X2" + } + case 0x3b: + outputBytes = 32 + op = "VEXTRACTI32X8" + if width64 { + op = "VEXTRACTI64X4" + } + default: + return Instr{}, 0, false, nil + } + ok = true + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for vector extract") + } + vectorBits := (p2 >> 5) & 3 + if outputBytes == 16 && vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("128-bit extract requires a 256- or 512-bit source") + } + if outputBytes == 32 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("256-bit extract requires a 512-bit source") + } + sourceName := [...]string{"", "Y", "Z"}[vectorBits] + destinationName := "X" + if outputBytes == 32 { + destinationName = "Y" + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + if zeroing && modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing is invalid for a memory destination") + } + sourceNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + destinationNumber := int(modRM&7) + bExt*8 + xExt*16 + if mode == 32 && (sourceNumber >= 8 || modRM>>6 == 3 && destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destination, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, destinationName, outputBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", sourceName, sourceNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86VSHUFPSInstruction recognizes Go 1.27's complete shared +// VSHUFPS/VSHUFPD operand table. x/arch v0.14 rejects some extended VEX +// encodings and all EVEX forms used by generated ecosystem assembly. +func decodedX86VSHUFPSInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedFloatShuffleInstruction(code, i, mode, segment, addressOverride) + } + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case len(code) >= i+3 && code[i] == 0xc5 && code[i+2] == 0xc6: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1 && code[i+3] == 0xc6: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + ok = true + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + op := Op("") + switch vexBits & 3 { + case 0: + op = "VSHUFPS" + case 1: + op = "VSHUFPD" + default: + return Instr{}, 0, true, fmt.Errorf("VEX.pp does not select VSHUFPS or VSHUFPD") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +func decodedX86EVEXPackedFloatShuffleInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+7 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || p1&0x04 == 0 || code[i+4] != 0xc6 { + return Instr{}, 0, false, nil + } + op := Op("") + switch { + case p1&3 == 0 && p1&0x80 == 0: + op = "VSHUFPS" + case p1&3 == 1 && p1&0x80 != 0: + op = "VSHUFPD" + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 4 + if op == "VSHUFPD" { + disp8Scale = 8 + } + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86PackedShuffleInstruction recognizes the complete Go 1.27 +// _yvpshufd family: VPSHUFD, VPSHUFHW, and VPSHUFLW across VEX.128, +// VEX.256, and EVEX.128/256/512, including memory, mask, zeroing, and the +// VPSHUFD broadcast forms. x/arch v0.14 can split the VEX bytes emitted by +// Weaviate into a relative branch, so recover the family before consulting +// the generic decoder. +func decodedX86PackedShuffleInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vectorPrefix + } + } + +vectorPrefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedShuffleInstruction(code, i, mode, segment, addressOverride) + } + + var vexLength, rExt, xExt, bExt int + var vexBits byte + switch { + case len(code) >= i+3 && code[i] == 0xc5 && code[i+2] == 0x70: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1 && code[i+3] == 0x70: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + default: + return Instr{}, 0, false, nil + } + op := map[byte]Op{1: "VPSHUFD", 2: "VPSHUFHW", 3: "VPSHUFLW"}[vexBits&3] + if op == "" { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, immediate.String(), source.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +func decodedX86EVEXPackedShuffleInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 1 || p1&0x04 == 0 || code[i+4] != 0x70 { + return Instr{}, 0, false, nil + } + op := map[byte]Op{1: "VPSHUFD", 2: "VPSHUFHW", 3: "VPSHUFLW"}[p1&3] + if op == "" || op == "VPSHUFD" && p1&0x80 != 0 { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 packed shuffle mask form exceeds the Go assembler frontend's operand limit") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && op != "VPSHUFD" { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is only available for VPSHUFD") + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 4 + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86Shuffle128BitBlocksInstruction recognizes the complete EVEX-only +// Go 1.27 _yvshuff32x4 family: VSHUFF32X4, VSHUFF64X2, VSHUFI32X4, and +// VSHUFI64X2. x/arch v0.14 rejects these raw encodings, including the +// VSHUFF64X2 sequence in github.com/minio/sha256-simd. +func decodedX86Shuffle128BitBlocksInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+7 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 3 || p1&0x07 != 5 || opcode != 0x23 && opcode != 0x43 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("VSHUFF/VSHUFI block shuffle requires a 256- or 512-bit vector") + } + vectorPrefix := [...]string{"", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{0, 32, 64}[vectorBits] + width64 := p1&0x80 != 0 + op := Op("VSHUFF32X4") + laneBytes := 4 + if opcode == 0x43 { + op = "VSHUFI32X4" + } + if width64 { + laneBytes = 8 + if opcode == 0x23 { + op = "VSHUFF64X2" + } else { + op = "VSHUFI64X2" + } + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86VPSHUFBInstruction recognizes every VEX and EVEX encoding behind +// Go 1.27's _yvandnpd table. x/arch v0.14 rejects the EVEX forms used by +// github.com/minio/sha256-simd. +func decodedX86VPSHUFBInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vectorPrefix + } + } + +vectorPrefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + if code[i] == 0xc4 && code[i+1]&0x1f == 2 && code[i+3] == 0x00 { + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vex0, vex1 := code[i+1], code[i+2] + if vex1&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + // Intel PSHUFB specifies WIG for both VEX and EVEX. Go emits W0, + // but that encoding choice cannot reject independently decoded W1. + if len(code) <= i+4 { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + vectorName := "X" + if vex1&0x04 != 0 { + vectorName = "Y" + } + modRMIndex := i + 4 + modRM := code[modRMIndex] + dataNumber := int(^vex1>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (dataNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + control, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, dataNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: "VPSHUFB", + Args: []Operand{control, data, destination}, + Raw: fmt.Sprintf("VPSHUFB %s, %s, %s", control.String(), data.String(), destination.String()), + x86Encoded: true, + }, modRMIndex + consumed, true, nil + } + + if len(code) < i+5 || code[i] != 0x62 || code[i+1]&0x0f != 2 || code[i+2]&0x07 != 5 || code[i+4] != 0x00 { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for VPSHUFB") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + dataNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (dataNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + control, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, vectorWidth) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, dataNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{control, data} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := Op("VPSHUFB") + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: true}, modRMIndex + consumed, true, nil +} + +type decodedX86PackedExtendProperties struct { + op Op + inputBits int + outputBits int +} + +func decodedX86PackedExtendOpcode(opcode byte) (decodedX86PackedExtendProperties, bool) { + properties := map[byte]decodedX86PackedExtendProperties{ + 0x20: {op: "VPMOVSXBW", inputBits: 8, outputBits: 16}, + 0x21: {op: "VPMOVSXBD", inputBits: 8, outputBits: 32}, + 0x22: {op: "VPMOVSXBQ", inputBits: 8, outputBits: 64}, + 0x23: {op: "VPMOVSXWD", inputBits: 16, outputBits: 32}, + 0x24: {op: "VPMOVSXWQ", inputBits: 16, outputBits: 64}, + 0x25: {op: "VPMOVSXDQ", inputBits: 32, outputBits: 64}, + 0x30: {op: "VPMOVZXBW", inputBits: 8, outputBits: 16}, + 0x31: {op: "VPMOVZXBD", inputBits: 8, outputBits: 32}, + 0x32: {op: "VPMOVZXBQ", inputBits: 8, outputBits: 64}, + 0x33: {op: "VPMOVZXWD", inputBits: 16, outputBits: 32}, + 0x34: {op: "VPMOVZXWQ", inputBits: 16, outputBits: 64}, + 0x35: {op: "VPMOVZXDQ", inputBits: 32, outputBits: 64}, + }[opcode] + return properties, properties.op != "" +} + +// decodedX86PackedExtendInstruction recognizes all six signed and all six +// unsigned packed extending moves in Go 1.27. It covers their VEX.128/256 and +// EVEX.128/256/512 register and memory forms, including high registers, +// writemasks, zeroing, and compressed displacements. EVEX.b is deliberately +// rejected because no broadcast form exists in the Go operand tables. +func decodedX86PackedExtendInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vectorPrefix + } + } + +vectorPrefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXPackedExtendInstruction(code, i, mode, segment, addressOverride) + } + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + vex0, vex1 := code[i+1], code[i+2] + properties, recognized := decodedX86PackedExtendOpcode(code[i+3]) + // Intel SDM PMOVSX/PMOVZX: VEX.W is ignored for all six ratios. + // Go chooses W0, but raw W1 is not an invalid instruction. + if !recognized || vex1&3 != 1 { + return Instr{}, 0, false, nil + } + ok = true + if vex1&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vex0>>7) & 1 + xExt := int(^vex0>>6) & 1 + bExt := int(^vex0>>5) & 1 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (destinationNumber >= 8 || code[modRMIndex]>>6 == 3 && int(code[modRMIndex]&7)+bExt*8 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destinationName := "X" + if vex1&0x04 != 0 { + destinationName = "Y" + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationName, destinationNumber))} + instruction = Instr{ + Op: properties.op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", properties.op, source.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXPackedExtendInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x0f != 2 || p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86PackedExtendOpcode(code[i+4]) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + // The EVEX DQ forms alone require W0; the other five ratios are WIG. + // Keep this physical constraint distinct from Go's W0 encoder choice. + if properties.inputBits == 32 && p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed dword-to-qword extension requires W0") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for packed extending moves") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + destinationName := [...]string{"X", "Y", "Z"}[vectorBits] + destinationBytes := [...]int{16, 32, 64}[vectorBits] + sourceBytes := destinationBytes * properties.inputBits / properties.outputBits + sourceName := "X" + if sourceBytes > 16 { + sourceName = "Y" + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + sourceNumber := int(modRM&7) + bExt*8 + xExt*16 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && sourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, sourceName, sourceBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationName, destinationNumber))} + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +type decodedX86PackedUniformShiftProperties struct { + op Op + laneBytes int + immediate bool +} + +func decodedX86PackedUniformShiftOpcode(opcode, modRM byte, evex, w bool) (decodedX86PackedUniformShiftProperties, bool) { + laneBytes := map[byte]int{0x71: 2, 0x72: 4, 0x73: 8}[opcode] + if laneBytes != 0 { + direction := modRM >> 3 & 7 + if direction != 2 && direction != 4 && direction != 6 { + return decodedX86PackedUniformShiftProperties{}, false + } + stem := "VPSRL" + switch direction { + case 4: + if laneBytes == 8 { + return decodedX86PackedUniformShiftProperties{}, false + } + stem = "VPSRA" + if evex && w && laneBytes == 4 { + laneBytes = 8 + } + case 6: + stem = "VPSLL" + } + suffix := map[int]string{2: "W", 4: "D", 8: "Q"}[laneBytes] + return decodedX86PackedUniformShiftProperties{op: Op(stem + suffix), laneBytes: laneBytes, immediate: true}, true + } + variable := map[byte]struct { + op Op + laneBytes int + }{ + 0xf1: {op: "VPSLLW", laneBytes: 2}, + 0xf2: {op: "VPSLLD", laneBytes: 4}, + 0xf3: {op: "VPSLLQ", laneBytes: 8}, + 0xd1: {op: "VPSRLW", laneBytes: 2}, + 0xd2: {op: "VPSRLD", laneBytes: 4}, + 0xd3: {op: "VPSRLQ", laneBytes: 8}, + 0xe1: {op: "VPSRAW", laneBytes: 2}, + 0xe2: {op: "VPSRAD", laneBytes: 4}, + } + properties, ok := variable[opcode] + if !ok { + return decodedX86PackedUniformShiftProperties{}, false + } + if evex && w && opcode == 0xe2 { + properties.op, properties.laneBytes = "VPSRAQ", 8 + } + return decodedX86PackedUniformShiftProperties{op: properties.op, laneBytes: properties.laneBytes}, true +} + +// decodedX86PackedUniformShiftInstruction recognizes the complete Go 1.27 +// immediate and XMM/m128-count grammar: six logical shifts and three +// arithmetic right shifts across VEX/EVEX X/Y/Z widths. x/arch v0.14 +// misdecodes several VEX/EVEX forms used by external assembly. +func decodedX86PackedUniformShiftInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vectorPrefix + } + } + +vectorPrefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + + var vexBits byte + var opcodeIndex, modRMIndex, rExt, xExt, bExt int + isVEX := false + vexHasW := false + switch { + case code[i] == 0xc5: + vexBits = code[i+1] + opcodeIndex = i + 2 + modRMIndex = i + 3 + rExt = int(^vexBits>>7) & 1 + isVEX = true + case len(code) >= i+5 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vex0 := code[i+1] + vexBits = code[i+2] + opcodeIndex = i + 3 + modRMIndex = i + 4 + rExt = int(^vex0>>7) & 1 + xExt = int(^vex0>>6) & 1 + bExt = int(^vex0>>5) & 1 + isVEX = true + vexHasW = true + } + if isVEX { + properties, recognized := decodedX86PackedUniformShiftOpcode(code[opcodeIndex], code[modRMIndex], false, vexHasW && vexBits&0x80 != 0) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexHasW && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + modRM := code[modRMIndex] + vvvvNumber := int(^vexBits>>3) & 15 + if properties.immediate { + if rExt != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.R must not extend the shift opcode selector") + } + if mode == 32 && vvvvNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + return Instr{ + Op: properties.op, + Args: []Operand{immediate, source, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", properties.op, immediate.String(), source.String(), destination.String()), + x86Encoded: true, + }, immediateIndex + 1, true, nil + } + + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (vvvvNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + count, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X") + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: properties.op, + Args: []Operand{count, data, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", properties.op, count.String(), data.String(), destination.String()), + x86Encoded: true, + }, modRMIndex + consumed, true, nil + } + + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + modRMIndex = i + 5 + modRM := code[modRMIndex] + properties, recognized := decodedX86PackedUniformShiftOpcode(code[i+4], modRM, true, p1&0x80 != 0) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if (p1&0x80 != 0) != (properties.laneBytes == 8) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match %s", properties.op) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + rExt = int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt = int(^p0>>6) & 1 + bExt = int(^p0>>5) & 1 + vvvvNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + broadcast := p2&0x10 != 0 + if properties.immediate { + if p0&0x90 != 0x90 { + return Instr{}, 0, true, fmt.Errorf("EVEX.R and EVEX.R' must not extend the shift opcode selector") + } + if broadcast && properties.laneBytes == 2 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for %s", properties.op) + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + if mode == 32 && (vvvvNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = properties.laneBytes + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: true}, immediateIndex + 1, true, nil + } + + if broadcast { + return Instr{}, 0, true, fmt.Errorf("EVEX.b is unavailable for the uniform-count form of %s", properties.op) + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (vvvvNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + count, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "X", 16) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, vvvvNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{count, data} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: true}, modRMIndex + consumed, true, nil +} + +// decodedX86TernaryLogicInstruction recognizes VPTERNLOGD/Q across the +// complete Go 1.27 _yvalignd table. x/arch v0.14 rejects the EVEX encodings +// used by github.com/minio/sha256-simd. +func decodedX86TernaryLogicInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + instruction, length, ok, err = decodedX86EVEXImmediateThreeVectorInstruction(code, mode, 0x25, "VPTERNLOGD", "VPTERNLOGQ", "VPTERNLOG") + if ok && err == nil { + _, _, err = x86TernaryLogicProperties(instruction.Op, instruction, "amd64") + } + return instruction, length, ok, err +} + +// decodedX86VectorAlignInstruction recognizes VALIGND/Q across the complete +// Go 1.27 _yvalignd table. +func decodedX86VectorAlignInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXImmediateThreeVectorInstruction(code, mode, 0x03, "VALIGND", "VALIGNQ", "VALIGN") +} + +// decodedX86FunnelShiftInstruction recognizes the complete Go 1.27 VBMI2 +// funnel-shift family: immediate/variable, left/right, and W/D/Q lanes. +// x/arch v0.14 rejects all of these EVEX encodings. +func decodedX86FunnelShiftInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + mapNumber := p0 & 0x0f + variable := false + switch mapNumber { + case 2: + variable = true + case 3: + default: + return Instr{}, 0, false, nil + } + opcode := code[i+4] + wBit := p1&0x80 != 0 + laneBits := 0 + left := false + switch opcode { + case 0x70: + left, laneBits = true, 16 + case 0x71: + left = true + if wBit { + laneBits = 64 + } else { + laneBits = 32 + } + case 0x72: + laneBits = 16 + case 0x73: + if wBit { + laneBits = 64 + } else { + laneBits = 32 + } + default: + return Instr{}, 0, false, nil + } + ok = true + if laneBits == 16 && !wBit { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be one for word funnel shifts") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if !variable && mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 immediate funnel shifts exceed the Go assembler frontend's operand limit") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if variable && mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 variable funnel-shift mask form exceeds the Go assembler frontend's operand limit") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && laneBits == 16 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for word funnel shifts") + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBits / 8 + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + opDirection := "R" + if left { + opDirection = "L" + } + laneSuffix := map[int]string{16: "W", 32: "D", 64: "Q"}[laneBits] + op := Op("VPSH" + opDirection + "D") + if variable { + op += "V" + } + op += Op(laneSuffix) + args := make([]Operand, 0, 5) + end := modRMIndex + consumed + if !variable { + if len(code) <= end { + return Instr{}, 0, false, nil + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[end])}) + end++ + } + args = append(args, firstSource, secondSource) + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil +} + +// decodedX86EVEXImmediateThreeVectorInstruction implements the EVEX immediate, +// r/m, vvvv, destination shape shared by Go 1.27's _yvalignd families. +func decodedX86EVEXImmediateThreeVectorInstruction(code []byte, mode int, opcode byte, dwordOp, qwordOp Op, family string) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXImmediateThreeVectorInstructionWithMemory(code, mode, opcode, dwordOp, qwordOp, family, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXImmediateThreeVectorInstructionWithMemory(code []byte, mode int, opcode byte, dwordOp, qwordOp Op, family string, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+7 || code[i] != 0x62 || code[i+1]&0x0f != 3 || code[i+2]&0x07 != 5 || code[i+4] != opcode { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if mode == 32 { + return Instr{}, 0, true, fmt.Errorf("386 %s is rejected by the Go assembler frontend's operand limit", family) + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + op := dwordOp + if p1&0x80 != 0 { + laneBytes = 8 + op = qwordOp + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +// decodedX86PackedImmediateRotateInstruction recognizes all four EVEX +// immediate rotates in Go 1.27's _yvprold table. x/arch v0.14 rejects these +// raw encodings, including VPRORD in github.com/minio/sha256-simd. +func decodedX86PackedImmediateRotateInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+7 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+2]&0x07 != 5 || code[i+4] != 0x72 { + return Instr{}, 0, false, nil + } + modRMIndex := i + 5 + modRM := code[modRMIndex] + selector := modRM >> 3 & 7 + if selector != 0 && selector != 1 { + // Opcode 72 is a grouped encoding. Selectors /2 and /6 are the + // VPSRL/VPSLL logical-shift family, and other selectors may belong to + // decoders added later. This decoder only owns the rotate selectors. + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p0&0x90 != 0x90 { + return Instr{}, 0, true, fmt.Errorf("EVEX.R and EVEX.R' must not extend the rotate opcode selector") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + laneBytes := 4 + left := false + switch selector { + case 0: + case 1: + left = true + } + op := Op("VPRORD") + if left { + op = "VPROLD" + } + if p1&0x80 != 0 { + laneBytes = 8 + if left { + op = "VPROLQ" + } else { + op = "VPRORQ" + } + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 %s mask form exceeds the Go assembler frontend's operand limit", op) + } + + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + destinationNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + source, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{immediate, source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, immediateIndex + 1, true, nil +} + +var decodedX86PackedIntegerArithmeticOps = map[byte]struct { + op Op + laneBytes int +}{ + 0xfc: {op: "VPADDB", laneBytes: 1}, + 0xfd: {op: "VPADDW", laneBytes: 2}, + 0xfe: {op: "VPADDD", laneBytes: 4}, + 0xd4: {op: "VPADDQ", laneBytes: 8}, + 0xec: {op: "VPADDSB", laneBytes: 1}, + 0xed: {op: "VPADDSW", laneBytes: 2}, + 0xdc: {op: "VPADDUSB", laneBytes: 1}, + 0xdd: {op: "VPADDUSW", laneBytes: 2}, + 0xf8: {op: "VPSUBB", laneBytes: 1}, + 0xf9: {op: "VPSUBW", laneBytes: 2}, + 0xfa: {op: "VPSUBD", laneBytes: 4}, + 0xfb: {op: "VPSUBQ", laneBytes: 8}, + 0xe8: {op: "VPSUBSB", laneBytes: 1}, + 0xe9: {op: "VPSUBSW", laneBytes: 2}, + 0xd8: {op: "VPSUBUSB", laneBytes: 1}, + 0xd9: {op: "VPSUBUSW", laneBytes: 2}, +} + +// decodedX86PackedIntegerArithmeticInstruction recognizes the complete +// VEX/EVEX VPADD and VPSUB families behind Go 1.27's shared _yvandnpd table. +// x/arch v0.14 rejects EVEX forms and can split VEX forms used by ecosystem +// assembly. +func decodedX86PackedIntegerArithmeticInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vectorPrefix + } + } + +vectorPrefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + + var vexBits byte + var opcodeIndex, modRMIndex, rExt, xExt, bExt int + isVEX := false + switch { + case code[i] == 0xc5: + vexBits = code[i+1] + opcodeIndex = i + 2 + modRMIndex = i + 3 + rExt = int(^vexBits>>7) & 1 + isVEX = true + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vex0 := code[i+1] + vexBits = code[i+2] + opcodeIndex = i + 3 + modRMIndex = i + 4 + rExt = int(^vex0>>7) & 1 + xExt = int(^vex0>>6) & 1 + bExt = int(^vex0>>5) & 1 + isVEX = true + } + if isVEX { + properties, recognized := decodedX86PackedIntegerArithmeticOps[code[opcodeIndex]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if vexBits&0x04 != 0 { + vectorName = "Y" + } + modRM := code[modRMIndex] + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + return Instr{ + Op: properties.op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", properties.op, firstSource.String(), secondSource.String(), destination.String()), + }, modRMIndex + consumed, true, nil + } + + if len(code) < i+5 || code[i] != 0x62 || code[i+1]&0x0f != 1 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86PackedIntegerArithmeticOps[code[i+4]] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + wBit := p1&0x80 != 0 + if wBit != (properties.laneBytes == 8) { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match %s", properties.op) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 %s mask form exceeds the Go assembler frontend's operand limit", properties.op) + } + + rExt = int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt = int(^p0>>6) & 1 + bExt = int(^p0>>5) & 1 + modRMIndex = i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && properties.laneBytes != 4 && properties.laneBytes != 8 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for %s", properties.op) + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = properties.laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := properties.op + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86IndexedPermuteInstruction recognizes the complete EVEX-only +// VPERMI2 and VPERMT2 families behind Go 1.27's shared _yvblendmpd table. +func decodedX86IndexedPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&0x07 != 5 || + (opcode < 0x75 || opcode > 0x77) && (opcode < 0x7d || opcode > 0x7f) { + return Instr{}, 0, false, nil + } + width64 := p1&0x80 != 0 + op := Op("") + laneBytes := 0 + switch opcode { + case 0x75: + if width64 { + op, laneBytes = "VPERMI2W", 2 + } else { + op, laneBytes = "VPERMI2B", 1 + } + case 0x76: + if width64 { + op, laneBytes = "VPERMI2Q", 8 + } else { + op, laneBytes = "VPERMI2D", 4 + } + case 0x77: + if width64 { + op, laneBytes = "VPERMI2PD", 8 + } else { + op, laneBytes = "VPERMI2PS", 4 + } + case 0x7d: + if width64 { + op, laneBytes = "VPERMT2W", 2 + } else { + op, laneBytes = "VPERMT2B", 1 + } + case 0x7e: + if width64 { + op, laneBytes = "VPERMT2Q", 8 + } else { + op, laneBytes = "VPERMT2D", 4 + } + case 0x7f: + if width64 { + op, laneBytes = "VPERMT2PD", 8 + } else { + op, laneBytes = "VPERMT2PS", 4 + } + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 indexed permute mask form exceeds the Go assembler operand limit") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && laneBytes < 4 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for byte and word VPERMI2") + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + if mode == 32 && vectorBits == 2 && + (destinationNumber >= 8 || secondSourceNumber >= 8 || + modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("386 Z register exceeds the Go assembler frontend's register class") + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// Go 1.27's _yvcompresspd and _yvexpandpd families share the same typed +// register/memory grammar, with the ModRM roles reversed for stores. +func decodedX86PackedCompressInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedCompactInstruction(code, mode, true) +} + +func decodedX86PackedExpandInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedCompactInstruction(code, mode, false) +} + +func decodedX86PackedCompactInstruction(code []byte, mode int, store bool) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + if store && opcode != 0x63 && opcode != 0x8a && opcode != 0x8b || + !store && opcode != 0x62 && opcode != 0x88 && opcode != 0x89 { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX packed compact reserves the vvvv register field") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if p2&0x10 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is unavailable for packed compact") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 packed compact mask form exceeds the Go assembler operand limit") + } + width64 := p1&0x80 != 0 + op := Op("") + laneBytes := 0 + switch opcode { + case 0x63: + if width64 { + op, laneBytes = "VPCOMPRESSW", 2 + } else { + op, laneBytes = "VPCOMPRESSB", 1 + } + case 0x8a: + if width64 { + op, laneBytes = "VCOMPRESSPD", 8 + } else { + op, laneBytes = "VCOMPRESSPS", 4 + } + case 0x8b: + if width64 { + op, laneBytes = "VPCOMPRESSQ", 8 + } else { + op, laneBytes = "VPCOMPRESSD", 4 + } + case 0x62: + if width64 { + op, laneBytes = "VPEXPANDW", 2 + } else { + op, laneBytes = "VPEXPANDB", 1 + } + case 0x88: + if width64 { + op, laneBytes = "VEXPANDPD", 8 + } else { + op, laneBytes = "VEXPANDPS", 4 + } + case 0x89: + if width64 { + op, laneBytes = "VPEXPANDQ", 8 + } else { + op, laneBytes = "VPEXPANDD", 4 + } + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + if mode == 32 && vectorBits == 2 && + (int(modRM>>3&7)+rExt*8+rHighExt*16 >= 8 || + modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("386 Z register exceeds the Go assembler frontend's register class") + } + regNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, regNumber))} + rm, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, laneBytes) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + args := []Operand{reg} + if !store { + args = []Operand{rm} + } + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + if store { + args = append(args, rm) + } else { + args = append(args, reg) + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86IFMAInstruction recognizes the complete EVEX-only +// VPMADD52HUQ/LUQ family behind Go 1.27's _yvblendmpd table. x/arch v0.14 +// rejects these encodings. +func decodedX86IFMAInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 || code[i+1]&0x0f != 2 || code[i+2]&0x07 != 5 { + return Instr{}, 0, false, nil + } + op := Op("") + switch code[i+4] { + case 0xb4: + op = "VPMADD52LUQ" + case 0xb5: + op = "VPMADD52HUQ" + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x80 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be one for %s", op) + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 %s mask form exceeds the Go assembler frontend's operand limit", op) + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 8 + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86VPINSRInstruction recognizes the complete VEX.128 VPINSR +// byte/word/dword/qword family, including two- and three-byte VEX encodings and +// every ModRM/SIB source. x/arch v0.14 rejects some extended-register VPINSRQ +// forms used by generated crypto assembly. EVEX forms continue through x/arch. +func decodedX86VPINSRInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits byte + op := Op("") + switch { + case len(code) >= i+3 && code[i] == 0xc5 && code[i+2] == 0xc4: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + op = "VPINSRW" + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1 && code[i+3] == 0xc4: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + op = "VPINSRW" + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 3 && code[i+3] == 0x20: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + op = "VPINSRB" + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 3 && code[i+3] == 0x22: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + op = "VPINSRD" + if vexBits&0x80 != 0 { + op = "VPINSRQ" + } + default: + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be zero") + } + if (op == "VPINSRB" || op == "VPINSRW") && vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero for %s", op) + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + firstVectorNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (op == "VPINSRQ" || rExt != 0 || xExt != 0 || bExt != 0 || firstVectorNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("64-bit width or extended register in 32-bit mode") + } + scalar, consumed, err := decodedX86VEXRMSource(code[modRMIndex:], mode, bExt, xExt, segment) + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := modRMIndex + consumed + if len(code) <= immediateIndex { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + firstVector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", firstVectorNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{immediate, scalar, firstVector, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), scalar.String(), firstVector.String(), destination.String()), + } + return instruction, immediateIndex + 1, true, nil +} + +// decodedX86VPUNPCKInstruction recognizes the complete VEX.128/VEX.256 +// low/high byte, word, dword, and qword unpack family. x/arch v0.14 can split +// or reject VEX encodings with extended registers, so recover every VEX +// register and ModRM/SIB memory form before the generic decoder. EVEX forms +// use the separate decodedX86EVEXPackedUnpackInstruction grammar. +func decodedX86VPUNPCKInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+3 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+4 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + op, recognized := map[byte]Op{ + 0x60: "VPUNPCKLBW", + 0x61: "VPUNPCKLWD", + 0x62: "VPUNPCKLDQ", + 0x6c: "VPUNPCKLQDQ", + 0x68: "VPUNPCKHBW", + 0x69: "VPUNPCKHWD", + 0x6a: "VPUNPCKHDQ", + 0x6d: "VPUNPCKHQDQ", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexLength == 3 && vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86VEXVectorRMOperand(code []byte, mode, bExt, xExt int, segment Reg, vectorPrefix string) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 != 3 { + return decodedX86VEXRMSource(code, mode, bExt, xExt, segment) + } + number := int(code[0]&7) + bExt*8 + if mode == 32 && number >= 8 { + return Operand{}, 0, fmt.Errorf("extended vector register in 32-bit mode") + } + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, number))}, 1, nil +} + +// decodedX86VNNIInstruction recognizes the complete VEX.128/VEX.256 +// AVX-VNNI dot-product family. Go's assembler exposes the equivalent named +// EVEX forms, while generated third-party assembly uses raw VEX encodings that +// x/arch v0.14 cannot decode. Recover all four opcodes and every register or +// ModRM/SIB memory source before consulting the generic decoder. +func decodedX86VNNIInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + vexBits := code[i+2] + opcode := code[i+3] + op, recognized := map[byte]Op{ + 0x50: "VPDPBUSD", + 0x51: "VPDPBUSDS", + 0x52: "VPDPWSSD", + 0x53: "VPDPWSSDS", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + vex1 := code[i+1] + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + secondSourceNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || secondSourceNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + firstSource, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if err != nil { + return Instr{}, 0, true, err + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, firstSource.String(), secondSource.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86VariableDwordPermuteInstruction recognizes the complete VEX.256 +// VPERMD/VPERMPS family shared by Go 1.27's _yvpermd table. x/arch v0.14 can +// reject or split these encodings, so recover every register and ModRM/SIB +// memory source before consulting the generic decoder. EVEX forms continue +// through x/arch. +func decodedX86VariableDwordPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 2 { + return Instr{}, 0, false, nil + } + vexBits := code[i+2] + opcode := code[i+3] + op, recognized := map[byte]Op{ + 0x36: "VPERMD", + 0x16: "VPERMPS", + }[opcode] + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if vexBits&3 != 1 { + return Instr{}, 0, true, fmt.Errorf("VEX.pp must be 66") + } + if vexBits&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be one") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + vex1 := code[i+1] + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + modRMIndex := i + 4 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + indicesNumber := int(^vexBits>>3) & 15 + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || indicesNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + data, consumed, err := decodedX86VEXVectorRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, "Y") + if err != nil { + return Instr{}, 0, true, err + } + indices := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", indicesNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("Y%d", destinationNumber))} + instruction = Instr{ + Op: op, + Args: []Operand{data, indices, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, data.String(), indices.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +// decodedX86EVEXVariableDwordPermuteInstruction covers the Y/Z EVEX forms of +// Go 1.27's shared VPERMD/VPERMPS _yvpermd table. +func decodedX86EVEXVariableDwordPermuteInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&0x07 != 5 || (opcode != 0x36 && opcode != 0x16) { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be zero for VPERMD/VPERMPS") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits != 1 && vectorBits != 2 { + return Instr{}, 0, true, fmt.Errorf("VPERMD/VPERMPS EVEX form requires Y or Z width") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + if mode == 32 && maskNumber != 0 { + return Instr{}, 0, true, fmt.Errorf("386 VPERMD/VPERMPS mask form exceeds the Go assembler operand limit") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = 4 + } + data, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + indicesNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && vectorBits == 2 && + (indicesNumber >= 8 || destinationNumber >= 8 || + modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("386 Z register exceeds the Go assembler frontend's register class") + } + indices := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, indicesNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{data, indices} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + op := Op("VPERMD") + if opcode == 0x16 { + op = "VPERMPS" + } + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, modRMIndex + consumed, true, nil +} + +// decodedX86RandomInstruction recognizes the complete register-only +// RDRAND/RDSEED/RDPID encoding family. RDPID shares RDSEED's ModRM extension +// but requires F3 and has only a 32-bit destination. x/arch v0.14 drops the +// operand from 64-bit RDRAND and cannot decode RDSEED or RDPID, so recover +// these encodings before consulting the generic decoder. +func decodedX86RandomInstruction(code []byte, mode int) (syntax string, length int, ok bool) { + i := 0 + rdpid := len(code) > 0 && code[0] == 0xf3 + if rdpid { + i++ + } + bits := 32 + if len(code) > i && code[i] == 0x66 { + bits = 16 + i++ + } + rex := byte(0) + if mode == 64 && len(code) > i && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + if rex&0x08 != 0 { + bits = 64 + } + } + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0xc7 { + return "", 0, false + } + modRM := code[i+2] + if modRM>>6 != 3 { + return "", 0, false + } + family := "" + switch modRM >> 3 & 7 { + case 6: + if rdpid { + return "", 0, false + } + family = "RDRAND" + case 7: + if rdpid { + if bits != 32 { + return "", 0, false + } + family = "RDPID" + } else { + family = "RDSEED" + } + default: + return "", 0, false + } + register := int(modRM & 7) + if mode == 64 && rex&0x01 != 0 { + register += 8 + } + registerNames := [...]string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + if rdpid { + return fmt.Sprintf("RDPID %s", registerNames[register]), i + 3, true + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + return fmt.Sprintf("%s%s %s", family, width, registerNames[register]), i + 3, true +} + +// decodedX86VBROADCAST128Instruction recognizes the complete AVX2 +// VBROADCASTF128/I128 m128, Y family. x/arch v0.14 misdecodes these VEX +// opcodes as legacy scalar instructions, so recover every ModRM/SIB memory +// form before consulting the generic decoder. +func decodedX86VBROADCAST128Instruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + // VEX.256.66.0F38.W0 1A/5A /r. The unused encoded vvvv field must be + // 1111, so the complete third VEX byte is 0x7d. + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+2] != 0x7d || + (code[i+3] != 0x1a && code[i+3] != 0x5a) { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + vex1 := code[i+1] + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + destinationNumber := int(code[i+4]>>3&7) + rExt*8 + if mode == 32 && (rExt != 0 || xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + source, consumed, err := decodedX86VEXRMSource(code[i+4:], mode, bExt, xExt, segment) + if err != nil { + return Instr{}, 0, true, err + } + if source.Kind != OpMem { + return Instr{}, 0, true, fmt.Errorf("source must be m128 memory") + } + destination := Reg(fmt.Sprintf("Y%d", destinationNumber)) + op := Op("VBROADCASTI128") + if code[i+3] == 0x1a { + op = "VBROADCASTF128" + } + instruction = Instr{ + Op: op, + Args: []Operand{source, {Kind: OpReg, Reg: destination}}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination), + } + return instruction, i + 4 + consumed, true, nil +} + +// decodedX86SHAInstruction recognizes the complete Go 1.27 Intel SHA table: +// SHA1NEXTE, SHA1MSG1/2, SHA1RNDS4, SHA256MSG1/2, and SHA256RNDS2. Older +// x/arch decoders reject some raw 0F38 encodings used by minio/sha256-simd. +func decodedX86SHAInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto rex + } + } + +rex: + rexPrefix := byte(0) + if mode == 64 && i < len(code) && code[i]&0xf0 == 0x40 { + rexPrefix = code[i] + i++ + } + if len(code) < i+4 || code[i] != 0x0f { + return Instr{}, 0, false, nil + } + + op := Op("") + immediate := false + switch { + case code[i+1] == 0x38: + switch code[i+2] { + case 0xc8: + op = "SHA1NEXTE" + case 0xc9: + op = "SHA1MSG1" + case 0xca: + op = "SHA1MSG2" + case 0xcb: + op = "SHA256RNDS2" + case 0xcc: + op = "SHA256MSG1" + case 0xcd: + op = "SHA256MSG2" + default: + return Instr{}, 0, false, nil + } + case code[i+1] == 0x3a && code[i+2] == 0xcc: + op = "SHA1RNDS4" + immediate = true + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 3 + modRM := code[modRMIndex] + rExt := int(rexPrefix>>2) & 1 + xExt := int(rexPrefix>>1) & 1 + bExt := int(rexPrefix) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + + var source Operand + consumed := 0 + if modRM>>6 == 3 { + sourceNumber := int(modRM&7) + bExt*8 + if mode == 32 && sourceNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source = Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", sourceNumber))} + consumed = 1 + } else { + var decodeErr error + source, consumed, decodeErr = decodedX86RMSource(code[modRMIndex:], mode, bExt, xExt, segment, 1) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + } + + args := []Operand{source, destination} + end := modRMIndex + consumed + if immediate { + if len(code) <= end { + return Instr{}, 0, true, fmt.Errorf("missing immediate byte") + } + // SHA1RNDS4 consumes only the low two bits. Normalize the raw byte to + // Go's Yu2 operand class so the recovered instruction remains valid. + args = []Operand{{Kind: OpImm, Imm: int64(code[end] & 3)}, source, destination} + end++ + } else if op == "SHA256RNDS2" { + args = []Operand{{Kind: OpReg, Reg: Reg("X0")}, source, destination} + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", "))}, end, true, nil +} + +// decodedX86BLSInstruction recognizes the complete BMI1 BLSI/BLSMSK/BLSR +// register and ModRM/SIB memory families. x/arch v0.14 reports these VEX.NDD +// encodings as unknown AVX opcodes, so recover them before generic decoding. +func decodedX86BLSInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+2]&0x07 != 0 || code[i+3] != 0xf3 { + return Instr{}, 0, false, nil + } + ok = true + vex1, vex2 := code[i+1], code[i+2] + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + width64 := vex2&0x80 != 0 + destinationNumber := int(^vex2>>3) & 15 + modRM := code[i+4] + family := "" + switch modRM >> 3 & 7 { + case 1: + family = "BLSR" + case 2: + family = "BLSMSK" + case 3: + family = "BLSI" + default: + return Instr{}, 0, true, fmt.Errorf("invalid opcode extension /%d", modRM>>3&7) + } + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if mode == 32 && (xExt != 0 || bExt != 0 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended register in 32-bit mode") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + source, consumed, err := decodedX86VEXRMSource(code[i+4:], mode, bExt, xExt, segment) + if err != nil { + return Instr{}, 0, true, err + } + destination, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination register %d", destinationNumber) + } + op := Op(family + "L") + if width64 && mode == 64 { + op = Op(family + "Q") + } + instruction = Instr{ + Op: op, + Args: []Operand{source, {Kind: OpReg, Reg: destination}}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination), + } + return instruction, i + 4 + consumed, true, nil +} + +// decodedX86MULXInstruction recognizes the complete BMI2 MULX register and +// ModRM/SIB memory family. x/arch rejects this VEX.NDD encoding even though +// Go's assembler and the semantic lowerer support MULXL/MULXQ. +func decodedX86MULXInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+5 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+2]&0x07 != 3 || code[i+3] != 0xf6 { + return Instr{}, 0, false, nil + } + ok = true + vex1, vex2 := code[i+1], code[i+2] + if vex2&0x04 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.L must be zero") + } + rExt := int(^vex1>>7) & 1 + xExt := int(^vex1>>6) & 1 + bExt := int(^vex1>>5) & 1 + width64 := vex2&0x80 != 0 + lowRegister := int(^vex2>>3) & 15 + modRM := code[i+4] + highRegister := int(modRM>>3&7) + rExt*8 + if mode == 32 && (width64 || rExt != 0 || xExt != 0 || bExt != 0 || lowRegister >= 8 || highRegister >= 8) { + return Instr{}, 0, true, fmt.Errorf("64-bit width or extended register in 32-bit mode") + } + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + source, consumed, err := decodedX86VEXRMSource(code[i+4:], mode, bExt, xExt, segment) + if err != nil { + return Instr{}, 0, true, err + } + low, valid := decodedX86GeneralRegister(lowRegister) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid low destination register %d", lowRegister) + } + high, valid := decodedX86GeneralRegister(highRegister) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid high destination register %d", highRegister) + } + op := Op("MULXL") + if width64 { + op = "MULXQ" + } + instruction = Instr{ + Op: op, + Args: []Operand{source, {Kind: OpReg, Reg: low}, {Kind: OpReg, Reg: high}}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, source.String(), low, high), + } + return instruction, i + 4 + consumed, true, nil +} + +func decodedX86VEXRMSource(code []byte, mode, bExt, xExt int, segment Reg) (Operand, int, error) { + return decodedX86RMSource(code, mode, bExt, xExt, segment, 1) +} + +func decodedX86RMSource(code []byte, mode, bExt, xExt int, segment Reg, disp8Scale int) (Operand, int, error) { + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, disp8Scale, false) +} + +func decodedX86RMSourceWithPC(code []byte, mode, bExt, xExt int, segment Reg, disp8Scale int, physicalPC bool) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if disp8Scale <= 0 { + return Operand{}, 0, fmt.Errorf("invalid disp8 scale %d", disp8Scale) + } + modRM := code[0] + mod := int(modRM >> 6) + rm := int(modRM & 7) + if mod == 3 { + register, ok := decodedX86GeneralRegister(rm + bExt*8) + if !ok || mode == 32 && rm+bExt*8 >= 8 { + return Operand{}, 0, fmt.Errorf("invalid source register") + } + return Operand{Kind: OpReg, Reg: register}, 1, nil + } + + mem := MemRef{Segment: segment} + consumed := 1 + readDisp8 := func() error { + if len(code) < consumed+1 { + return fmt.Errorf("truncated disp8") + } + mem.Off = int64(int8(code[consumed])) * int64(disp8Scale) + consumed++ + return nil + } + readDisp32 := func() error { + if len(code) < consumed+4 { + return fmt.Errorf("truncated disp32") + } + mem.Off = int64(int32(binary.LittleEndian.Uint32(code[consumed : consumed+4]))) + consumed += 4 + return nil + } + + if rm == 4 { + if len(code) < consumed+1 { + return Operand{}, 0, fmt.Errorf("truncated SIB byte") + } + sib := code[consumed] + consumed++ + indexNumber := int(sib>>3&7) + xExt*8 + if int(sib>>3&7) != 4 || xExt != 0 { + index, ok := decodedX86GeneralRegister(indexNumber) + if !ok || mode == 32 && indexNumber >= 8 { + return Operand{}, 0, fmt.Errorf("invalid SIB index register") + } + mem.Index = index + mem.Scale = int64(1 << (sib >> 6)) + } + baseBits := int(sib & 7) + if mod == 0 && baseBits == 5 { + if err := readDisp32(); err != nil { + return Operand{}, 0, err + } + } else { + baseNumber := baseBits + bExt*8 + base, ok := decodedX86GeneralRegister(baseNumber) + if !ok || mode == 32 && baseNumber >= 8 { + return Operand{}, 0, fmt.Errorf("invalid SIB base register") + } + mem.Base = base + } + } else if mod == 0 && rm == 5 { + if mode == 64 { + if !physicalPC { + return Operand{}, 0, fmt.Errorf("RIP-relative source is not source-layout safe") + } + mem.Base = "RIP" + } + if err := readDisp32(); err != nil { + return Operand{}, 0, err + } + } else { + baseNumber := rm + bExt*8 + base, ok := decodedX86GeneralRegister(baseNumber) + if !ok || mode == 32 && baseNumber >= 8 { + return Operand{}, 0, fmt.Errorf("invalid base register") + } + mem.Base = base + } + + switch mod { + case 1: + if err := readDisp8(); err != nil { + return Operand{}, 0, err + } + case 2: + if err := readDisp32(); err != nil { + return Operand{}, 0, err + } + } + return Operand{Kind: OpMem, Mem: mem}, consumed, nil +} + +func decodedX86GeneralRegister(number int) (Reg, bool) { + names := [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + if number < 0 || number >= len(names) { + return "", false + } + return names[number], true +} + +// x86RawRetpolineTarget recognizes the complete amd64 RETPOLINE(reg) byte +// family emitted by the Go runtime. Its local CALL and backward PAUSE loop are +// a speculative-execution barrier; architecturally the sequence replaces the +// temporary return address with reg and returns to it, which is an indirect +// tail jump. Requiring every fixed byte prevents arbitrary PC-relative raw +// control flow from being accepted through this path. +func x86RawRetpolineTarget(code []byte) (Reg, bool) { + if len(code) != 14 || + code[0] != 0xe8 || code[1] != 0x04 || code[2] != 0 || code[3] != 0 || code[4] != 0 || + code[5] != 0xf3 || code[6] != 0x90 || code[7] != 0xeb || code[8] != 0xfc || + (code[9] != 0x48 && code[9] != 0x4c) || code[10] != 0x89 || code[11]&0xc7 != 0x04 || + code[12] != 0x24 || code[13] != 0xc3 { + return "", false + } + register := int(code[11]>>3&7) + int(code[9]>>2&1)*8 + if register == 4 { + return "", false + } + names := [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + return names[register], true +} + +// Go's assembler omits Intel's N for these no-wait x87 encodings. Keep the +// entire control/environment alias family on the existing named lowerers. +var x86RawX87ControlAliases = map[x86asm.Op]Op{ + x86asm.FNCLEX: "FCLEX", + x86asm.FNINIT: "FINIT", + x86asm.FNSAVE: "FSAVE", + x86asm.FNSTCW: "FSTCW", + x86asm.FNSTENV: "FSTENV", + x86asm.FNSTSW: "FSTSW", +} + +func decodedX86GoSyntax(inst x86asm.Inst, encoding []byte) (string, error) { + if err := decodedX86PrefixError(inst); err != nil { + return "", err + } + for _, argument := range inst.Args { + if memory, ok := argument.(x86asm.Mem); ok && memory.Base == x86asm.EIP && inst.Op != x86asm.NOP { + // The pinned decoder leaves PCRel unset for 67-prefixed EIP + // addressing. Its wrapped native instruction address is not a + // numerical GP input or a proved source-group RIP relocation. + return "", fmt.Errorf("%w: raw x86 EIP-relative memory requires its native instruction address", ErrProbeNeedsContext) + } + } + inst = normalizeDecodedX86MemoryDisplacements(inst) + inst = normalizeDecodedX86LEASegments(inst) + for _, prefix := range inst.Prefix { + if !prefix.IsVEX() { + continue + } + // The pinned x/arch decoder recognizes only these VEX operations. + // For other VEX bytes it can incorrectly return an ordinary legacy + // opcode with a successful status and the wrong instruction length. + // Family-specific decoders above must handle those encodings; never + // let generic fallback turn a missing family into valid but wrong IR. + switch inst.Op { + case x86asm.VMOVDQA, x86asm.VMOVDQU, x86asm.VMOVNTDQ, x86asm.VMOVNTDQA: + default: + return "", fmt.Errorf("unrecognized VEX instruction decoded as legacy %s", inst.Op) + } + break + } + if syntax, recognized, err := decodedX86FarReturn(inst); recognized { + return syntax, err + } + syntax := x86asm.GoSyntax(inst, 0, nil) + replaceOp := func(op string) { + // A repeater is a separate pseudo-instruction; an actual segment + // prefix can precede the mnemonic. Never replace either prefix + // with an alias and move the real opcode into its operand list. + start := strings.LastIndexByte(syntax, ';') + 1 + for start < len(syntax) && syntax[start] == ' ' { + start++ + } + for _, prefix := range inst.Prefix { + switch prefix & 0xff { + case x86asm.PrefixES, x86asm.PrefixCS, x86asm.PrefixSS, x86asm.PrefixDS, x86asm.PrefixFS, x86asm.PrefixGS: + if strings.HasPrefix(syntax[start:], prefix.String()+" ") { + start += len(prefix.String()) + 1 + } + } + } + end := strings.IndexByte(syntax[start:], ' ') + if end < 0 { + end = len(syntax) + } else { + end += start + } + syntax = syntax[:start] + op + syntax[end:] + } + if op, ok := decodedX86CacheLineWriteback(encoding); ok { + replaceOp(op) + return syntax, nil + } + if op, ok := x86RawX87ControlAliases[inst.Op]; ok { + // The Go memory mnemonics encode the 32-bit environment layout. + // Mapping a 66-prefixed 14/94-byte legacy layout to them would change + // the memory footprint. Keep that unsupported format explicit. + if inst.DataSize == 16 && (inst.Op == x86asm.FNSAVE || inst.Op == x86asm.FNSTENV) { + return "", fmt.Errorf("raw %s uses a 16-bit environment layout not represented by Go %s", inst.Op, op) + } + replaceOp(string(op)) + return syntax, nil + } + if op, recognized, err := decodedX86X87EnvironmentLoad(inst, encoding); recognized { + if err != nil { + return "", err + } + replaceOp(string(op)) + return syntax, nil + } + + switch inst.Op { + case x86asm.RDFSBASE, x86asm.RDGSBASE, x86asm.WRFSBASE, x86asm.WRGSBASE: + // x/arch prints widthless Intel names. Reuse the complete named + // FSGSBASE grammar, selecting L/Q from the actual GPR: an ignored + // 66 prefix can set DataSize=16 while the operand remains 32-bit. + register, ok := inst.Args[0].(x86asm.Reg) + bits := decodedX86RegisterBits(register) + if !ok || inst.Mode != 64 || (bits != 32 && bits != 64) { + return "", fmt.Errorf("raw %s requires a 32/64-bit GPR in 64-bit mode", inst.Op) + } + for _, extra := range inst.Args[1:] { + if extra != nil { + return "", fmt.Errorf("raw %s requires exactly one register", inst.Op) + } + } + for _, prefix := range inst.Prefix { + if prefix&0xff == x86asm.PrefixLOCK || prefix&x86asm.PrefixInvalid != 0 { + return "", fmt.Errorf("raw %s has an invalid prefix %s", inst.Op, prefix) + } + } + // Segment/address/operand overrides do not affect this register-only + // family. Clear them only after rejecting invalid (not ignored) ones. + inst.Prefix = x86asm.Prefixes{} + syntax = x86asm.GoSyntax(inst, 0, nil) + width := "L" + if bits == 64 { + width = "Q" + } + replaceOp(inst.Op.String() + width) + case x86asm.INC, x86asm.DEC: + bits := 0 + switch destination := inst.Args[0].(type) { + case x86asm.Reg: + bits = decodedX86RegisterBits(destination) + case x86asm.Mem: + bits = inst.MemBytes * 8 + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("raw %s has unsupported destination width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + case x86asm.SLDT, x86asm.STR, x86asm.SMSW: + // x/arch prints the unsuffixed Intel names, while Go's descriptor + // grammar selects the destination width in the mnemonic. Memory + // destinations are always the architectural 16-bit selector/status. + bits := 0 + switch destination := inst.Args[0].(type) { + case x86asm.Mem: + bits = 16 + case x86asm.Reg: + bits = decodedX86RegisterBits(destination) + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("raw %s has unsupported destination width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + case x86asm.PSHUFW: + // Physical imm8 bits have no signedness, but Go's complete ymshuf + // source table requires Yi8. Render that signed representative in + // both modes; the 32-bit generic printer otherwise masks it unsigned. + selector, ok := inst.Args[2].(x86asm.Imm) + if !ok || selector < 0 || selector > 255 || inst.Args[3] != nil { + return "", fmt.Errorf("raw PSHUFW expected an 8-bit selector") + } + firstSpace := strings.IndexByte(syntax, ' ') + firstComma := strings.IndexByte(syntax, ',') + if firstSpace < 0 || firstComma < firstSpace { + return "", fmt.Errorf("raw PSHUFW has malformed operands") + } + syntax = syntax[:firstSpace+1] + fmt.Sprintf("$%d", int8(selector)) + syntax[firstComma:] + case x86asm.SHLD, x86asm.SHRD: + // Go encodes double shifts with the three-operand SHL/SHR form, + // whose immediate class is signed Yi8 (single shifts use Yu8). + if count, ok := inst.Args[2].(x86asm.Imm); ok { + firstSpace := strings.IndexByte(syntax, ' ') + firstComma := strings.IndexByte(syntax, ',') + if firstSpace < 0 || firstComma < firstSpace { + return "", fmt.Errorf("raw %s has malformed double-shift operands", inst.Op) + } + syntax = syntax[:firstSpace+1] + fmt.Sprintf("$%d", int8(count)) + syntax[firstComma:] + } + case x86asm.CMPPD, x86asm.CMPPS, x86asm.CMPSD_XMM, x86asm.CMPSS: + predicate, ok := inst.Args[2].(x86asm.Imm) + if !ok || predicate < 0 || predicate > 255 || inst.Args[3] != nil { + return "", fmt.Errorf("raw %s expected an unsigned 8-bit predicate", inst.Op) + } + // Go's legacy yxcmpi table places the signed predicate last, unlike + // both Intel syntax and the vector VCMP family. Reorder typed args + // before rendering, preserving register and segment/address details. + inst.Args[0], inst.Args[1], inst.Args[2] = x86asm.Imm(int8(predicate)), inst.Args[0], inst.Args[1] + syntax = x86asm.GoSyntax(inst, 0, nil) + // The 32-bit printer masks negative immediates to uint32. The Go + // encoder requires a signed byte, not that unsigned spelling. + syntax = syntax[:strings.LastIndexByte(syntax, ',')+1] + fmt.Sprintf(" $%d", int8(predicate)) + if inst.Op == x86asm.CMPSD_XMM { + replaceOp("CMPSD") + } + return syntax, nil + case x86asm.NOP: + // The decoder prints NOPL/NOPW according to the encoded data size. + // The address operand of a multi-byte x86 NOP is not read, so the + // semantic instruction is the operand-free Plan 9 NOP. + return "NOP", nil + case x86asm.MOVSXD: + return decodedX86MOVSXD(inst, encoding) + case x86asm.MOVD: + // Intel MOVD transfers 32 bits. Go's MOVD spelling instead aliases + // AMOVQ and transfers 64 bits for XMM/memory forms. + replaceOp("MOVL") + case x86asm.MOVDQA: + replaceOp("MOVO") + case x86asm.MOVDQU: + replaceOp("MOVOU") + case x86asm.MOVNTDQ: + // Go's assembler calls the legacy 128-bit non-temporal store MOVNTO. + // MOVNTDQ is reserved for the VEX/EVEX VMOVNTDQ spelling. + replaceOp("MOVNTO") + case x86asm.MOVSD_XMM: + replaceOp("MOVSD") + case x86asm.CVTDQ2PS, x86asm.CVTPI2PS: + // Go's yxcvm2 table shares CVTPL2PS across SSE X and MMX M sources. + replaceOp("CVTPL2PS") + case x86asm.CVTPS2DQ, x86asm.CVTPS2PI: + // Go's yxcvm1 table likewise names both destinations CVTPS2PL. + replaceOp("CVTPS2PL") + case x86asm.CVTTPS2DQ, x86asm.CVTTPS2PI: + replaceOp("CVTTPS2PL") + case x86asm.CVTDQ2PD, x86asm.CVTPI2PD: + replaceOp("CVTPL2PD") + case x86asm.CVTPD2DQ, x86asm.CVTPD2PI: + replaceOp("CVTPD2PL") + case x86asm.CVTTPD2DQ, x86asm.CVTTPD2PI: + replaceOp("CVTTPD2PL") + case x86asm.MUL, x86asm.IMUL, x86asm.DIV, x86asm.IDIV: + argumentCount := 0 + for _, argument := range inst.Args { + if argument != nil { + argumentCount++ + } + } + if argumentCount == 1 { + bits := inst.MemBytes * 8 + if source, ok := inst.Args[0].(x86asm.Reg); ok { + bits = decodedX86RegisterBits(source) + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("%s has unsupported source width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + } else if inst.Op == x86asm.IMUL && argumentCount == 3 { + destination, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("three-operand IMUL destination is not a register") + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[decodedX86RegisterBits(destination)] + if width == "" { + return "", fmt.Errorf("three-operand IMUL has unsupported destination width %d", decodedX86RegisterBits(destination)) + } + replaceOp("IMUL3" + width) + } + case x86asm.BSWAP: + destination, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("BSWAP destination is not a register") + } + width := map[int]string{32: "L", 64: "Q"}[decodedX86RegisterBits(destination)] + if width == "" { + return "", fmt.Errorf("BSWAP has unsupported destination width %d", decodedX86RegisterBits(destination)) + } + replaceOp("BSWAP" + width) + case x86asm.MOVBE: + bits := inst.MemBytes * 8 + for _, operand := range inst.Args { + if register, ok := operand.(x86asm.Reg); ok { + bits = decodedX86RegisterBits(register) + break + } + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("MOVBE has unsupported operand width %d", bits) + } + replaceOp("MOVBE" + width) + case x86asm.LZCNT, x86asm.TZCNT: + // Go names each bit-count width explicitly. x/arch prints the + // widthless Intel mnemonic, even for 16- and 64-bit encodings. + destination, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("%s destination is not a register", inst.Op) + } + bits := decodedX86RegisterBits(destination) + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("%s has unsupported destination width %d", inst.Op, bits) + } + replaceOp(inst.Op.String() + width) + case x86asm.CVTSI2SS, x86asm.CVTSI2SD: + sourceBits := inst.MemBytes * 8 + if source, ok := inst.Args[1].(x86asm.Reg); ok { + sourceBits = decodedX86RegisterBits(source) + } + sourceWidth := map[int]string{32: "L", 64: "Q"}[sourceBits] + if sourceWidth == "" { + return "", fmt.Errorf("%s has unsupported decoded source width %d", inst.Op, sourceBits) + } + resultWidth := "SS" + if inst.Op == x86asm.CVTSI2SD { + resultWidth = "SD" + } + replaceOp("CVTS" + sourceWidth + "2" + resultWidth) + case x86asm.CVTSS2SI, x86asm.CVTSD2SI, x86asm.CVTTSS2SI, x86asm.CVTTSD2SI: + // x/arch prints Intel's SI destination stem (and, in some memory + // cases, derives the wrong Plan 9 width). Go's assembler names this + // legacy family CVT{T}{SS,SD}2S{L,Q}. Recover the destination width + // from REX.W, which is the architectural selector for all four ops. + outputWidth := "L" + if inst.Mode == 64 { + for _, prefix := range inst.Prefix { + if prefix.IsREX() && prefix&x86asm.PrefixREXW != 0 { + outputWidth = "Q" + break + } + } + } + name := map[x86asm.Op]string{ + x86asm.CVTSS2SI: "CVTSS2S", + x86asm.CVTSD2SI: "CVTSD2S", + x86asm.CVTTSS2SI: "CVTTSS2S", + x86asm.CVTTSD2SI: "CVTTSD2S", + }[inst.Op] + replaceOp(name + outputWidth) + case x86asm.PACKSSDW: + replaceOp("PACKSSLW") + case x86asm.PCMPEQD: + replaceOp("PCMPEQL") + case x86asm.PCMPGTD: + replaceOp("PCMPGTL") + case x86asm.PMADDWD: + replaceOp("PMADDWL") + case x86asm.PMULUDQ: + replaceOp("PMULULQ") + case x86asm.PSLLD: + replaceOp("PSLLL") + case x86asm.PSRAD: + replaceOp("PSRAL") + case x86asm.PSRLD: + replaceOp("PSRLL") + case x86asm.PSUBD: + replaceOp("PSUBL") + case x86asm.PUNPCKHDQ: + replaceOp("PUNPCKHLQ") + case x86asm.PUNPCKHWD: + replaceOp("PUNPCKHWL") + case x86asm.PUNPCKLDQ: + replaceOp("PUNPCKLLQ") + case x86asm.PUNPCKLWD: + replaceOp("PUNPCKLWL") + case x86asm.CRC32: + bits := inst.MemBytes * 8 + if source, ok := inst.Args[1].(x86asm.Reg); ok { + bits = decodedX86RegisterBits(source) + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[bits] + if width == "" { + return "", fmt.Errorf("CRC32 has unsupported decoded source width %d", bits) + } + replaceOp("CRC32" + width) + case x86asm.MOVSX, x86asm.MOVZX: + dst, ok := inst.Args[0].(x86asm.Reg) + if !ok { + return "", fmt.Errorf("%s destination is not a register", inst.Op) + } + destinationBits := decodedX86RegisterBits(dst) + sourceBits := inst.MemBytes * 8 + if source, ok := inst.Args[1].(x86asm.Reg); ok { + sourceBits = decodedX86RegisterBits(source) + } + widthLetter := func(bits int) string { + switch bits { + case 8: + return "B" + case 16: + return "W" + case 32: + return "L" + case 64: + return "Q" + default: + return "" + } + } + sourceLetter := widthLetter(sourceBits) + destinationLetter := widthLetter(destinationBits) + if sourceLetter == "" || destinationLetter == "" { + return "", fmt.Errorf("%s has unsupported decoded widths %d -> %d", inst.Op, sourceBits, destinationBits) + } + extension := "SX" + if inst.Op == x86asm.MOVZX { + extension = "ZX" + } + if sourceBits == 16 && destinationBits == 16 { + name := "MOVSWW" + if inst.Op == x86asm.MOVZX { + name = "MOVZWW" + } + replaceOp(name) + } else { + replaceOp("MOV" + sourceLetter + destinationLetter + extension) + } + } + normalizeDecodedX86GoOpcode(inst, &syntax, replaceOp) + syntax = normalizeDecodedX86SegmentMemorySyntax(syntax) + return syntax, nil +} + +// The pinned x/arch decoder retains ModR/M/SIB disp32 as an unsigned +// magnitude. Rendering that value as a named Go address changes 64-bit +// arithmetic and violates Go's signed-displacement checks. Preserve the +// physical field's sign before every opcode-specific rendering path. +// MOV A0-A3 instead contains a full absolute moffs address: it must never +// be narrowed or sign-extended as though it were a ModR/M displacement. +func normalizeDecodedX86MemoryDisplacements(inst x86asm.Inst) x86asm.Inst { + if inst.AddrSize != 32 && inst.AddrSize != 64 { + return inst + } + if opcode := byte(inst.Opcode >> 24); opcode >= 0xa0 && opcode <= 0xa3 { + return inst + } + for index, argument := range inst.Args { + if memory, ok := argument.(x86asm.Mem); ok { + memory.Disp = int64(int32(memory.Disp)) + inst.Args[index] = memory + } + } + return inst +} + +// LEA computes only the effective offset. All six segment override prefixes +// are architecturally irrelevant to it, unlike an actual memory load/store. +func normalizeDecodedX86LEASegments(inst x86asm.Inst) x86asm.Inst { + if inst.Op != x86asm.LEA { + return inst + } + for index, argument := range inst.Args { + if memory, ok := argument.(x86asm.Mem); ok { + memory.Segment = 0 + inst.Args[index] = memory + } + } + var prefixes x86asm.Prefixes + next := 0 + for _, prefix := range inst.Prefix { + if prefix == 0 { + break + } + switch byte(prefix) { + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65: + continue + } + prefixes[next] = prefix + next++ + } + inst.Prefix = prefixes + return inst +} + +// x/arch prints a raw segment override as GS:0x30 or GS:0x30(AX). The +// Plan 9 parser represents those same addresses as 48(GS) and 48(AX)(GS). +// Rewrite only FS/GS operands; other segment and RIP-relative forms retain +// their original spelling so unsupported address semantics fail closed. +func normalizeDecodedX86SegmentMemorySyntax(syntax string) string { + space := strings.IndexByte(syntax, ' ') + if space < 0 { + return syntax + } + args := splitTopLevelCSV(syntax[space+1:]) + for i, arg := range args { + arg = strings.TrimSpace(arg) + segment := "" + switch { + case strings.HasPrefix(arg, "FS:"): + segment = "FS" + case strings.HasPrefix(arg, "GS:"): + segment = "GS" + default: + continue + } + address := strings.TrimPrefix(arg, segment+":") + open := strings.IndexByte(address, '(') + offset := address + base := "" + if open >= 0 { + offset = address[:open] + base = address[open:] + } + if offset == "" { + offset = "0" + } + value, err := strconv.ParseInt(offset, 0, 64) + if err != nil || strings.Contains(base, "(IP)") || strings.Contains(base, "(RIP)") { + continue + } + args[i] = fmt.Sprintf("%d%s(%s)", value, base, segment) + } + return syntax[:space+1] + strings.Join(args, ", ") +} + +// normalizeDecodedX86GoOpcode bridges the architectural names emitted by +// x/arch's decoder and the semantic names/classes used by Go 1.27's +// cmd/internal/obj/x86 tables. Go's assembler intentionally uses aliases such +// as SETOS and CMOVQLE, while x/arch prints Intel names such as SETO and +// CMOVLE. x/arch also prints the byte-immediate ADD opcode as ADDL even when +// its destination is an 8-bit register; that spelling is rejected by Go's +// Yml/Ymb operand classes and must be corrected from the decoded operands. +func normalizeDecodedX86GoOpcode(inst x86asm.Inst, syntax *string, replaceOp func(string)) { + op := strings.ToUpper(inst.Op.String()) + setAliases := map[string]string{ + "SETA": "SETHI", "SETAE": "SETCC", "SETB": "SETCS", "SETBE": "SETLS", + "SETE": "SETEQ", "SETG": "SETGT", "SETGE": "SETGE", "SETL": "SETLT", + "SETLE": "SETLE", "SETNE": "SETNE", "SETNO": "SETOC", "SETNP": "SETPC", + "SETNS": "SETPL", "SETO": "SETOS", "SETP": "SETPS", "SETS": "SETMI", + } + if alias, ok := setAliases[op]; ok { + replaceOp(alias) + // GoSyntax deliberately omits the B suffix for extended byte registers + // (for example it prints SETO R9). Reconstruct the decoded register's + // actual byte spelling before reparsing the instruction. + if len(inst.Args) > 0 { + if reg, ok := inst.Args[0].(x86asm.Reg); ok { + *syntax = alias + " " + reg.String() + } + } + return + } + + cmovConditions := map[string]string{ + "CMOVA": "HI", "CMOVAE": "CC", "CMOVB": "CS", "CMOVBE": "LS", + "CMOVE": "EQ", "CMOVG": "GT", "CMOVGE": "GE", "CMOVL": "LT", + "CMOVLE": "LE", "CMOVNE": "NE", "CMOVNO": "OC", "CMOVNP": "PC", + "CMOVNS": "PL", "CMOVO": "OS", "CMOVP": "PS", "CMOVS": "MI", + } + if condition, ok := cmovConditions[op]; ok { + bits := 0 + for i := 0; i < 2 && i < len(inst.Args); i++ { + if reg, ok := inst.Args[i].(x86asm.Reg); ok { + if n := decodedX86RegisterBits(reg); n > bits { + bits = n + } + } + } + if bits == 0 { + bits = inst.MemBytes * 8 + } + width := map[int]string{16: "W", 32: "L", 64: "Q"}[bits] + if width != "" { + replaceOp("CMOV" + width + condition) + } + return + } + + widthOpcode := decodedX86WidthOpcode(op) + if widthOpcode == "" || len(inst.Args) == 0 { + return + } + dstBits := 0 + if reg, ok := inst.Args[0].(x86asm.Reg); ok { + dstBits = decodedX86RegisterBits(reg) + } else if _, ok := inst.Args[0].(x86asm.Mem); ok { + dstBits = inst.MemBytes * 8 + } + width := map[int]string{8: "B", 16: "W", 32: "L", 64: "Q"}[dstBits] + if width != "" { + replaceOp(widthOpcode + width) + } +} + +func decodedX86WidthOpcode(op string) string { + switch op { + case "ADD", "ADC", "SBB", "SUB", "AND", "OR", "XOR", "CMP", "TEST": + return op + case "ROL", "ROR", "RCL", "RCR", "SHL", "SHR", "SAR": + return op + case "SHLD": + return "SHL" + case "SHRD": + return "SHR" + default: + return "" + } +} + +// Go 1.27 names 0F 18 /0-/3 but not the newer /4, /6 and /7 cache +// hints. Decode their complete ModRM memory forms directly; /5 and register +// operands are not instructions. RIP-relative addresses remain rejected by +// decodedX86RMSource because their byte layout cannot be inferred here. +func decodedX86ExtendedPrefetchInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + rex := byte(0) + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + rex = 0 + case 0x65: + segment = GS + rex = 0 + case 0x66: + rex = 0 + case 0x67: + addressOverride = true + rex = 0 + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0x18 { + return Instr{}, 0, false, nil + } + extension := code[i+2] >> 3 & 7 + op := Op("") + switch extension { + case 4: + op = "PREFETCHRST2" + case 6: + op = "PREFETCHIT1" + case 7: + op = "PREFETCHIT0" + default: + return Instr{}, 0, false, nil + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if code[i+2]>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("%s requires a memory operand", op) + } + address, consumed, err := decodedX86RMSource(code[i+2:], mode, int(rex&1), int(rex>>1&1), segment, 1) + if err != nil { + return Instr{}, 0, true, err + } + return Instr{Op: op, Args: []Operand{address}, Raw: fmt.Sprintf("%s %s", op, address.String())}, i + 2 + consumed, true, nil +} + +// decodedX86CacheLineWritebackInstruction recognizes the complete raw +// CLFLUSH/CLFLUSHOPT/CLWB memory family before x/arch's generic decoder. In +// particular, generated assembly in github.com/wencode/hack places a REX +// prefix before the mandatory 0x66 prefix, which x/arch v0.14 rejects even +// though x86 hardware accepts the prefix sequence. +func decodedX86CacheLineWritebackInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + hasDataSizePrefix := false + addressOverride := false + segment := Reg("") + rex := byte(0) + hasREX := false + for i < len(code) { + switch code[i] { + case 0x64: + // A legacy prefix after REX makes that REX byte ineffective. Only + // the REX prefix immediately preceding the opcode escape applies. + rex = 0 + hasREX = false + segment = FS + i++ + case 0x65: + rex = 0 + hasREX = false + segment = GS + i++ + case 0x66: + rex = 0 + hasREX = false + hasDataSizePrefix = true + i++ + case 0x67: + rex = 0 + hasREX = false + addressOverride = true + i++ + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + hasREX = true + i++ + continue + } + goto opcode + } + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0xae { + return Instr{}, 0, false, nil + } + modRM := code[i+2] + extension := modRM >> 3 & 7 + op := Op("") + switch { + case extension == 7 && !hasDataSizePrefix: + op = "CLFLUSH" + case extension == 7 && hasDataSizePrefix: + op = "CLFLUSHOPT" + case extension == 6 && hasDataSizePrefix: + op = "CLWB" + default: + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if hasREX && rex&0x0c != 0 { + return Instr{}, 0, true, fmt.Errorf("REX.W and REX.R are absent from Go 1.27's %s encoding", op) + } + if modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("%s requires a memory operand", op) + } + bExt := int(rex & 1) + xExt := int(rex>>1) & 1 + memory, consumed, decodeErr := decodedX86VEXRMSource(code[i+2:], mode, bExt, xExt, segment) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + instruction = Instr{ + Op: op, + Args: []Operand{memory}, + Raw: fmt.Sprintf("%s %s", op, memory.String()), + } + return instruction, i + 2 + consumed, true, nil +} + +// decodedX86CacheLineWriteback recovers instructions which x/arch v0.14 +// cannot distinguish from their older 0F AE encodings. The explicit 0x66 +// mandatory prefix plus the ModRM extension is the architectural distinction: +// /7 is CLFLUSHOPT and /6 is CLWB. Parse only the actual prefix/opcode region +// so an identical byte sequence inside a displacement cannot be mistaken for +// an opcode. +func decodedX86CacheLineWriteback(encoding []byte) (string, bool) { + hasDataSizePrefix := false + i := 0 + for i < len(encoding) { + switch encoding[i] { + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65, 0x67, 0xf0, 0xf2, 0xf3: + i++ + case 0x66: + hasDataSizePrefix = true + i++ + default: + if encoding[i] >= 0x40 && encoding[i] <= 0x4f { + i++ + continue + } + goto opcode + } + } + +opcode: + if !hasDataSizePrefix || i+2 >= len(encoding) || encoding[i] != 0x0f || encoding[i+1] != 0xae { + return "", false + } + modRM := encoding[i+2] + if modRM>>6 == 3 { + return "", false + } + switch modRM >> 3 & 7 { + case 7: + return "CLFLUSHOPT", true + case 6: + return "CLWB", true + default: + return "", false + } +} + +func decodedX86RegisterBits(reg x86asm.Reg) int { + switch { + case x86asm.AL <= reg && reg <= x86asm.R15B: + return 8 + case x86asm.AX <= reg && reg <= x86asm.R15W: + return 16 + case x86asm.EAX <= reg && reg <= x86asm.R15L: + return 32 + case x86asm.RAX <= reg && reg <= x86asm.R15: + return 64 + default: + return 0 + } +} + +func isX86RawDirective(ins Instr) bool { + switch Op(strings.ToUpper(string(ins.Op))) { + case OpBYTE, OpWORD, "LONG", "QUAD": + return true + default: + return false + } +} + +func x86RawDirectiveBytes(ins Instr) ([]byte, error) { + if len(ins.Args) != 1 || ins.Args[0].Kind != OpImm || ins.Args[0].ImmRaw != "" { + return nil, fmt.Errorf("%s expects exactly one resolved integer constant: %q", ins.Op, ins.Raw) + } + value := uint64(ins.Args[0].Imm) + switch Op(strings.ToUpper(string(ins.Op))) { + case OpBYTE: + return []byte{byte(value)}, nil + case OpWORD: + var buf [2]byte + binary.LittleEndian.PutUint16(buf[:], uint16(value)) + return buf[:], nil + case "LONG": + var buf [4]byte + binary.LittleEndian.PutUint32(buf[:], uint32(value)) + return buf[:], nil + case "QUAD": + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], value) + return buf[:], nil + default: + return nil, fmt.Errorf("not an x86 raw directive: %q", ins.Raw) + } +} + +func parseDecodedX86Instruction(syntax string) ([]Instr, error) { + file, err := Parse(ArchAMD64, "TEXT decoded(SB), NOSPLIT, $0-0\n"+syntax+"\n") + if err != nil { + return nil, err + } + if len(file.Funcs) != 1 || len(file.Funcs[0].Instrs) < 2 { + return nil, fmt.Errorf("decoder syntax produced no instruction") + } + return append([]Instr(nil), file.Funcs[0].Instrs[1:]...), nil +} diff --git a/amd64_raw_abs.go b/amd64_raw_abs.go new file mode 100644 index 000000000..aeb0f71c5 --- /dev/null +++ b/amd64_raw_abs.go @@ -0,0 +1,79 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawPackedAbsInstruction implements Go's complete VPABS B/W/D/Q +// rows: VEX X/Y for B/W/D, EVEX X/Y/Z with optional K masking, and EVEX +// D/Q memory broadcast. VPABSQ is EVEX-only; W is fixed by the lane width. +func decodedX86RawPackedAbsInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.pp != 1 || p.opcode < 0x1c || p.opcode > 0x1f { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VPABS: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.evex && !p.fixed || p.vectorLength > 2 || p.upper != 0 { + return fail("invalid vector prefix or reserved vvvv bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + laneBytes := [...]int{1, 2, 4, 8}[p.opcode-0x1c] + if p.w != (laneBytes == 8) || !p.evex && (laneBytes == 8 || p.vectorLength == 2) { + return fail("invalid W bit or vector length for opcode") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + registerSource := code[p.modRM]>>6 == 3 + if p.broadcast && (!p.evex || laneBytes < 4 || registerSource) { + return fail("broadcast requires EVEX D/Q memory source") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + scale := 16 << p.vectorLength + if p.broadcast { + scale = laneBytes + } + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (destinationNumber >= 8 || registerSource && (p.b != 0 || p.x != 0)) { + return fail("extended register in 32-bit mode") + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))}) + op := Op("VPABS" + [...]string{"B", "W", "D", "Q"}[p.opcode-0x1c]) + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_abs_test.go b/amd64_raw_abs_test.go new file mode 100644 index 000000000..7a3c233cb --- /dev/null +++ b/amd64_raw_abs_test.go @@ -0,0 +1,235 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawPackedAbsForms = []struct { + opcode byte + op Op + lane int + evex bool +}{ + {0x1c, "VPABSB", 1, false}, + {0x1d, "VPABSW", 2, false}, + {0x1e, "VPABSD", 4, false}, + {0x1f, "VPABSQ", 8, true}, +} + +func encodeX86RawPackedAbs(formIndex, width, mask, destination, source int, zeroing, broadcast bool) []byte { + form := x86RawPackedAbsForms[formIndex] + vectorBits := map[int]int{16: 0, 32: 1, 64: 2}[width] + var code []byte + if width == 64 || mask != 0 || zeroing || broadcast || form.evex { + code = encodeX86EVEXPackedWordMultiply(2, form.opcode, vectorBits, mask, form.evex, zeroing, destination, 0, source) + if broadcast { + code[3] |= 0x10 + } + } else { + code = encodeX86VEXPackedWordMultiply(2, form.opcode, width == 32, false, destination, 0, source) + } + return code +} + +func TestDecodedX86RawPackedAbsCompleteFamily(t *testing.T) { + for formIndex, form := range x86RawPackedAbsForms { + for _, width := range []int{16, 32, 64} { + for _, masked := range []bool{false, true} { + for _, zeroing := range []bool{false, true} { + if !masked && zeroing { + continue + } + for _, source := range []int{0, 7, 16, 31} { + if !masked && width != 64 && !form.evex && source >= 16 { + continue + } + mask := 0 + if masked { + mask = 3 + } + name := fmt.Sprintf("%s/%d/k%d/z%t/src%d", form.op, width, mask, zeroing, source) + t.Run(name, func(t *testing.T) { + code := encodeX86RawPackedAbs(formIndex, width, mask, 19, source, zeroing, false) + if width != 64 && !masked && !form.evex { + code = encodeX86RawPackedAbs(formIndex, width, mask, 11, source, zeroing, false) + } + got, length, ok, err := decodedX86RawPackedAbsInstruction(code, 64) + if err != nil || !ok || length != len(code) || !strings.HasPrefix(string(got.Op), string(form.op)) || + got.Args[0].String() != fmt.Sprintf("%s%d", map[int]string{16: "X", 32: "Y", 64: "Z"}[width], source) { + t.Fatalf("decode %x = %+v length=%d ok=%t err=%v", code, got, length, ok, err) + } + }) + } + } + } + } + } + // Actual bytes from simd@v1.21.1 arith_avx2_amd64.s. + real := []byte{0xc4, 0xa2, 0x7d, 0x1e, 0x04, 0x86} + got, length, ok, err := decodedX86RawPackedAbsInstruction(real, 64) + if err != nil || !ok || length != len(real) || got.Op != "VPABSD" || + got.Args[0].String() != "0(SI)(R8*4)" || got.Args[1].String() != "Y0" { + t.Fatalf("decode real VPABSD %x = %+v length=%d ok=%t err=%v", real, got, length, ok, err) + } +} + +func TestDecodedX86RawPackedAbsRejectsInvalidForms(t *testing.T) { + valid := encodeX86RawPackedAbs(2, 32, 0, 1, 2, false, false) + for name, edit := range map[string]func([]byte) []byte{ + "reserved vvvv": func(b []byte) []byte { b[2] &^= 0x08; return b }, + "W1": func(b []byte) []byte { b[2] |= 0x80; return b }, + "truncated": func(b []byte) []byte { return b[:4] }, + "address override": func(b []byte) []byte { return append([]byte{0x67}, b...) }, + } { + t.Run(name, func(t *testing.T) { + bad := edit(append([]byte(nil), valid...)) + if _, _, matched, err := decodedX86RawPackedAbsInstruction(bad, 64); !matched || err == nil { + t.Fatalf("accepted invalid packed absolute %x: matched=%t err=%v", bad, matched, err) + } + }) + } +} + +func TestTranslateX86RawPackedAbsObjects(t *testing.T) { + const source = `TEXT rawPackedAbs(SB),$0-0 + BYTE $0xc4 + BYTE $0xa2 + BYTE $0x7d + BYTE $0x1e + BYTE $0x04 + BYTE $0x86 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawPackedAbs": {Name: "rawPackedAbs", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-packed-abs.ll", "raw-packed-abs.o", ir) + }) + } +} + +func x86RawPackedAbsConstant(formIndex, width int, broadcast bool) []byte { + form := x86RawPackedAbsForms[formIndex] + code := encodeX86RawPackedAbs(formIndex, width, 0, 0, 0, false, broadcast) + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = form.lane + } + for i := 0; i < dataWidth; i++ { + code = append(code, byte(i*23+5)) + } + return code +} + +func TestDecodeX86RawPackedAbsConstantCompleteFamily(t *testing.T) { + for formIndex, form := range x86RawPackedAbsForms { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + if broadcast && form.lane < 4 { + continue + } + name := fmt.Sprintf("%s/%d/bcst%t", form.op, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedAbsConstant(formIndex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := form.op + if broadcast { + want += ".BCST" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } +} + +func TestDecodeX86RawPackedAbsConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawPackedAbsConstant(2, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedAbsConstantObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, form := range []struct { + formIndex, width int + broadcast bool + }{ + {0, 32, false}, {1, 64, false}, {2, 64, true}, {3, 64, true}, + } { + code := x86RawPackedAbsConstant(form.formIndex, form.width, form.broadcast) + name := fmt.Sprintf("rawPackedAbsConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-packed-abs-constant.ll", "raw-packed-abs-constant.o", ir) + }) + } +} diff --git a/amd64_raw_address_override_test.go b/amd64_raw_address_override_test.go new file mode 100644 index 000000000..f479a0001 --- /dev/null +++ b/amd64_raw_address_override_test.go @@ -0,0 +1,701 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawAddressOverrideDoesNotBorrowCallerPointer(t *testing.T) { + for _, access := range []struct { + name, old, replacement string + }{ + {"load", "MOVL $7,AX", "BYTE $0x67\nBYTE $0x8b\nBYTE $0x07"}, + {"store", "MOVL AX,(DX)", "BYTE $0x67\nBYTE $0x89\nBYTE $0x02"}, + } { + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(access.name+"/"+triple, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, access.old, access.replacement, 1) + // These are real stock-Go objects, not executions of a program + // that could dereference a truncated incoming address. + x86LEAActualTargetGoObject(t, "amd64", triple, assembly) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("physical EA32 %s borrowed a full-width caller pointer: %v", access.name, err) + } + }) + } + } +} + +func x86AddressOverrideDecoded(t *testing.T, mode int, code []byte) []Instr { + t.Helper() + instructions, err := x86AddressOverrideDecode(mode, code) + if err != nil { + t.Fatal(err) + } + return instructions +} + +func x86AddressOverrideDecode(mode int, code []byte) ([]Instr, error) { + arch := "amd64" + if mode == 32 { + arch = "386" + } + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+x86RawDisplacementDirectives(code)+"RET\n") + if err != nil { + return nil, err + } + function, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + return nil, err + } + return function.Instrs, nil +} + +func x86AddressOverrideNamedBytes(t *testing.T, arch, instruction string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, arch, "TEXT raw(SB),4,$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatalf("actual named implicit opcode: %v\n%s", err, listing) + } + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != "0x0000" || len(fields[1]) != 2 { + continue + } + var code []byte + for _, field := range fields[1:] { + if len(field) != 2 { + break + } + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + t.Fatalf("actual named opcode byte: %v\n%s", err, listing) + } + code = append(code, byte(value)) + } + return code + } + t.Fatalf("actual named opcode lacks its byte listing:\n%s", listing) + return nil +} + +func TestX86RawAddressOverrideGenericMemoryAndImplicitStrings(t *testing.T) { + // Cover the generic decoder's explicit read/write/RMW, scalar/vector and + // implicit memory shapes. Specialized prefix parsers are a separate scope. + forms := []struct { + name string + code []byte + }{ + {"byte-load", []byte{0x8a, 0x07}}, {"byte-store", []byte{0x88, 0x07}}, + {"word-load", []byte{0x66, 0x8b, 0x07}}, {"word-store", []byte{0x66, 0x89, 0x07}}, + {"long-load", []byte{0x8b, 0x07}}, {"long-store", []byte{0x89, 0x07}}, + {"add-read", []byte{0x03, 0x07}}, {"add-rmw", []byte{0x01, 0x07}}, + {"compare", []byte{0x3b, 0x07}}, {"test", []byte{0x85, 0x07}}, + {"exchange", []byte{0x87, 0x07}}, {"increment", []byte{0xff, 0x07}}, + {"shift-rmw", []byte{0xc1, 0x27, 0x01}}, + {"movups-load", []byte{0x0f, 0x10, 0x07}}, {"movups-store", []byte{0x0f, 0x11, 0x07}}, + {"addps", []byte{0x0f, 0x58, 0x07}}, + {"movsb", []byte{0xa4}}, {"cmpsb", []byte{0xa6}}, + {"stosb", []byte{0xaa}}, {"lodsb", []byte{0xac}}, {"scasb", []byte{0xae}}, + {"rep-movsb", []byte{0xf3, 0xa4}}, {"repne-scasb", []byte{0xf2, 0xae}}, + {"insb", []byte{0x6c}}, {"outsb", []byte{0x6e}}, {"xlat", []byte{0xd7}}, + {"insw", []byte{0x66, 0x6d}}, {"insl", []byte{0x6d}}, + {"outsw", []byte{0x66, 0x6f}}, {"outsl", []byte{0x6f}}, + {"movsw", []byte{0x66, 0xa5}}, {"movsl", []byte{0xa5}}, + {"cmpsw", []byte{0x66, 0xa7}}, {"cmpsl", []byte{0xa7}}, + {"stosw", []byte{0x66, 0xab}}, {"stosl", []byte{0xab}}, + {"lodsw", []byte{0x66, 0xad}}, {"lodsl", []byte{0xad}}, + {"scasw", []byte{0x66, 0xaf}}, {"scasl", []byte{0xaf}}, + } + for _, mode := range []int{32, 64} { + modeForms := append([]struct { + name string + code []byte + }(nil), forms...) + if mode == 64 { + modeForms = append(modeForms, struct { + name string + code []byte + }{"quad-load", []byte{0x48, 0x8b, 0x07}}, struct { + name string + code []byte + }{"quad-store", []byte{0x48, 0x89, 0x07}}) + for _, form := range []struct { + name string + code []byte + }{ + {"movsq", []byte{0x48, 0xa5}}, {"cmpsq", []byte{0x48, 0xa7}}, + {"stosq", []byte{0x48, 0xab}}, {"lodsq", []byte{0x48, 0xad}}, + {"scasq", []byte{0x48, 0xaf}}, + } { + modeForms = append(modeForms, form) + } + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + var complete []byte + for _, form := range modeForms { + for _, segment := range []byte{0, 0x64, 0x65} { + complete = append(complete, 0x67) + if segment != 0 { + complete = append(complete, segment) + } + complete = append(complete, form.code...) + } + } + x86HandoffOriginalBytes(t, arch, append(complete, 0xc3)) + for _, form := range modeForms { + for _, segment := range []byte{0, 0x64, 0x65} { + t.Run(fmt.Sprintf("mode%d/%s/segment%02x", mode, form.name, segment), func(t *testing.T) { + code := []byte{0x67} + if segment != 0 { + code = append(code, segment) + } + code = append(code, form.code...) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) || physical.AddrSize != mode/2 { + t.Fatalf("actual decoder input: %+v, %v", physical, err) + } + instructions, err := x86AddressOverrideDecode(mode, code) + if strings.Contains(form.name, "stos") || strings.Contains(form.name, "scas") || strings.HasPrefix(form.name, "ins") { + if segment != 0 { + // The decoded destination remains ES, but x/arch + // leaves this unused FS/GS prefix unmarked. Require + // explicit Context, not a GoSyntax parser accident. + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "unbound physical") || instructions != nil { + t.Fatalf("unbound segment did not retain Context: %v", err) + } + return + } + } + if err != nil { + t.Fatal(err) + } + marked := 0 + for _, instruction := range instructions { + if instruction.Op == OpTEXT || instruction.Op == OpRET { + continue + } + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("generic physical memory override lost its restriction: %+v", instruction) + } + marked++ + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + if mode == 32 { + ctx.goarch = "386" + } + before := output.String() + if terminated, err := ctx.lowerInstr(0, 0, instruction, nil, nil); terminated || !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("unbound memory emitted IR: terminated=%t, error=%v", terminated, err) + } + } + if marked == 0 { + t.Fatal("lost complete physical memory instruction") + } + }) + } + } + } +} + +func TestX86RawAddressOverrideAllAddressShapesAndMoffs(t *testing.T) { + for _, mode := range []int{32, 64} { + shapes := [][]byte{} + if mode == 32 { + // All eight EA16 r/m forms, including disp16-only and BP+disp8. + for rm := byte(0); rm < 8; rm++ { + shape := []byte{rm} + if rm == 6 { + shape = append(shape, 0xc0, 0xff) + } + shapes = append(shapes, shape, []byte{0x40 | rm, 0xc0}, []byte{0x80 | rm, 0xc0, 0xff}) + } + } else { + // All base registers and SIB scales; EIP-relative r/m=5 is + // deliberately covered by the existing closed EIP boundary. + for rm := byte(0); rm < 8; rm++ { + if rm != 4 && rm != 5 { + shapes = append(shapes, []byte{rm}) + } + if rm != 4 { + shapes = append(shapes, []byte{0x40 | rm, 0xc0}, []byte{0x80 | rm, 0xc0, 0xff, 0xff, 0xff}) + } + } + for scale := byte(0); scale < 4; scale++ { + shapes = append(shapes, + []byte{0x04, scale<<6 | 0x18}, + []byte{0x04, scale<<6 | 0x1d, 0xc0, 0xff, 0xff, 0xff}) + } + shapes = append(shapes, []byte{0x04, 0x20}, []byte{0x04, 0x25, 0xc0, 0xff, 0xff, 0xff}) + } + for _, opcode := range []byte{0x8b, 0x89} { + for index, shape := range shapes { + t.Run(fmt.Sprintf("mode%d/op%02x/shape%d", mode, opcode, index), func(t *testing.T) { + code := append([]byte{0x67, opcode}, shape...) + found := false + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.Op == OpMOVL { + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("physical memory shape lost override: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost actual ModR/M address shape") + } + }) + } + } + for _, opcode := range []byte{0xa0, 0xa1, 0xa2, 0xa3} { + t.Run(fmt.Sprintf("mode%d/moffs%02x", mode, opcode), func(t *testing.T) { + code := []byte{0x67, opcode, 0xc0, 0xff} + if mode == 64 { + code = append(code, 0xff, 0xff) + } + found := false + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.Op == "MOVB" || instruction.Op == OpMOVL { + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("absolute address override borrowed default addressing: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost actual moffs operand") + } + }) + } + } +} + +func TestX86RawAddressOverrideEA32ExtendedSIBFieldsStayRestricted(t *testing.T) { + // Physical REX.B/X extend EA32 base/index fields in 64-bit mode. The + // index field 4 without REX.X means no index, not an incoming SP grant. + for base := byte(0); base < 16; base++ { + for index := byte(0); index < 16; index++ { + for scale := byte(0); scale < 4; scale++ { + t.Run(fmt.Sprintf("base%d/index%d/scale%d", base, index, scale), func(t *testing.T) { + rex := byte(0x40) | base>>3 | (index>>3)<<1 + code := []byte{0x67, rex, 0x8b, 0x44, scale<<6 | (index&7)<<3 | base&7, 0x80} + physical, err := x86asm.Decode(code, 64) + if err != nil || physical.Len != len(code) || physical.AddrSize != 32 { + t.Fatalf("physical extended EA32 fields: %+v, %v", physical, err) + } + found := false + for _, instruction := range x86AddressOverrideDecoded(t, 64, code) { + if instruction.Op == OpMOVL { + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 32 { + t.Fatalf("extended EA32 acquired native pointer semantics: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost extended physical memory form") + } + }) + } + } + } +} + +func TestX86RawAddressOverrideExistingTypedConsumersRemainBound(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + forms := []struct { + name string + code []byte + }{ + {"ADCXL", []byte{0x67, 0x66, 0x0f, 0x38, 0xf6, 0x18}}, + {"ADOXL", []byte{0x67, 0xf3, 0x0f, 0x38, 0xf6, 0x18}}, + {"ENQCMD", []byte{0x67, 0xf2, 0x0f, 0x38, 0xf8, 0x00}}, + {"ENQCMDS", []byte{0x67, 0xf3, 0x0f, 0x38, 0xf8, 0x00}}, + {"UMONITOR", []byte{0x67, 0xf3, 0x0f, 0xae, 0xf3}}, + } + if mode == 64 { + forms = append(forms, struct { + name string + code []byte + }{"ADCXQ", []byte{0x67, 0x66, 0x48, 0x0f, 0x38, 0xf6, 0x18}}, struct { + name string + code []byte + }{"ADOXQ", []byte{0x67, 0xf3, 0x48, 0x0f, 0x38, 0xf6, 0x18}}) + } + for _, form := range forms { + t.Run(target.triple+"/"+form.name, func(t *testing.T) { + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(form.code) + "RET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, assembly) + found := false + for _, instruction := range x86AddressOverrideDecoded(t, mode, form.code) { + if instruction.Op != Op(form.name) { + continue + } + found = true + if instruction.x86SourceRaw == nil || instruction.x86AddressBits != mode/2 || instruction.x86UnboundAddressOverride { + t.Fatalf("dedicated typed address receipt changed: %+v", instruction) + } + } + if !found { + t.Fatal("lost dedicated width-consuming instruction") + } + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}, + }) + if err != nil || !strings.Contains(ir, fmt.Sprintf("to i%d", mode/2)) { + t.Fatalf("dedicated consumer lost physical truncation: %v", err) + } + compileLLVMToObject(t, llc, target.triple, "address-consumer.ll", "address-consumer.o", ir) + }) + } + } +} + +func TestX86RawAddressOverrideImplicitMaskedStoreAndMonitor(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, form := range []struct { + name, named string + code []byte + }{ + {"MASKMOVQ", "MASKMOVQ M1,M0", []byte{0x67, 0x0f, 0xf7, 0xc1}}, + {"MASKMOVDQU", "MASKMOVDQU X1,X0", []byte{0x67, 0x66, 0x0f, 0xf7, 0xc1}}, + {"MONITOR", "MONITOR", []byte{0x67, 0x0f, 0x01, 0xc8}}, + } { + t.Run(target.triple+"/"+form.name, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, "TEXT raw(SB),4,$0-0\n"+form.named+"\nRET\n") + if encoded := x86AddressOverrideNamedBytes(t, target.arch, form.named); !bytes.HasPrefix(encoded, form.code[1:]) { + t.Fatalf("named Go form has different physical opcode: %x, want prefix %x", encoded, form.code[1:]) + } + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(form.code) + "RET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, assembly) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("implicit physical address override borrowed default addressing: %v", err) + } + + // The same actual opcode without an address override retains + // its prior ordinary lowering. This is object coverage, not + // permission to execute an unknown implicit native address. + control := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(form.code[1:]) + "RET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, control) + file, err = Parse(ArchAMD64, control) + if err != nil { + t.Fatal(err) + } + ir, err = Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}, + }) + if err != nil { + t.Fatalf("default-address control regressed: %v", err) + } + compileLLVMToObject(t, llc, target.triple, "implicit-address.ll", "implicit-address.o", ir) + }) + } + } +} + +func TestX86RawAddressOverrideImplicitMaskedStoreAllRegisterFields(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, xmm := range []bool{false, true} { + last := 7 + if xmm && mode == 64 { + last = 15 + } + for source := 0; source <= last; source++ { + for mask := 0; mask <= last; mask++ { + t.Run(fmt.Sprintf("mode%d/xmm%t/source%d/mask%d", mode, xmm, source, mask), func(t *testing.T) { + code := []byte{0x67} + if xmm { + code = append(code, 0x66) + } + if source >= 8 || mask >= 8 { + rex := byte(0x40) + if source >= 8 { + rex |= 4 + } + if mask >= 8 { + rex |= 1 + } + code = append(code, rex) + } + code = append(code, 0x0f, 0xf7, byte(0xc0|(source&7)<<3|(mask&7))) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) || physical.AddrSize != mode/2 { + t.Fatalf("physical implicit store: %+v, %v", physical, err) + } + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.Op == OpTEXT || instruction.Op == OpRET { + continue + } + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != mode/2 { + t.Fatalf("implicit address restriction lost at register alias: %+v", instruction) + } + } + }) + } + } + } + } +} + +func TestX86RawAddressOverrideUnsupportedImplicitDecoderBoundaries(t *testing.T) { + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + // Go accepts its named VEX spelling, while the pinned x/arch + // generic decoder does not implement this opcode. Do not grant + // a legacy/default address interpretation to its raw bytes. + x86LEAActualTargetGoObject(t, target.arch, target.triple, "TEXT raw(SB),4,$0-0\nVMASKMOVDQU X0,X1\nRET\n") + for _, code := range [][]byte{ + {0xc5, 0xf9, 0xf7, 0xc1}, {0x67, 0xc5, 0xf9, 0xf7, 0xc1}, + {0x0f, 0x01, 0xfa}, {0x67, 0x0f, 0x01, 0xfa}, // MONITORX + } { + mode := 64 + if target.arch == "386" { + mode = 32 + } + if _, err := x86AddressOverrideDecode(mode, code); err == nil { + t.Fatalf("unsupported implicit raw decoder acquired a consumer: %x", code) + } + } + requireLEANativeGoObject(t, target.arch, "TEXT raw(SB),4,$0-0\nMONITORX\nRET\n", false) + }) + } +} + +func TestX86RawAddressOverrideDoesNotInventMemoryForRegisterForms(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, code := range [][]byte{{0x67, 0x8b, 0xc2}, {0x67, 0x89, 0xc2}, {0x67, 0x03, 0xc2}} { + t.Run(fmt.Sprintf("mode%d/%x", mode, code), func(t *testing.T) { + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("physical register control: %+v, %v", physical, err) + } + for _, instruction := range x86AddressOverrideDecoded(t, mode, code) { + if instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 0 { + t.Fatalf("ignored register prefix gained physical memory semantics: %+v", instruction) + } + } + }) + } + } +} + +func TestX86RawAddressOverrideRestrictionSurvivesNormalizationAndMutation(t *testing.T) { + assembly := "TEXT raw(SB),4,$0-0\nBYTE $0x67\nBYTE $0x8b\nBYTE $0x07\nRET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + for pass := 0; pass < 3; pass++ { + file, err = normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + copy := *file + copy.Funcs = append([]Func(nil), file.Funcs...) + copy.Funcs[0].Instrs = append([]Instr(nil), file.Funcs[0].Instrs...) + file = © + found := false + for _, instruction := range file.Funcs[0].Instrs { + if instruction.Op != OpMOVL { + continue + } + found = true + if !instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 32 { + t.Fatal("normalization/copy lost restrictive decoder identity") + } + for _, width := range []int{0, 16, 32, 64, 128} { + mutated := instruction + mutated.x86AddressBits = width + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{mutated}}, FuncSig{Ret: Void}, nil) + before := output.String() + if _, err := ctx.lowerInstr(0, 0, mutated, nil, nil); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("width %d laundered decoder restriction: %v", width, err) + } + } + } + if !found { + t.Fatal("normalization/copy lost source instruction") + } + } +} + +func TestX86RawAddressOverrideCannotReuseDefaultCallerCertificate(t *testing.T) { + instruction := Instr{Op: OpMOVL, Args: []Operand{ + {Kind: OpReg, Reg: AX}, {Kind: OpMem, Mem: MemRef{Base: DX}}, + }} + x86CallerRecordMemory(&instruction, 1, 4, true) + if !x86CallerMemoryCertified(instruction, 1) { + t.Fatal("default-width certificate control is invalid") + } + for _, width := range []int{0, 16, 32, 64} { + mutated := instruction + mutated.x86UnboundAddressOverride = true + mutated.x86AddressBits = width + if x86CallerMemoryCertified(mutated, 1) { + t.Fatalf("decoder restriction reused a default-width certificate at width %d", width) + } + // A change to the spelling or operands cannot turn a restrictive + // decoder receipt into a positive LEA/address capability either. + mutated.Op = "LEAQ" + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{mutated}}, FuncSig{Ret: Void}, nil) + before := output.String() + if _, err := ctx.lowerInstr(0, 0, mutated, nil, nil); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("changed opcode bypassed restrictive source identity: %v", err) + } + } +} + +func TestX86RawAddressOverrideDefaultCallerControlsAndPhysicalSegments(t *testing.T) { + for _, replacement := range []string{ + "MOVL $7,AX\nMOVL AX,(DX)", "MOVL (DI),AX\nADDL $7,AX\nMOVL AX,(DX)", + } { + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple+"/"+replacement, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", replacement, 1) + x86LEAActualTargetGoObject(t, "amd64", triple, assembly) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + }) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + space int + }{{0x64, FS, 257}, {0x65, GS, 256}} { + t.Run(string(segment.reg), func(t *testing.T) { + code := []byte{segment.prefix, 0x8b, 0x07} + found := false + for _, instruction := range x86AddressOverrideDecoded(t, 64, code) { + if instruction.Op == OpMOVL { + found = true + if instruction.x86UnboundAddressOverride || instruction.x86AddressBits != 0 || instruction.Args[0].Mem.Segment != segment.reg { + t.Fatalf("ordinary segment/address identity changed: %+v", instruction) + } + } + } + if !found { + t.Fatal("lost segmented MOVL control") + } + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(code) + "RET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Sigs: map[string]FuncSig{"raw": {Name: "raw", Ret: Void}}}) + if err != nil || !strings.Contains(ir, fmt.Sprintf("addrspace(%d)", segment.space)) { + t.Fatalf("default-width physical segment was erased: %v", err) + } + }) + } +} + +func TestX86RawAddressOverrideFiveTargetsAndThreeAPIs(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, opcode := range []byte{0x8b, 0x89} { + t.Run(fmt.Sprintf("%s/%02x", target.triple, opcode), func(t *testing.T) { + assembly, sig, pkg := x86SourceStackNumericalFixture(target.arch, + "@MOV x+0(FP),DI\n"+x86RawDisplacementDirectives([]byte{0x67, opcode, 0x07})+"@MOV AX,ret+@WORD(FP)\nRET") + x86LEAActualTargetGoObject(t, target.arch, target.triple, assembly) + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + ir, err := Translate(file, opt) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("ordinary API acquired overridden pointer: %v", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module API acquired overridden pointer: %v", err) + } + translated, err := TranslateGoModule(pkg, []byte(assembly), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: opt.ResolveSym, + }) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("typed API acquired overridden pointer: %v", err) + } + }) + } + } +} + +func TestX86RawAddressOverrideRetainsGenericPhysicalMemoryWidth(t *testing.T) { + for _, mode := range []int{32, 64} { + arch := "amd64" + if mode == 32 { + arch = "386" + } + for _, opcode := range []byte{0x8b, 0x89} { + t.Run(fmt.Sprintf("mode%d/%02x", mode, opcode), func(t *testing.T) { + code := []byte{0x67, opcode, 0x07} + assembly := "TEXT raw(SB),4,$0-0\n" + x86RawDisplacementDirectives(code) + "RET\n" + file, err := Parse(ArchAMD64, assembly) + if err != nil { + t.Fatal(err) + } + function, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + for _, instruction := range function.Instrs { + if instruction.Op == OpMOVL { + if instruction.x86AddressBits != mode/2 { + t.Fatalf("decoded physical address width lost: got %d, want %d; %+v", instruction.x86AddressBits, mode/2, instruction) + } + return + } + } + t.Fatal("decoded raw source lost its MOVL") + }) + } + } +} diff --git a/amd64_raw_adx.go b/amd64_raw_adx.go new file mode 100644 index 000000000..8165cb257 --- /dev/null +++ b/amd64_raw_adx.go @@ -0,0 +1,140 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel SDM Vol. 2A specifies 66/F3 0F 38 F6 /r for ADCX/ADOX. REX.W +// alone selects 64 bits: 0x66 is mandatory, not a 16-bit width override. +// Reuse the decoder's ADD r/m,reg ModRM/SIB grammar, never its flag semantics. +func decodeX86RawADX(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 32 && mode != 64 || offset < 0 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + bytes := code[offset:] + i := 0 + var repeat, segment, rex byte + operandOverride, addressOverride, locked := false, false, false + for i < len(bytes) { + switch bytes[i] { + case 0xf2, 0xf3: + repeat, rex = bytes[i], 0 + case 0x66: + operandOverride, rex = true, 0 + case 0x67: + addressOverride, rex = true, 0 + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65: + segment, rex = bytes[i], 0 + case 0xf0: + locked, rex = true, 0 + default: + if mode == 64 && bytes[i] >= 0x40 && bytes[i] <= 0x4f { + rex = bytes[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + prefix := repeat + if prefix == 0 && operandOverride { + prefix = 0x66 + } + if prefix != 0x66 && prefix != 0xf3 || len(bytes) < i+3 || + bytes[i] != 0x0f || bytes[i+1] != 0x38 || bytes[i+2] != 0xf6 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + fail := func(reason string) (Instr, int, x86RawLiteralRange, bool, error) { + return Instr{}, 0, x86RawLiteralRange{}, true, fmt.Errorf("ADX: %s", reason) + } + if locked { + return fail("LOCK is not permitted") + } + bits := 32 + if rex&8 != 0 { + bits = 64 + } + var op Op + for name, spec := range amd64ADXSpecs { + if spec.prefix == prefix && spec.bits == bits { + op = name + break + } + } + var equivalent []byte + if segment == 0x64 || segment == 0x65 { + equivalent = append(equivalent, segment) + } + if addressOverride { + equivalent = append(equivalent, 0x67) + } + if rex != 0 { + equivalent = append(equivalent, rex) + } + equivalent = append(equivalent, 0x03) + header := len(equivalent) + equivalent = append(equivalent, bytes[i+3:]...) + decoded, err := x86asm.Decode(equivalent, mode) + if err != nil || decoded.Op != x86asm.ADD || decoded.Len <= header { + return fail("truncated or invalid ModRM/SIB address") + } + length := i + 3 + decoded.Len - header + if length > 15 { + return fail("encoding exceeds the 15-byte instruction limit") + } + if decoded.PCRel != 0 { + if decoded.AddrSize != 64 || segment == 0x64 || segment == 0x65 { + return fail("PC-relative source needs an unsegmented source-local literal") + } + patched, data, literal, err := x86RawRIPBytes(code, offset, offset+i+3, bits/8) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, _, _, matched, err := decodeX86RawADX(patched, 0, mode) + if err != nil || !matched { + return fail("cannot represent patched source-local literal") + } + setX86RawRIPDataOperand(&instruction, 0, data) + return instruction, length, literal, true, nil + } + memory, isMemory := decoded.Args[1].(x86asm.Mem) + if isMemory { + // The textual parser has only two address groups. Attach FS/GS + // structurally so a 16-bit BX+SI address does not gain a third one. + memory.Segment = 0 + decoded.Args[1] = memory + } + syntax, err := decodedX86GoSyntax(decoded, equivalent[:decoded.Len]) + if err != nil { + return fail("cannot represent decoded operands") + } + parsed, err := parseDecodedX86Instruction(syntax) + if err != nil || len(parsed) != 1 || len(parsed[0].Args) != 2 { + return fail("cannot represent decoded register/memory form") + } + instruction := parsed[0] + if isMemory { + if memory.Base == 0 && memory.Index == 0 { + instruction.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Off: memory.Disp}} + } + if instruction.Args[0].Kind != OpMem { + return fail("cannot represent decoded source as memory") + } + switch segment { + case 0x64: + instruction.Args[0].Mem.Segment = FS + case 0x65: + instruction.Args[0].Mem.Segment = GS + } + } + instruction.Op = op + instruction.x86Encoded = true + instruction.x86AddressBits = decoded.AddrSize + instruction.Raw = fmt.Sprintf("%s %s, %s", op, instruction.Args[0], instruction.Args[1]) + return instruction, length, x86RawLiteralRange{}, true, nil +} diff --git a/amd64_raw_adx_test.go b/amd64_raw_adx_test.go new file mode 100644 index 000000000..762d7c40c --- /dev/null +++ b/amd64_raw_adx_test.go @@ -0,0 +1,285 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86ADXGrammarMatchesCompleteGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + text := string(data) + table := regexp.MustCompile(`(?s)var yml_rl = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(text) + if len(table) != 2 || strings.Count(table[1], "argList{") != 1 || + !strings.Contains(table[1], "argList{Yml, Yrl}") { + t.Fatal("Go ADX register/memory operand grammar changed") + } + rows := regexp.MustCompile(`\{A((?:ADCX|ADOX)[LQ]),\s*yml_rl,\s*(Pq[45]w?),\s*opBytes\{0xf6\}\}`).FindAllStringSubmatch(text, -1) + if len(rows) != 4 || len(amd64ADXSpecs) != len(rows) { + t.Fatal("typed ADX grammar does not match the complete Go encoder family") + } + for _, row := range rows { + spec, ok := amd64ADXSpecs[Op(row[1])] + bits, prefix, carry := 32, byte(0x66), amd64ADXCarryCF + if strings.HasSuffix(row[2], "w") { + bits = 64 + } + if strings.HasPrefix(row[2], "Pq5") { + prefix, carry = 0xf3, amd64ADXCarryOF + } + if !ok || spec.bits != bits || spec.prefix != prefix || spec.carry != carry { + t.Fatalf("%s spec %+v does not match Go encoder row %v", row[1], spec, row) + } + } +} + +func TestX86RawADXEveryRegisterEncoding(t *testing.T) { + for op, spec := range amd64ADXSpecs { + for _, mode := range []int{32, 64} { + if mode == 32 && spec.bits == 64 { + continue + } + registers := 16 + if mode == 32 { + registers = 8 + } + for source := 0; source < registers; source++ { + for destination := 0; destination < registers; destination++ { + code := []byte{spec.prefix} + rex := byte(0x40 | source>>3 | (destination>>3)<<2) + if spec.bits == 64 { + rex |= 8 + } + if rex != 0x40 { + code = append(code, rex) + } + code = append(code, 0x0f, 0x38, 0xf6, byte(0xc0|(destination&7)<<3|source&7)) + got, length, _, matched, err := decodeX86RawADX(code, 0, mode) + wantSource, _ := decodedX86GeneralRegister(source) + wantDestination, _ := decodedX86GeneralRegister(destination) + if err != nil || !matched || length != len(code) || got.Op != op || + got.Args[0].Reg != wantSource || got.Args[1].Reg != wantDestination { + t.Fatalf("decode %x mode=%d: %+v error=%v", code, mode, got, err) + } + } + } + } + } +} + +func TestX86RawADXAddressAndSegmentForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, mode := range []int{32, 64} { + for _, segment := range []byte{0, 0x64, 0x65} { + for _, override := range []bool{false, true} { + for _, prefix := range []byte{0x66, 0xf3} { + code := []byte{prefix} + if segment != 0 { + code = append(code, segment) + } + bits := mode + if override { + code = append(code, 0x67) + bits /= 2 + } + code = append(code, 0x0f, 0x38, 0xf6, 0x18) + ins, length, _, matched, err := decodeX86RawADX(code, 0, mode) + if err != nil || !matched || length != len(code) || ins.x86AddressBits != bits || ins.Args[0].Kind != OpMem { + t.Fatalf("decode %x mode=%d: %+v error=%v", code, mode, ins, err) + } + wantSegment := Reg("") + if segment == 0x64 { + wantSegment = FS + } else if segment == 0x65 { + wantSegment = GS + } + if ins.Args[0].Mem.Segment != wantSegment { + t.Fatalf("segment lost: %+v", ins) + } + var source strings.Builder + source.WriteString("TEXT adxaddress(SB),$0-0\n") + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + arch, triple := "amd64", "x86_64-unknown-linux-gnu" + if mode == 32 { + arch, triple = "386", "i386-unknown-linux-gnu" + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"adxaddress": {Name: "adxaddress", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if override && !strings.Contains(ir, fmt.Sprintf(" to i%d", bits)) { + t.Fatal("address-size wrapping was not lowered") + } + compileLLVMToObject(t, llc, triple, "adx-address.ll", "adx-address.o", ir) + } + } + } + } +} + +func TestX86RawADXSourceLocalRIPAndInvalidEncodings(t *testing.T) { + for op, spec := range amd64ADXSpecs { + code := []byte{spec.prefix} + if spec.bits == 64 { + code = append(code, 0x48) + } + code = append(code, 0x0f, 0x38, 0xf6, 0x1d, 1, 0, 0, 0, 0xc3) + for i := 0; i < spec.bits/8; i++ { + code = append(code, byte(i+1)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, string(op), map[string]bool{}) + if err != nil || len(decoded) != 2 || decoded[0].Op != op || len(decoded[0].x86RIPLiteralData) != spec.bits/8 { + t.Fatalf("source-local literal %x: %+v error=%v", code, decoded, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated literal", map[string]bool{}); err == nil { + t.Fatal("accepted incomplete literal") + } + } + for _, code := range [][]byte{ + {0xf0, 0x66, 0x0f, 0x38, 0xf6, 0xc0}, + {0x66, 0x0f, 0x38, 0xf6}, + {0xf3, 0x0f, 0x38, 0xf6, 0x04}, + {0x66, 0x0f, 0x38, 0xf6, 0x80}, + append([]byte{0x66}, append([]byte(strings.Repeat("\x67", 11)), 0x0f, 0x38, 0xf6, 0xc0)...), + } { + if _, _, _, matched, err := decodeX86RawADX(code, 0, 64); !matched || err == nil { + t.Fatalf("accepted invalid ADX %x", code) + } + } +} + +func TestX86RawADXCompleteRegisterAndMemoryForms(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, prefix := range []byte{0x66, 0xf3} { + for _, bits := range []int{32, 64} { + if mode == 32 && bits == 64 { + continue + } + for _, form := range []struct { + name string + modrm []byte + source string + }{ + {"register", []byte{0xd8}, "AX"}, + {"memory", []byte{0x58, 0x20}, "32(AX)"}, + {"indexed", []byte{0x5c, 0x88, 0xf0}, "-16(AX)(CX*4)"}, + } { + name := fmt.Sprintf("mode%d/prefix%x/bits%d/%s", mode, prefix, bits, form.name) + t.Run(name, func(t *testing.T) { + code := []byte{prefix} + if bits == 64 { + code = append(code, 0x48) + } + code = append(code, 0x0f, 0x38, 0xf6) + code = append(code, form.modrm...) + code = append(code, 0xc3) + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + stem, width := "ADCX", "L" + if prefix == 0xf3 { + stem = "ADOX" + } + if bits == 64 { + width = "Q" + } + if len(decoded) != 2 || decoded[0].Op != Op(stem+width) || decoded[1].Op != OpRET || + len(decoded[0].Args) != 2 || decoded[0].Args[0].String() != form.source || decoded[0].Args[1].Reg != BX { + t.Fatalf("decoded %x as %+v, want %s%s %s, BX; RET", code, decoded, stem, width, form.source) + } + }) + } + } + } + } +} + +func TestX86RawADXExtendedRegistersFromEthereum(t *testing.T) { + for _, form := range []struct { + code []byte + op Op + source, destination Reg + }{ + {[]byte{0x66, 0x4c, 0x0f, 0x38, 0xf6, 0xc8}, "ADCXQ", AX, "R9"}, + {[]byte{0xf3, 0x4c, 0x0f, 0x38, 0xf6, 0xd3}, "ADOXQ", BX, "R10"}, + {[]byte{0x66, 0x4d, 0x0f, 0x38, 0xf6, 0xe7}, "ADCXQ", "R15", "R12"}, + } { + code := append(append([]byte(nil), form.code...), 0xc3) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "Ethereum ADX", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.op || + decoded[0].Args[0].Reg != form.source || decoded[0].Args[1].Reg != form.destination { + t.Fatalf("decoded %x as %+v", code, decoded) + } + } +} + +func TestX86RawADXObjectsAcrossTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT adxraw(SB),$0-0\n") + for _, prefix := range []byte{0x66, 0xf3} { + for _, rex := range []byte{0, 0x48} { + if target.arch == "386" && rex != 0 { + continue + } + for _, modrm := range [][]byte{{0xd8}, {0x58, 0x20}, {0x5c, 0x88, 0xf0}} { + code := []byte{prefix} + if rex != 0 { + code = append(code, rex) + } + code = append(code, 0x0f, 0x38, 0xf6) + code = append(code, modrm...) + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"adxraw": {Name: "adxraw", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "adx-raw.ll", "adx-raw.o", ir) + }) + } +} diff --git a/amd64_raw_aes.go b/amd64_raw_aes.go new file mode 100644 index 000000000..3d38233f1 --- /dev/null +++ b/amd64_raw_aes.go @@ -0,0 +1,162 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// VAES is absent from the pinned generic decoder. Decode its complete actual +// Go VEX/EVEX family without lending another instruction's operand/PC proof. +func decodedX86AESInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86AESInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86AESInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i, segment, addressOverride := 0, Reg(""), false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto prefix + } + } +prefix: + if i >= len(code) { + return Instr{}, 0, false, nil + } + var opcodeIndex, modRMIndex, mapSelect, width, rExt, rHigh, xExt, bExt, vRegister, vHigh, rmHigh int + evex := false + switch code[i] { + case 0xc4: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + p0, p1 := code[i+1], code[i+2] + opcodeIndex, modRMIndex, mapSelect = i+3, i+4, int(p0&31) + if p1&3 != 1 || x86AESRawOpcode(mapSelect, code[opcodeIndex]) == "" { + return Instr{}, 0, false, nil + } + rExt, xExt, bExt = int(^p0>>7)&1, int(^p0>>6)&1, int(^p0>>5)&1 + vRegister = int(^p1>>3) & 15 + width = 16 + if p1&4 != 0 { + width = 32 + } + case 0x62: + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcodeIndex, modRMIndex, mapSelect = i+4, i+5, int(p0&3) + if p0&0x0c != 0 || p1&3 != 1 || x86AESRawOpcode(mapSelect, code[opcodeIndex]) == "" { + return Instr{}, 0, false, nil + } + evex = true + if p1&4 != 4 || p2&0x97 != 0 { + return Instr{}, 0, true, fmt.Errorf("AES EVEX has reserved bit/mask/zeroing/broadcast attributes") + } + rExt, rHigh, xExt, bExt = int(^p0>>7)&1, int(^p0>>4)&1, int(^p0>>6)&1, int(^p0>>5)&1 + vRegister, vHigh, rmHigh = int(^p1>>3)&15, int(^p2>>3)&1, xExt + switch p2 >> 5 & 3 { + case 0: + width = 16 + case 1: + width = 32 + case 2: + width = 64 + default: + return Instr{}, 0, true, fmt.Errorf("reserved AES EVEX vector length") + } + default: + return Instr{}, 0, false, nil + } + op := x86AESRawOpcode(mapSelect, code[opcodeIndex]) + unary := op == "VAESIMC" || op == "VAESKEYGENASSIST" + if unary && (evex || width != 16 || vRegister != 0) { + return Instr{}, 0, true, fmt.Errorf("AES unary requires VEX.128 with reserved vvvv") + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing AES ModRM") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("%w: AES native address-size override is not bound", ErrProbeNeedsContext) + } + destinationNumber := int(code[modRMIndex]>>3&7) + rExt*8 + rHigh*16 + stateNumber := vRegister + vHigh*16 + if mode == 32 && (destinationNumber >= 8 || stateNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: extended AES vector in physical mode32", ErrProbeNeedsContext) + } + if mode == 32 && code[modRMIndex]>>6 == 3 && int(code[modRMIndex]&7)+bExt*8+rmHigh*16 >= 8 { + return Instr{}, 0, true, fmt.Errorf("%w: extended AES source vector in physical mode32", ErrProbeNeedsContext) + } + destination, valid := decodedX86VectorRegister(destinationNumber, width) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid AES destination") + } + disp8Scale := 1 + if evex { + disp8Scale = width + } + var source Operand + var consumed int + var err error + if code[modRMIndex]>>6 == 3 { + source, consumed, err = decodedX86VectorRMSource(code[modRMIndex:], mode, bExt, xExt, rmHigh, width, segment, disp8Scale) + } else { + source, consumed, err = decodedX86RMSourceWithPC(code[modRMIndex:], mode, bExt, xExt, segment, disp8Scale, memory.physicalPC) + } + if err != nil { + return Instr{}, 0, true, err + } + length := modRMIndex + consumed + args := []Operand{source} + if op == "VAESKEYGENASSIST" { + if len(code) <= length { + return Instr{}, 0, true, fmt.Errorf("missing AES imm8") + } + args = append([]Operand{{Kind: OpImm, Imm: int64(code[length])}}, args...) + length++ + } + if !unary { + state, _ := decodedX86VectorRegister(stateNumber, width) + args = append(args, Operand{Kind: OpReg, Reg: state}) + } + args = append(args, Operand{Kind: OpReg, Reg: destination}) + spelling := make([]string, len(args)) + for index, a := range args { + spelling[index] = a.String() + } + return Instr{Op: op, Args: args, Raw: string(op) + " " + strings.Join(spelling, ", "), x86Encoded: true}, length, true, nil +} + +func x86AESRawOpcode(mapSelect int, opcode byte) Op { + if mapSelect == 3 && opcode == 0xdf { + return "VAESKEYGENASSIST" + } + if mapSelect == 2 { + switch opcode { + case 0xdb: + return "VAESIMC" + case 0xdc: + return "VAESENC" + case 0xdd: + return "VAESENCLAST" + case 0xde: + return "VAESDEC" + case 0xdf: + return "VAESDECLAST" + } + } + return "" +} diff --git a/amd64_raw_alias_test.go b/amd64_raw_alias_test.go new file mode 100644 index 000000000..5e5829cd8 --- /dev/null +++ b/amd64_raw_alias_test.go @@ -0,0 +1,582 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86RawDecoderPreservesGoOpcodeAliasesAndMandatoryPrefixes(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + }{ + {name: "PACKSSDW is Go PACKSSLW", code: []byte{0x66, 0x0f, 0x6b, 0xca}, want: "PACKSSLW"}, + {name: "PCMPEQD is Go PCMPEQL", code: []byte{0x66, 0x0f, 0x76, 0xca}, want: "PCMPEQL"}, + {name: "PCMPGTD is Go PCMPGTL", code: []byte{0x66, 0x0f, 0x66, 0xca}, want: "PCMPGTL"}, + {name: "PMADDWD is Go PMADDWL", code: []byte{0x66, 0x0f, 0xf5, 0xca}, want: "PMADDWL"}, + {name: "PMULUDQ is Go PMULULQ", code: []byte{0x66, 0x0f, 0xf4, 0xca}, want: "PMULULQ"}, + {name: "PSLLD is Go PSLLL", code: []byte{0x66, 0x0f, 0xf2, 0xca}, want: "PSLLL"}, + {name: "PSRAD is Go PSRAL", code: []byte{0x66, 0x0f, 0xe2, 0xca}, want: "PSRAL"}, + {name: "PSRLD is Go PSRLL", code: []byte{0x66, 0x0f, 0xd2, 0xca}, want: "PSRLL"}, + {name: "PSUBD is Go PSUBL", code: []byte{0x66, 0x0f, 0xfa, 0xca}, want: "PSUBL"}, + {name: "PUNPCKHDQ is Go PUNPCKHLQ", code: []byte{0x66, 0x0f, 0x6a, 0xca}, want: "PUNPCKHLQ"}, + {name: "PUNPCKHWD is Go PUNPCKHWL", code: []byte{0x66, 0x0f, 0x69, 0xca}, want: "PUNPCKHWL"}, + {name: "PUNPCKLDQ is Go PUNPCKLLQ", code: []byte{0x66, 0x0f, 0x62, 0xca}, want: "PUNPCKLLQ"}, + {name: "PUNPCKLWD is Go PUNPCKLWL", code: []byte{0x66, 0x0f, 0x61, 0xca}, want: "PUNPCKLWL"}, + {name: "MOVDQA is Go MOVO", code: []byte{0x66, 0x0f, 0x6f, 0xc1}, want: "MOVO"}, + {name: "MOVDQU is Go MOVOU", code: []byte{0xf3, 0x0f, 0x6f, 0xc1}, want: "MOVOU"}, + {name: "MOVSD XMM is Go MOVSD", code: []byte{0xf2, 0x0f, 0x10, 0xc1}, want: "MOVSD"}, + {name: "CLFLUSHOPT mandatory prefix", code: []byte{0x66, 0x0f, 0xae, 0x38}, want: "CLFLUSHOPT"}, + {name: "CLWB mandatory prefix", code: []byte{0x66, 0x0f, 0xae, 0x30}, want: "CLWB"}, + } + for _, goarch := range []string{"386", "amd64"} { + for _, test := range tests { + t.Run(goarch+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want { + t.Fatalf("decoded %#x as %#v, want one %s instruction", test.code, got.Instrs, test.want) + } + }) + } + } +} + +func TestX86RawDecoderVBROADCASTI128CompleteFamily(t *testing.T) { + tests := []struct { + name string + goarch string + code []byte + source MemRef + dest Reg + }{ + {name: "low registers", goarch: "amd64", code: []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x13}, source: MemRef{Base: BX}, dest: "Y2"}, + {name: "extended base", goarch: "amd64", code: []byte{0xc4, 0xc2, 0x7d, 0x5a, 0x13}, source: MemRef{Base: "R11"}, dest: "Y2"}, + {name: "extended destination", goarch: "amd64", code: []byte{0xc4, 0x62, 0x7d, 0x5a, 0x1b}, source: MemRef{Base: BX}, dest: "Y11"}, + {name: "extended base and destination with displacement", goarch: "amd64", code: []byte{0xc4, 0x42, 0x7d, 0x5a, 0x70, 0x10}, source: MemRef{Base: "R8", Off: 16}, dest: "Y14"}, + {name: "extended SIB", goarch: "amd64", code: []byte{0xc4, 0x02, 0x7d, 0x5a, 0x64, 0x88, 0x20}, source: MemRef{Base: "R8", Index: "R9", Scale: 4, Off: 32}, dest: "Y12"}, + {name: "FS segment", goarch: "amd64", code: []byte{0x64, 0xc4, 0xe2, 0x7d, 0x5a, 0x13}, source: MemRef{Segment: FS, Base: BX}, dest: "Y2"}, + {name: "32-bit base", goarch: "386", code: []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x53, 0x7f}, source: MemRef{Base: BX, Off: 127}, dest: "Y2"}, + {name: "32-bit absolute", goarch: "386", code: []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x15, 0x78, 0x56, 0x34, 0x12}, source: MemRef{Off: 0x12345678}, dest: "Y2"}, + } + for _, test := range tests { + t.Run(test.goarch+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, test.goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 { + t.Fatalf("decoded %#x as %#v, want one instruction", test.code, got.Instrs) + } + ins := got.Instrs[0] + if ins.Op != "VBROADCASTI128" || len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.Args[0].Mem != test.source || ins.Args[1].Kind != OpReg || ins.Args[1].Reg != test.dest { + t.Fatalf("decoded %#x as %#v, want VBROADCASTI128 %#v, %s", test.code, ins, test.source, test.dest) + } + }) + } +} + +func TestX86RawDecoderNormalizesArchitecturalAliasesToGo127Forms(t *testing.T) { + code := []byte{ + 0x41, 0x0f, 0x90, 0xc1, // SETO R9B -> SETOS R9B + 0x41, 0x0f, 0x92, 0xc1, // SETB R9B -> SETCS R9B + 0x48, 0x0f, 0x4e, 0xc1, // CMOVLE RAX, RCX -> CMOVQLE AX, CX + 0x04, 0x01, // ADD $1, AL -> ADDB $1, AL + } + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + want := []Op{"SETOS", "SETCS", "CMOVQLE", "ADDB"} + if len(got.Instrs) != len(want) { + t.Fatalf("decoded %#x as %#v, want %d instructions", code, got.Instrs, len(want)) + } + for i, op := range want { + if got.Instrs[i].Op != op { + t.Errorf("decoded instruction %d as %s, want %s (%q)", i, got.Instrs[i].Op, op, got.Instrs[i].Raw) + } + } + if got.Instrs[0].Args[0].Reg != R9B || got.Instrs[1].Args[0].Reg != R9B { + t.Errorf("SETcc byte aliases = %s and %s, want R9B", got.Instrs[0].Args[0].Reg, got.Instrs[1].Args[0].Reg) + } +} + +func TestX86RawDecoderNormalizesEveryThreeOperandIMULWidth(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + }{ + {name: "word imm8 register", code: []byte{0x66, 0x6b, 0xf9, 0x07}, want: "IMUL3W"}, + {name: "long imm8 register", code: []byte{0x6b, 0xf9, 0x07}, want: "IMUL3L"}, + {name: "quad imm8 register", code: []byte{0x48, 0x6b, 0xf9, 0x07}, want: "IMUL3Q"}, + {name: "word imm16 memory", code: []byte{0x66, 0x69, 0x78, 0x08, 0x34, 0x12}, want: "IMUL3W"}, + {name: "long imm32 memory", code: []byte{0x69, 0x78, 0x08, 0x78, 0x56, 0x34, 0x12}, want: "IMUL3L"}, + {name: "quad imm32 memory", code: []byte{0x48, 0x69, 0x78, 0x08, 0x78, 0x56, 0x34, 0x12}, want: "IMUL3Q"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want || len(got.Instrs[0].Args) != 3 { + t.Fatalf("decoded %#x as %#v, want one three-operand %s", test.code, got.Instrs, test.want) + } + }) + } +} + +func TestX86RawDecoderNormalizesCompleteBSWAPFamily(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + reg Reg + }{ + {name: "long low register", code: []byte{0x0f, 0xc8}, want: "BSWAPL", reg: AX}, + {name: "long extended register", code: []byte{0x41, 0x0f, 0xcf}, want: "BSWAPL", reg: Reg("R15")}, + {name: "quad low register", code: []byte{0x48, 0x0f, 0xc8}, want: "BSWAPQ", reg: AX}, + {name: "quad extended register", code: []byte{0x49, 0x0f, 0xcf}, want: "BSWAPQ", reg: Reg("R15")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want || len(got.Instrs[0].Args) != 1 || got.Instrs[0].Args[0].Reg != test.reg { + t.Fatalf("decoded %#x as %#v, want %s %s", test.code, got.Instrs, test.want, test.reg) + } + }) + } +} + +func TestX86RawDecoderNormalizesCompleteScalarIntegerFloatFamily(t *testing.T) { + tests := []struct { + name string + code []byte + want Op + }{ + {name: "long to float register", code: []byte{0xf3, 0x0f, 0x2a, 0xc8}, want: "CVTSL2SS"}, + {name: "long to float memory", code: []byte{0xf3, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSL2SS"}, + {name: "quad to float register", code: []byte{0xf3, 0x48, 0x0f, 0x2a, 0xc8}, want: "CVTSQ2SS"}, + {name: "quad to float memory", code: []byte{0xf3, 0x48, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSQ2SS"}, + {name: "long to double register", code: []byte{0xf2, 0x0f, 0x2a, 0xc8}, want: "CVTSL2SD"}, + {name: "long to double memory", code: []byte{0xf2, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSL2SD"}, + {name: "quad to double register", code: []byte{0xf2, 0x48, 0x0f, 0x2a, 0xc8}, want: "CVTSQ2SD"}, + {name: "quad to double memory", code: []byte{0xf2, 0x48, 0x0f, 0x2a, 0x48, 0x08}, want: "CVTSQ2SD"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.want || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want one %s", test.code, got.Instrs, test.want) + } + }) + } +} + +func TestX86RawDecoderNormalizesCompleteScalarFloatIntegerFamily(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + for _, precision := range []struct { + name string + prefix byte + }{ + {name: "SS", prefix: 0xf3}, + {name: "SD", prefix: 0xf2}, + } { + for _, conversion := range []struct { + name string + opcode byte + }{ + {name: "CVT", opcode: 0x2d}, + {name: "CVTT", opcode: 0x2c}, + } { + for _, width := range []struct { + name string + rex []byte + }{ + {name: "L"}, + {name: "Q", rex: []byte{0x48}}, + } { + if goarch == "386" && width.name == "Q" { + continue + } + for _, source := range []struct { + name string + modRM byte + disp []byte + }{ + {name: "register", modRM: 0xc1}, + {name: "memory", modRM: 0x43, disp: []byte{0x08}}, + } { + name := goarch + "/" + conversion.name + precision.name + "2S" + width.name + "/" + source.name + t.Run(name, func(t *testing.T) { + code := []byte{precision.prefix} + code = append(code, width.rex...) + code = append(code, 0x0f, conversion.opcode, source.modRM) + code = append(code, source.disp...) + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + want := Op(conversion.name + precision.name + "2S" + width.name) + if len(got.Instrs) != 1 || got.Instrs[0].Op != want || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want one %s", code, got.Instrs, want) + } + }) + } + } + } + } + } +} + +func TestTranslateX86RawScalarFloatIntegerReportedEncoding(t *testing.T) { + const source = `TEXT rawScalarFloatInteger(SB),$0-0 + LONG $0x2c0f48f2 + BYTE $0xc0 // CVTTSD2SQ X0, AX + RET +` + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "rawScalarFloatInteger": {Name: "rawScalarFloatInteger", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `call i64 asm sideeffect "cvttsd2siq $1, $0"`) || strings.Contains(ir, "fptosi double") { + t.Fatalf("raw CVTTSD2SQ semantics missing:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "raw-scalar-float-integer.ll", "raw-scalar-float-integer.o", ir) + }) + } +} + +func TestX86RawDecoderNormalizesCompleteMOVNTDQFamily(t *testing.T) { + tests := []struct { + name string + code []byte + reg Reg + base Reg + }{ + {name: "low vector and base registers", code: []byte{0x66, 0x0f, 0xe7, 0x13}, reg: Reg("X2"), base: BX}, + {name: "extended vector register", code: []byte{0x66, 0x44, 0x0f, 0xe7, 0x1b}, reg: Reg("X11"), base: BX}, + {name: "extended base register", code: []byte{0x66, 0x41, 0x0f, 0xe7, 0x13}, reg: Reg("X2"), base: Reg("R11")}, + {name: "extended vector and base registers", code: []byte{0x66, 0x45, 0x0f, 0xe7, 0x1b}, reg: Reg("X11"), base: Reg("R11")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != "MOVNTO" || len(got.Instrs[0].Args) != 2 || got.Instrs[0].Args[0].Reg != test.reg || got.Instrs[0].Args[1].Mem.Base != test.base { + t.Fatalf("decoded %#x as %#v, want MOVNTO %s, (%s)", test.code, got.Instrs, test.reg, test.base) + } + }) + } +} + +func TestTranslateAMD64MOVNTOCompleteOperandForm(t *testing.T) { + const source = `TEXT movnto(SB),$0-0 + MOVNTO X2, 32(BX)(R8*1) + MOVNTO X11, sink<>(SB) + RET + +GLOBL sink<>(SB),$16 +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"movnto": {Name: "movnto", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if count := strings.Count(ir, "store <16 x i8> %"); count != 2 { + t.Fatalf("MOVNTO emitted %d vector stores, want 2:\n%s", count, ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-movnto.ll", "amd64-movnto.o", ir) +} + +func TestDecodedX86CacheLineWritebackCoversMemoryEncodingsWithoutFalsePositives(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + ok bool + }{ + {name: "CLFLUSHOPT displacement", code: []byte{0x67, 0x66, 0x0f, 0xae, 0x78, 0x08}, want: "CLFLUSHOPT", ok: true}, + {name: "CLWB SIB displacement", code: []byte{0x66, 0x0f, 0xae, 0xb4, 0x24, 0x78, 0x56, 0x34, 0x12}, want: "CLWB", ok: true}, + {name: "CLWB REX base", code: []byte{0x66, 0x41, 0x0f, 0xae, 0x30}, want: "CLWB", ok: true}, + {name: "plain CLFLUSH", code: []byte{0x0f, 0xae, 0x38}}, + {name: "plain XSAVEOPT", code: []byte{0x0f, 0xae, 0x30}}, + {name: "register ModRM", code: []byte{0x66, 0x0f, 0xae, 0xf8}}, + {name: "opcode bytes in displacement", code: []byte{0x66, 0x8b, 0x80, 0x0f, 0xae, 0x38, 0x00}}, + } { + t.Run(test.name, func(t *testing.T) { + got, ok := decodedX86CacheLineWriteback(test.code) + if got != test.want || ok != test.ok { + t.Fatalf("decodedX86CacheLineWriteback(%#x) = %q, %v; want %q, %v", test.code, got, ok, test.want, test.ok) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedWencodeCLFLUSHOPT(t *testing.T) { + code := []byte{0x41, 0x66, 0x0f, 0xae, 0x3c, 0x09} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "wencode CLFLUSHOPT sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "CLFLUSHOPT" || len(decoded[0].Args) != 1 || decoded[0].Args[0].String() != "0(CX)(CX*1)" { + t.Fatalf("decoded %x as %#v, want CLFLUSHOPT 0(CX)(CX*1)", code, decoded) + } +} + +func encodeX86RawCacheLineWriteback(op Op, mode, base, index int) []byte { + extension := byte(7) + mandatory := op != "CLFLUSH" + if op == "CLWB" { + extension = 6 + } + var prefixes []byte + if mode == 64 { + rex := byte(0x40 | base/8 | index/8<<1) + if mandatory { + prefixes = append(prefixes, 0x66) + } + prefixes = append(prefixes, rex) + } else if mandatory { + prefixes = append(prefixes, 0x66) + } + mod := byte(0) + if base&7 == 5 { + mod = 1 + } + modRM := mod<<6 | extension<<3 | 4 + sib := byte(2<<6 | (index&7)<<3 | base&7) + code := append(prefixes, 0x0f, 0xae, modRM, sib) + if mod == 1 { + code = append(code, 0) + } + return code +} + +func TestDecodedX86RawCacheLineWritebackCompleteRegisterAddressFamily(t *testing.T) { + operations := []Op{"CLFLUSH", "CLFLUSHOPT", "CLWB"} + count := 0 + for _, mode := range []int{32, 64} { + registers := 8 + if mode == 64 { + registers = 16 + } + for _, op := range operations { + for base := 0; base < registers; base++ { + for index := 0; index < registers; index++ { + if index == 4 { + continue // SIB index 4 without X is the no-index encoding. + } + code := encodeX86RawCacheLineWriteback(op, mode, base, index) + got, length, ok, err := decodedX86CacheLineWritebackInstruction(code, mode) + baseReg, _ := decodedX86GeneralRegister(base) + indexReg, _ := decodedX86GeneralRegister(index) + wantMemory := MemRef{Base: baseReg, Index: indexReg, Scale: 4} + if err != nil || !ok || length != len(code) || got.Op != op || len(got.Args) != 1 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantMemory { + t.Fatalf("decode mode=%d %x = %+v, length=%d, ok=%v, err=%v; want %s %+v", mode, code, got, length, ok, err, op, wantMemory) + } + count++ + } + } + } + } + if count != 888 { + t.Fatalf("covered %d cache-line register-address encodings, want 888", count) + } +} + +func TestDecodedX86RawCacheLineWritebackMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + op Op + mem MemRef + }{ + {name: "CLFLUSH base", code: []byte{0x0f, 0xae, 0x3b}, mode: 64, op: "CLFLUSH", mem: MemRef{Base: BX}}, + {name: "CLFLUSHOPT disp8", code: []byte{0x66, 0x0f, 0xae, 0x7e, 0x7f}, mode: 64, op: "CLFLUSHOPT", mem: MemRef{Base: SI, Off: 127}}, + {name: "CLWB disp32", code: []byte{0x66, 0x0f, 0xae, 0xb7, 0x78, 0x56, 0x34, 0x12}, mode: 64, op: "CLWB", mem: MemRef{Base: DI, Off: 0x12345678}}, + {name: "CLFLUSHOPT ineffective REX before 66", code: []byte{0x41, 0x66, 0x0f, 0xae, 0x3c, 0x09}, mode: 64, op: "CLFLUSHOPT", mem: MemRef{Base: CX, Index: CX, Scale: 1}}, + {name: "CLFLUSHOPT effective REX after 66", code: []byte{0x66, 0x41, 0x0f, 0xae, 0x3c, 0x09}, mode: 64, op: "CLFLUSHOPT", mem: MemRef{Base: "R9", Index: CX, Scale: 1}}, + {name: "CLWB FS extended SIB", code: []byte{0x64, 0x66, 0x43, 0x0f, 0xae, 0x74, 0x8b, 0x20}, mode: 64, op: "CLWB", mem: MemRef{Segment: FS, Base: "R11", Index: "R9", Scale: 4, Off: 32}}, + {name: "CLFLUSHOPT 386 absolute", code: []byte{0x66, 0x0f, 0xae, 0x3d, 0x78, 0x56, 0x34, 0x12}, mode: 32, op: "CLFLUSHOPT", mem: MemRef{Off: 0x12345678}}, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86CacheLineWritebackInstruction(test.code, test.mode) + if err != nil || !ok || length != len(test.code) || got.Op != test.op || len(got.Args) != 1 || got.Args[0].Kind != OpMem || got.Args[0].Mem != test.mem { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s %+v", test.code, got, length, ok, err, test.op, test.mem) + } + }) + } + + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "address override", code: []byte{0x67, 0x66, 0x0f, 0xae, 0x38}, mode: 64}, + {name: "REX.W", code: []byte{0x66, 0x48, 0x0f, 0xae, 0x38}, mode: 64}, + {name: "REX.R", code: []byte{0x66, 0x44, 0x0f, 0xae, 0x38}, mode: 64}, + {name: "register operand", code: []byte{0x66, 0x0f, 0xae, 0xf8}, mode: 64}, + {name: "RIP relative", code: []byte{0x66, 0x0f, 0xae, 0x3d, 0, 0, 0, 0}, mode: 64}, + {name: "truncated SIB", code: []byte{0x66, 0x0f, 0xae, 0x3c}, mode: 64}, + {name: "truncated disp8", code: []byte{0x66, 0x0f, 0xae, 0x78}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86CacheLineWritebackInstruction(test.code, test.mode) + if !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", test.code, ok, err) + } + }) + } + for _, code := range [][]byte{ + {0x0f, 0xae, 0x30}, // XSAVEOPT, not CLWB. + {0x66, 0x0f, 0xae, 0x28}, // unrelated ModRM extension. + {0x66, 0x0f, 0xaf, 0x38}, // unrelated opcode. + {0x66, 0x0f}, // truncated before the opcode can be identified. + {0xf2, 0x66, 0x0f, 0xae, 0x38}, // unsupported mandatory prefix combination. + {0x66, 0x0f, 0xae, 0x38, 0x0f, 0xae, 0x38}, // only the first instruction is consumed. + } { + _, length, ok, err := decodedX86CacheLineWritebackInstruction(code, 64) + if len(code) == 7 { + if err != nil || !ok || length != 4 { + t.Fatalf("concatenated encoding %x returned length=%d ok=%v err=%v", code, length, ok, err) + } + continue + } + if ok || err != nil { + t.Fatalf("unrelated encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawAliasesThroughLLVM22(t *testing.T) { + const source = `TEXT rawaliases(SB),$0-0 + BYTE $0x66; BYTE $0x0f; BYTE $0x6b; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x76; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x66; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xf5; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xf4; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xf2; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xe2; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xd2; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0xfa; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x6a; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x69; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x62; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x61; BYTE $0xca + BYTE $0x66; BYTE $0x0f; BYTE $0x6f; BYTE $0xc1 + BYTE $0xf3; BYTE $0x0f; BYTE $0x6f; BYTE $0xc1 + BYTE $0xf2; BYTE $0x0f; BYTE $0x10; BYTE $0xc1 + BYTE $0x66; BYTE $0x0f; BYTE $0xae; BYTE $0x38 + BYTE $0x66; BYTE $0x0f; BYTE $0xae; BYTE $0x30 + RET +` + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawaliases": {Name: "rawaliases", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"clflushopt $0", "clwb $0", "+clflushopt,+clwb"} { + if !strings.Contains(ir, want) { + t.Fatalf("raw alias lowering omitted %q:\n%s", want, ir) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-raw-aliases.ll", "x86-raw-aliases.o", ir) + }) + } +} diff --git a/amd64_raw_align_right_test.go b/amd64_raw_align_right_test.go new file mode 100644 index 000000000..c9491493e --- /dev/null +++ b/amd64_raw_align_right_test.go @@ -0,0 +1,214 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawPALIGNR(immediate byte, mode, source, destination int) []byte { + code := []byte{0x66} + if mode == 64 { + code = append(code, byte(0x40|destination/8<<2|source/8)) + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + return append(code, 0x0f, 0x3a, 0x0f, modRM, immediate) +} + +func encodeX86RawVEXVPALIGNR(immediate byte, wide bool, first, second, destination int) []byte { + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-first/8)<<5 | 3) + p1 := byte((^second&15)<<3 | 1) + if wide { + p1 |= 4 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0xc4, p0, p1, 0x0f, modRM, immediate} +} + +func encodeX86RawEVEXVPALIGNR(immediate, vectorBits byte, mask int, zeroing bool, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := byte(^second&15)<<3 | 5 + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0x62, p0, p1, p2, 0x0f, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedIPFSVPALIGNR(t *testing.T) { + code := []byte{0xc4, 0xe3, 0x41, 0x0f, 0xc6, 0x04} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "IPFS VPALIGNR sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPALIGNR $4, X6, X7, X0" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawPALIGNRCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, mode := range []int{32, 64} { + registers := 8 + if mode == 64 { + registers = 16 + } + for source := 0; source < registers; source++ { + for destination := 0; destination < registers; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawPALIGNR(immediate, mode, source, destination) + got, length, ok, err := decodedX86PackedAlignRightInstruction(code, mode) + want := fmt.Sprintf("PALIGNR $%d, X%d, X%d", immediate, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode mode=%d %x = %+v, length=%d, ok=%v, err=%v; want %q", mode, code, got, length, ok, err, want) + } + count++ + } + } + } + if count != 320 { + t.Fatalf("covered %d PALIGNR register encodings, want 320", count) + } +} + +func TestDecodedX86RawVPALIGNRCompleteVEXRegisterFamily(t *testing.T) { + count := 0 + for _, wide := range []bool{false, true} { + vectorName := "X" + if wide { + vectorName = "Y" + } + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(first*16 + destination) + code := encodeX86RawVEXVPALIGNR(immediate, wide, first, second, destination) + got, length, ok, err := decodedX86PackedAlignRightInstruction(code, 64) + want := fmt.Sprintf("VPALIGNR $%d, %s%d, %s%d, %s%d", immediate, vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VEX VPALIGNR register encodings, want 8192", count) + } +} + +func TestDecodedX86RawVPALIGNRCompleteEVEXRegisterFamily(t *testing.T) { + count := 0 + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(first*8 + destination) + code := encodeX86RawEVEXVPALIGNR(immediate, byte(vectorBits), masking.mask, masking.zeroing, first, second, destination) + got, length, ok, err := decodedX86PackedAlignRightInstruction(code, 64) + args := []string{ + fmt.Sprintf("$%d", immediate), + fmt.Sprintf("%s%d", vectorName, first), + fmt.Sprintf("%s%d", vectorName, second), + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := "VPALIGNR" + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + if count != 294912 { + t.Fatalf("covered %d EVEX VPALIGNR register encodings, want 294912", count) + } +} + +func TestDecodedX86RawPackedAlignRightMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + { + name: "legacy extended SIB", + code: []byte{0x64, 0x66, 0x47, 0x0f, 0x3a, 0x0f, 0x64, 0x8b, 0x20, 0xff}, + want: "PALIGNR $255, 32(R11)(R9*4)(FS), X12", + }, + { + name: "VEX extended SIB", + code: []byte{0x65, 0xc4, 0x03, 0x15, 0x0f, 0x64, 0x8b, 0x20, 0xfe}, + want: "VPALIGNR $254, 32(R11)(R9*4)(GS), Y13, Y12", + }, + { + name: "EVEX compressed disp8 zero mask", + code: []byte{0x64, 0x62, 0x03, 0x15, 0xc2, 0x0f, 0x64, 0x8b, 0x7f, 0xfd}, + want: "VPALIGNR.Z $253, 8128(R11)(R9*4)(FS), Z29, K2, Z28", + }, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86PackedAlignRightInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) || got.Raw != test.want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", test.code, got, length, ok, err, test.want) + } + }) + } + + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "VEX 386 operand limit", mode: 32, encoding: encodeX86RawVEXVPALIGNR(1, false, 0, 1, 2), wantOK: true}, + {name: "EVEX 386 operand limit", mode: 32, encoding: encodeX86RawEVEXVPALIGNR(1, 0, 0, false, 0, 1, 2), wantOK: true}, + {name: "VEX wrong pp", mode: 64, encoding: []byte{0xc4, 0xe3, 0x40, 0x0f, 0xc6, 4}, wantOK: true}, + {name: "VEX W", mode: 64, encoding: []byte{0xc4, 0xe3, 0xc1, 0x0f, 0xc6, 4}, wantOK: true}, + {name: "VEX truncated immediate", mode: 64, encoding: []byte{0xc4, 0xe3, 0x41, 0x0f, 0xc6}, wantOK: true}, + {name: "EVEX W", mode: 64, encoding: []byte{0x62, 0xf3, 0x85, 0x08, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "EVEX broadcast", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x18, 0x0f, 0x02, 4}, wantOK: true}, + {name: "EVEX reserved vector length", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x68, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "EVEX zero without mask", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x88, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "EVEX truncated immediate", mode: 64, encoding: []byte{0x62, 0xf3, 0x05, 0x08, 0x0f, 0xc2}, wantOK: true}, + {name: "legacy REX.W", mode: 64, encoding: []byte{0x66, 0x48, 0x0f, 0x3a, 0x0f, 0xc2, 4}, wantOK: true}, + {name: "legacy RIP relative", mode: 64, encoding: []byte{0x66, 0x0f, 0x3a, 0x0f, 0x05, 0, 0, 0, 0, 4}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe3, 0x41, 0x0f, 0xc6, 4}, wantOK: true}, + {name: "wrong opcode", mode: 64, encoding: []byte{0xc4, 0xe3, 0x41, 0x0e, 0xc6, 4}}, + {name: "wrong map", mode: 64, encoding: []byte{0xc4, 0xe2, 0x41, 0x0f, 0xc6, 4}}, + {name: "legacy missing 66", mode: 64, encoding: []byte{0x0f, 0x3a, 0x0f, 0xc2, 4}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86PackedAlignRightInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x returned ok=%v err=%v, want ok=%v", test.encoding, ok, err, test.wantOK) + } + if test.wantOK && err == nil { + t.Fatalf("invalid encoding %x was accepted", test.encoding) + } + if !test.wantOK && err != nil { + t.Fatalf("unrelated encoding %x returned err=%v", test.encoding, err) + } + }) + } +} diff --git a/amd64_raw_andn.go b/amd64_raw_andn.go new file mode 100644 index 000000000..3ded5133c --- /dev/null +++ b/amd64_raw_andn.go @@ -0,0 +1,79 @@ +package plan9asm + +import "fmt" + +// decodedX86ANDNInstruction recognizes the complete BMI1 ANDNL/ANDNQ +// Yml,Yrl,Yrl family. x/arch v0.14 reports this VEX.NDS encoding as an +// unknown AVX opcode, so recover every register and safe ModRM/SIB memory form +// before generic decoding. +func decodedX86ANDNInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 2 || code[i+3] != 0xf2 { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + vexExt, vexByte := code[i+1], code[i+2] + if vexByte&7 != 0 { + return Instr{}, 0, true, fmt.Errorf("reserved ANDN VEX prefix") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if len(code) <= i+4 { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vexExt>>7) & 1 + xExt := int(^vexExt>>6) & 1 + bExt := int(^vexExt>>5) & 1 + width64 := vexByte&0x80 != 0 + secondNumber := int(^vexByte>>3) & 15 + modRM := code[i+4] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (secondNumber >= 8 || destinationNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended source or destination register in 32-bit mode") + } + second, valid := decodedX86GeneralRegister(secondNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid second-source register %d", secondNumber) + } + destination, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination register %d", destinationNumber) + } + first, consumed, err := decodedX86VEXRMSource(code[i+4:], mode, bExt, xExt, segment) + if err != nil { + return Instr{}, 0, true, err + } + op := Op("ANDNL") + if width64 && mode == 64 { + op = "ANDNQ" + } + instruction = Instr{ + Op: op, + Args: []Operand{first, {Kind: OpReg, Reg: second}, {Kind: OpReg, Reg: destination}}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, first.String(), second, destination), + } + return instruction, i + 4 + consumed, true, nil +} diff --git a/amd64_raw_average.go b/amd64_raw_average.go new file mode 100644 index 000000000..48cb2b4ab --- /dev/null +++ b/amd64_raw_average.go @@ -0,0 +1,21 @@ +package plan9asm + +import "fmt" + +// The complete VEX/EVEX physical decoder is the only producer of the private +// marker which distinguishes raw mode32 masks from named Go's operand limit. +func decodedX86PackedAverageInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.pp != 1 || p.opcode != 0xe0 && p.opcode != 0xe3 { + return Instr{}, 0, false, nil + } + if p.broadcast { + return Instr{}, 0, true, fmt.Errorf("packed average has no broadcast or rounding encoding") + } + op, bits := Op("VPAVGB"), 8 + if p.opcode == 0xe3 { + op, bits = "VPAVGW", 16 + } + // WIG is a hardware property; the named Go encoder always emits W0. + return decodedX86BinaryVectorOperands(code, p, mode, op, bits) +} diff --git a/amd64_raw_bf16_convert.go b/amd64_raw_bf16_convert.go new file mode 100644 index 000000000..ec190bbeb --- /dev/null +++ b/amd64_raw_bf16_convert.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27 has no named AVX-512 BF16 encodings. Conversion and dot-product +// instructions share widths, masks and dword broadcasts; only conversion +// narrows its destination and reserves vvvv. +func decodedX86RawBF16Instruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || !p.evex || p.mapNumber != 2 || p.pp != 2 || (p.opcode != 0x72 && p.opcode != 0x52) { + return Instr{}, 0, false, nil + } + dot := p.opcode == 0x52 + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw BF16: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.w || !dot && p.upper != 0 || p.vectorLength > 2 { + return fail("reserved EVEX width, source or vector-length field") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero K mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + registerSource := code[p.modRM]>>6 == 3 + if p.broadcast && registerSource { + return fail("broadcast requires a memory source") + } + if mode == 32 && (!registerSource && (p.b != 0 || p.x != 0) || p.r != 0 || p.upper >= 8) { + return fail("extended address or destination register in 32-bit mode") + } + + sourceBytes := 16 << p.vectorLength + sourcePrefix := [...]string{"X", "Y", "Z"}[p.vectorLength] + destinationPrefix := [...]string{"X", "X", "Y"}[p.vectorLength] + if dot { + destinationPrefix = sourcePrefix + } + accessBytes := sourceBytes + if p.broadcast { + accessBytes = 4 + } + source, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix, accessBytes) + if err != nil { + return Instr{}, 0, true, err + } + if mode == 32 && registerSource { + index, _ := amd64VectorRegisterIndex(source.Reg, sourceBytes) + if index > 7 { + return fail("extended source register in 32-bit mode") + } + } + destination := int(code[p.modRM]>>3&7) + p.r*8 + op := [...]Op{"VCVTNEPS2BF16X", "VCVTNEPS2BF16Y", "VCVTNEPS2BF16"}[p.vectorLength] + if dot { + op = "VDPBF16PS" + } + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + args := []Operand{source} + if dot { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", sourcePrefix, p.upper))}) + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationPrefix, destination))}) + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), + x86Encoded: true, + }, p.modRM + consumed, true, nil +} + +func decodeX86RawBF16RIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, ok := decodeX86RawVectorEncoding(code[offset:]) + if !ok || !p.evex || p.mapNumber != 2 || p.pp != 2 || (p.opcode != 0x52 && p.opcode != 0x72) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRM := offset + p.modRM + if len(code) <= modRM || code[modRM]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + } + return x86RawRIPDataThroughDecoder(code, offset, mode, modRM, width, decodedX86RawBF16Instruction, "BF16") +} diff --git a/amd64_raw_bf16_convert_test.go b/amd64_raw_bf16_convert_test.go new file mode 100644 index 000000000..2add8c162 --- /dev/null +++ b/amd64_raw_bf16_convert_test.go @@ -0,0 +1,231 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawBF16Convert(vectorBits, source, destination, mask int, zeroing, broadcast, memory bool) []byte { + if memory { + source = 0 + } + p0 := byte((1-destination/8&1)<<7 | (1-source/16)<<6 | + (1-source/8&1)<<5 | (1-destination/16)<<4 | 2) + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + if memory { + modRM = byte(0x40 | destination&7<<3) + } + code := []byte{0x62, p0, 0x7e, p2, 0x72, modRM} + if memory { + code = append(code, 7) + } + return code +} + +func TestDecodedX86RawBF16ConvertCompleteLLVM22Forms(t *testing.T) { + for vectorBits, names := range []struct { + op Op + source string + dest string + }{ + {"VCVTNEPS2BF16X", "X", "X"}, + {"VCVTNEPS2BF16Y", "Y", "X"}, + {"VCVTNEPS2BF16", "Z", "Y"}, + } { + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zeroing := range []bool{false, true} { + if zeroing && !masked { + continue + } + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + code := encodeX86RawBF16Convert(vectorBits, 20, 21, mask, zeroing, broadcast, memory) + got, length, ok, err := decodedX86RawBF16Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := names.op + if broadcast { + wantOp += ".BCST" + } + if zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s21", names.dest) { + t.Fatalf("decode %x = %+v, want %s -> %s21", code, got, wantOp, names.dest) + } + if !memory && got.Args[0].String() != fmt.Sprintf("%s20", names.source) { + t.Fatalf("decode %x source = %s, want %s20", code, got.Args[0], names.source) + } + if masked && (len(got.Args) != 3 || got.Args[1].String() != "K3") { + t.Fatalf("decode %x lost K3: %+v", code, got) + } + } + } + } + } + } +} + +func TestDecodedX86RawBF16ConvertRejectsReservedFields(t *testing.T) { + valid := encodeX86RawBF16Convert(2, 1, 2, 0, false, false, false) + for _, change := range []struct { + index int + bits byte + }{ + {2, 0x80}, // EVEX.W + {2, 0x08}, // reserved vvvv + {3, 0x20}, // reserved LL + {3, 0x80}, // zeroing without a K mask + {3, 0x10}, // broadcast with register source + } { + bad := append([]byte(nil), valid...) + bad[change.index] ^= change.bits + if _, _, ok, err := decodedX86RawBF16Instruction(bad, 64); !ok || err == nil { + t.Fatalf("accepted reserved BF16 conversion %x: ok=%v err=%v", bad, ok, err) + } + } + if _, _, ok, err := decodedX86RawBF16Instruction(valid[:5], 64); ok && err == nil { + t.Fatal("accepted truncated BF16 conversion") + } +} + +func TestTranslateRawBF16ConvertMaskAndBroadcastTargets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBF16ConvertForms(SB),$0-0\n") + for vectorBits := 0; vectorBits < 3; vectorBits++ { + for _, memory := range []bool{false, true} { + code := encodeX86RawBF16Convert(vectorBits, 1, 2, 3, true, memory, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawBF16ConvertForms": {Name: "rawBF16ConvertForms", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "llvm.x86.vcvtneps2bf16128", + "llvm.x86.vcvtneps2bf16256", + "llvm.x86.avx512bf16.cvtneps2bf16.512", + "llvm.masked.load.v16i32", + } { + if !strings.Contains(ir, want) { + t.Fatalf("BF16 lowering omitted %q", want) + } + } + compileLLVMToObject(t, llc, triple, "amd64-raw-bf16-convert-forms.ll", "amd64-raw-bf16-convert-forms.o", ir) + }) + } +} + +func TestTranslateRawBF16Convert386Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBF16Convert386(SB),$0-0\n") + for vectorBits := 0; vectorBits < 3; vectorBits++ { + code := encodeX86RawBF16Convert(vectorBits, 1, 2, 0, false, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "386", + Sigs: map[string]FuncSig{ + "rawBF16Convert386": {Name: "rawBF16Convert386", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "386-raw-bf16-convert.ll", "386-raw-bf16-convert.o", ir) + }) + } +} + +func TestTranslateRawBF16ConvertGoHighwayRegression(t *testing.T) { + const source = ` +TEXT rawBF16Convert(SB),$0-0 + LONG $0x487ef262; WORD $0x0472; BYTE $0x97 // VCVTNEPS2BF16 (DI)(DX*4), Y0 + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + triple string + goos string + }{ + {"x86_64-apple-darwin", "darwin"}, + {"x86_64-unknown-linux-gnu", "linux"}, + {"x86_64-pc-windows-msvc", "windows"}, + } { + t.Run(target.goos, func(t *testing.T) { + ir, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawBF16Convert": {Name: "rawBF16Convert", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "amd64-raw-bf16-convert.ll", "amd64-raw-bf16-convert.o", ir) + }) + } +} diff --git a/amd64_raw_bf16_dot_test.go b/amd64_raw_bf16_dot_test.go new file mode 100644 index 000000000..af4ca0d1a --- /dev/null +++ b/amd64_raw_bf16_dot_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +func rawBF16Dot(length byte, source1, source2, destination, mask int, memory, broadcast, zero bool) []byte { + code := encodeX86EVEXFMA3(0x52, false, length, broadcast, zero, mask, destination, source1, source2) + code[2] = code[2]&^3 | 2 // EVEX.F3.0F38.W0. + if memory { + code[1] |= 0x60 + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestDecodeRawBF16DotCompleteFormats(t *testing.T) { + for length := byte(0); length < 3; length++ { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := rawBF16Dot(length, 20, 21, 22, mask, memory, broadcast, zero) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "BF16 dot", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op("VDPBF16PS") + if broadcast { + want += ".BCST" + } + if zero { + want += ".Z" + } + prefix := [...]string{"X", "Y", "Z"}[length] + if len(got) != 1 || got[0].Op != want || !got[0].x86Encoded || + got[0].Args[1].String() != prefix+"20" || got[0].Args[len(got[0].Args)-1].String() != prefix+"22" { + t.Fatalf("%x: got %+v, want raw %s %s20 -> %s22", code, got, want, prefix, prefix) + } + wantSource := prefix + "21" + if memory { + width := 16 << length + if broadcast { + width = 4 // Broadcast one BF16 pair, not one 16-bit input. + } + wantSource = fmt.Sprintf("%d(AX)", width) + } + if got[0].Args[0].String() != wantSource { + t.Fatalf("%x: source %s, want %s", code, got[0].Args[0], wantSource) + } + } + } + } + } + } +} + +func TestDecodeRawBF16DotInvalidFormats(t *testing.T) { + for _, change := range []struct { + index int + bits byte + }{ + {2, 0x80}, {2, 0x04}, {3, 0x80}, {3, 0x60}, {3, 0x10}, + } { + code := rawBF16Dot(0, 1, 2, 3, 0, false, false, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid BF16 dot", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved BF16 dot %x", code) + } + } + for _, code := range [][]byte{ + rawBF16Dot(0, 8, 1, 2, 0, false, false, false), + rawBF16Dot(1, 1, 8, 2, 0, false, false, false), + rawBF16Dot(2, 1, 2, 8, 0, false, false, false), + } { + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 BF16 extended register", map[string]bool{}); err == nil { + t.Fatalf("accepted 386 extended register %x", code) + } + } +} + +func TestTranslateRawBF16DotLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT rawBF16Dot(SB),4,$0-0\n") + for length := byte(0); length < 3; length++ { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + code := rawBF16Dot(length, 1, 2, 3, 1, memory, broadcast, true) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + } + } + source.WriteString("RET\n") + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawBF16Dot": {Name: "rawBF16Dot", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.x86.avx512bf16.dpbf16ps.128", "@llvm.x86.avx512bf16.dpbf16ps.256", + "@llvm.x86.avx512bf16.dpbf16ps.512", "@llvm.masked.load.v4i32", "phi i32", "+avx512bf16"} { + if !strings.Contains(ir, want) { + t.Fatalf("missing %s", want) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-bf16-dot.ll", "raw-bf16-dot.o", ir) + cmd := exec.Command(llc, "-O0", "-mtriple="+target.triple, "-filetype=asm", "-o", "-", "-") + cmd.Stdin = strings.NewReader(ir) + assembly, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22 assembly: %v\n%s", err, assembly) + } + if !strings.Contains(string(assembly), "vdpbf16ps") { + t.Fatalf("BF16 intrinsic did not lower to the native instruction:\n%s", assembly) + } + }) + } +} + +func TestRawBF16DotRejectsNamedGoForms(t *testing.T) { + const source = "TEXT namedBF16Dot(SB),4,$0-0\nVDPBF16PS X0, X1, X2\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedBF16Dot": {Name: "namedBF16Dot", Ret: Void}}, + }); err == nil { + t.Fatal("accepted VDPBF16PS absent from Go's named assembler table") + } +} + +func TestDecodeRawBF16RIPLiteral(t *testing.T) { + for length := byte(0); length < 3; length++ { + for _, broadcast := range []bool{false, true} { + for _, dot := range []bool{false, true} { + code := rawBF16Dot(length, 1, 0, 2, 1, false, broadcast, true) + if !dot { + code = encodeX86RawBF16Convert(int(length), 0, 2, 1, true, broadcast, false) + } + code[5] = 0x15 + code = append(code, 1, 0, 0, 0, 0xc3) + width := 16 << length + if broadcast { + width = 4 + } + for index := 0; index < width; index++ { + code = append(code, byte(index+1)) + } + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "BF16 RIP literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Args[0].Kind != OpSym || !got[0].x86RIPLiteral || got[1].Op != OpRET { + t.Fatalf("decoded %x as %+v, want source-local literal and RET", code, got) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated BF16 pool", map[string]bool{}); err == nil { + t.Fatal("accepted truncated source-local BF16 pool") + } + } + } + } +} diff --git a/amd64_raw_binary_float_literal_test.go b/amd64_raw_binary_float_literal_test.go new file mode 100644 index 000000000..1544521f4 --- /dev/null +++ b/amd64_raw_binary_float_literal_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawBinaryFloatLiteral(opcode, pp byte, evex bool, width int, broadcast bool) []byte { + var code []byte + if evex { + code = encodeX86EVEXBinaryFloat(opcode, pp, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), broadcast, true, 7, 0, 0, 0) + } else { + code = encodeX86VEXBinaryFloat(opcode, pp, true, width == 32, false, 0, 0, 0) + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*11+17)) + } + return code +} + +func TestDecodeX86RawBinaryFloatRIPDataCompleteFamily(t *testing.T) { + for _, opcode := range []byte{0x51, 0x58, 0x59, 0x5c, 0x5d, 0x5e, 0x5f} { + for pp := byte(0); pp < 4; pp++ { + if opcode == 0x51 && pp < 2 { + continue + } + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + if pp >= 2 { + widths = []int{16} + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && pp < 2 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + stem := map[byte]string{0x51: "VSQRT", 0x58: "VADD", 0x59: "VMUL", 0x5c: "VSUB", 0x5d: "VMIN", 0x5e: "VDIV", 0x5f: "VMAX"}[opcode] + name := fmt.Sprintf("%s%s/evex%t/%d/bcst%t", stem, []string{"PS", "PD", "SS", "SD"}[pp], evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawBinaryFloatLiteral(opcode, pp, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(stem + []string{"PS", "PD", "SS", "SD"}[pp]) + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } + } +} + +func TestDecodeX86RawBinaryFloatRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawBinaryFloatLiteral(0x58, 0, true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawBinaryFloatRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + index := 0 + for _, opcode := range []byte{0x51, 0x58, 0x59, 0x5c, 0x5d, 0x5e, 0x5f} { + for pp := byte(0); pp < 4; pp++ { + if opcode == 0x51 && pp < 2 { + continue + } + evex := index%2 == 0 + width := 32 + if evex { + width = 64 + } + if pp >= 2 { + width = 16 + } + broadcast := evex && pp < 2 + code := x86RawBinaryFloatLiteral(opcode, pp, evex, width, broadcast) + name := fmt.Sprintf("binaryFloatLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + pools = append(pools, code[len(code)-dataWidth:]) + index++ + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "binary-float-literal.ll", "binary-float-literal.o", ir) + }) + } +} diff --git a/amd64_raw_binary_vector.go b/amd64_raw_binary_vector.go new file mode 100644 index 000000000..98b201886 --- /dev/null +++ b/amd64_raw_binary_vector.go @@ -0,0 +1,80 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Decode the shared two-source vector grammar after the family has selected +// its operation and validated generation/W. The memory source is first in +// Plan 9 order; EVEX masks and broadcast retain their typed semantics. +func decodedX86BinaryVectorOperands(code []byte, p x86RawVectorEncoding, mode int, op Op, laneBits int) (Instr, int, bool, error) { + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s: %s", op, message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.vectorLength > 2 || p.evex && !p.fixed { + return fail("invalid prefix, fixed bit or vector length") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires K1-K7") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.broadcast && (laneBits < 32 || code[p.modRM]>>6 == 3) { + return fail("broadcast requires D/Q memory source") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + + width := []string{"X", "Y", "Z"}[p.vectorLength] + scale := 16 << p.vectorLength + if p.broadcast { + scale = laneBits / 8 + op += ".BCST" + } + if p.zero { + op += ".Z" + } + + var source Operand + var size int + var err error + if p.evex { + source, size, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, size, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + + args := []Operand{ + source, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))}, + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + destination := int(code[p.modRM]>>3&7) + p.r*8 + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destination))}) + + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + instruction := Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), + x86Encoded: true, + } + return instruction, p.modRM + size, true, nil +} diff --git a/amd64_raw_bitcount_test.go b/amd64_raw_bitcount_test.go new file mode 100644 index 000000000..40b62101f --- /dev/null +++ b/amd64_raw_bitcount_test.go @@ -0,0 +1,254 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestX86RawBitCountCompleteGo127EncodingFamily(t *testing.T) { + type form struct { + name string + op Op + code []byte + mem bool + } + var forms []form + for _, family := range []struct { + name string + opcode byte + }{ + {"LZCNT", 0xbd}, + {"TZCNT", 0xbc}, + } { + for _, width := range []struct { + suffix string + prefix []byte + }{ + {"W", []byte{0x66, 0xf3}}, + {"L", []byte{0xf3}}, + {"Q", []byte{0xf3, 0x48}}, + } { + for _, source := range []struct { + name string + modRM byte + mem bool + }{ + {"register", 0xc1, false}, + {"memory", 0x08, true}, + } { + code := append(append([]byte{}, width.prefix...), 0x0f, family.opcode, source.modRM) + forms = append(forms, form{ + name: family.name + width.suffix + "/" + source.name, + op: Op(family.name + width.suffix), code: code, mem: source.mem, + }) + } + } + } + for _, goarch := range []string{"386", "amd64"} { + for _, form := range forms { + if goarch == "386" && strings.Contains(form.name, "Q/") { + continue + } + t.Run(goarch+"/"+form.name, func(t *testing.T) { + fn := Func{} + for _, value := range form.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != form.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want one %s", form.code, got.Instrs, form.op) + } + ins := got.Instrs[0] + wantDestination := AX + if form.mem { + wantDestination = CX + } + if ins.Args[1].Kind != OpReg || ins.Args[1].Reg != wantDestination { + t.Fatalf("decoded destination %#v, want %s", ins.Args[1], wantDestination) + } + if form.mem { + if ins.Args[0].Kind != OpMem || ins.Args[0].Mem.Base != AX { + t.Fatalf("decoded memory source %#v, want (AX)", ins.Args[0]) + } + } else if ins.Args[0].Kind != OpReg || ins.Args[0].Reg != CX { + t.Fatalf("decoded register source %#v, want CX", ins.Args[0]) + } + }) + } + } +} + +func TestX86RawBitCountExtendedRegisters(t *testing.T) { + for _, family := range []struct { + opcode byte + op Op + }{ + {0xbc, "TZCNTQ"}, + {0xbd, "LZCNTQ"}, + } { + for _, test := range []struct { + name string + code []byte + source Operand + dest Reg + }{ + { + name: "extended register", + code: []byte{0xf3, 0x4d, 0x0f, family.opcode, 0xc1}, + source: Operand{Kind: OpReg, Reg: "R9"}, dest: "R8", + }, + { + name: "extended memory base", + code: []byte{0xf3, 0x49, 0x0f, family.opcode, 0x08}, + source: Operand{Kind: OpMem, Mem: MemRef{Base: "R8"}}, dest: CX, + }, + } { + t.Run(string(family.op)+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != family.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", test.code, got.Instrs, family.op) + } + ins := got.Instrs[0] + if ins.Args[0].Kind != test.source.Kind || ins.Args[0].Reg != test.source.Reg || ins.Args[0].Mem != test.source.Mem || ins.Args[1].Reg != test.dest { + t.Fatalf("decoded %#x operands as %#v, want %#v, %s", test.code, ins.Args, test.source, test.dest) + } + }) + } + } +} + +func TestX86RawBitCountCompilesAllForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawbitcount(SB),NOSPLIT,$0-0\n") + for _, opcode := range []byte{0xbc, 0xbd} { + for _, prefix := range [][]byte{{0x66, 0xf3}, {0xf3}, {0xf3, 0x48}} { + if target.goarch == "386" && len(prefix) == 2 && prefix[1] == 0x48 { + continue + } + for _, modRM := range []byte{0xc1, 0x08} { + for _, value := range append(append([]byte{}, prefix...), 0x0f, opcode, modRM) { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawbitcount": {Name: "rawbitcount", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-bitcount.ll", "raw-bitcount.o", ir) + }) + } +} + +func TestX86RawBitCountRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64; required amd64 CI covers this host-inapplicable test") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT rawbitcountsemantics(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVQ $0x12340000, AX + BYTE $0x66; BYTE $0xf3; BYTE $0x0f; BYTE $0xbc; BYTE $0xc0 // TZCNTW AX, AX + MOVQ AX, 0(DI) + SETCS 8(DI) + SETEQ 9(DI) + MOVQ $0x1122334480000000, AX + BYTE $0xf3; BYTE $0x0f; BYTE $0xbd; BYTE $0xc0 // LZCNTL AX, AX + MOVQ AX, 16(DI) + SETCS 24(DI) + SETEQ 25(DI) + MOVQ $0, AX + BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xbc; BYTE $0xc0 // TZCNTQ AX, AX + MOVQ AX, 32(DI) + SETCS 40(DI) + SETEQ 41(DI) + MOVQ $0x100, 48(DI) + BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xbd; BYTE $0x4f; BYTE $0x30 // LZCNTQ 48(DI), CX + MOVQ CX, 56(DI) + SETCS 64(DI) + SETEQ 65(DI) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{ + "rawbitcountsemantics": { + Name: "rawbitcountsemantics", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void rawbitcountsemantics(uint8_t *); +int main(void) { + uint8_t out[72] = {0}; + rawbitcountsemantics(out); + if (*(uint64_t *)(out+0) != UINT64_C(0x12340010)) return 10; + if (out[8] != 1 || out[9] != 0) return 11; + if (*(uint64_t *)(out+16) != 0) return 12; + if (out[24] != 0 || out[25] != 1) return 13; + if (*(uint64_t *)(out+32) != 64) return 14; + if (out[40] != 1 || out[41] != 0) return 15; + if (*(uint64_t *)(out+56) != 55) return 16; + if (out[64] != 0 || out[65] != 0) return 17; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_bitcount_semantics", triple, ir, mainC, runPrefix) +} diff --git a/amd64_raw_bitcount_vector.go b/amd64_raw_bitcount_vector.go new file mode 100644 index 000000000..0f9214b45 --- /dev/null +++ b/amd64_raw_bitcount_vector.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawVectorBitCountForm struct { + op Op + opcode int + w bool +} + +// The six Go 1.27 _yvexpandpd bit-count rows share the same X/Y/Z and +// mask grammar. Opcode and W choose the lane operation; only D/Q rows +// enable scalar-memory broadcast. +var x86RawVectorBitCountForms = [...]x86RawVectorBitCountForm{ + {op: "VPLZCNTD", opcode: 0x44}, + {op: "VPLZCNTQ", opcode: 0x44, w: true}, + {op: "VPOPCNTB", opcode: 0x54}, + {op: "VPOPCNTW", opcode: 0x54, w: true}, + {op: "VPOPCNTD", opcode: 0x55}, + {op: "VPOPCNTQ", opcode: 0x55, w: true}, +} + +func decodedX86RawVectorBitCountInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || !p.evex || p.mapNumber != 2 || p.pp != 1 { + return Instr{}, 0, false, nil + } + var form x86RawVectorBitCountForm + for _, candidate := range x86RawVectorBitCountForms { + if candidate.opcode == p.opcode && candidate.w == p.w { + form = candidate + break + } + } + if form.op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector bit count: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.upper != 0 || p.vectorLength > 2 { + return fail("invalid reserved EVEX encoding bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + spec := amd64PackedBitCountSpecs[form.op] + modRM := code[p.modRM] + registerSource := modRM>>6 == 3 + if p.broadcast && (registerSource || !spec.broadcast) { + return fail("broadcast requires a supported scalar-memory source") + } + if mode == 32 && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + + vectorPrefix := [...]string{"X", "Y", "Z"}[p.vectorLength] + vectorBytes := 16 << p.vectorLength + registerNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && p.vectorLength == 2 && registerNumber >= 8 { + return fail("386 Z destination register exceeds Go's encoder class") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, registerNumber))} + accessBytes := vectorBytes + if p.broadcast { + accessBytes = spec.laneBits / 8 + } + source, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, vectorPrefix, accessBytes) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_bitcount_vector_test.go b/amd64_raw_bitcount_vector_test.go new file mode 100644 index 000000000..d5fc3f82f --- /dev/null +++ b/amd64_raw_bitcount_vector_test.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVectorBitCount(form x86RawVectorBitCountForm, vectorBits, destination, source, mask int, zeroing, broadcast, memory bool) []byte { + if memory { + source = 0 + } + p0 := byte((1-destination/8&1)<<7 | (1-source/16)<<6 | + (1-source/8&1)<<5 | (1-destination/16)<<4 | 2) + p1 := byte(0x7d) + if form.w { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if memory { + modRM = byte(0x40 | (destination&7)<<3) + } + code := []byte{0x62, p0, p1, p2, byte(form.opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestTranslateRawBitCountGorseRegression(t *testing.T) { + const source = `TEXT rawVectorPopcount(SB), $0-0 + LONG $0x48fdf262 + WORD $0xdb55 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawVectorPopcount": {Name: "rawVectorPopcount", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "llvm.ctpop.v8i64") { + t.Fatalf("raw VPOPCNTQ omitted eight-lane population count:\n%s", ir) + } +} + +func TestDecodedX86RawVectorBitCountCompleteGoRows(t *testing.T) { + if len(x86RawVectorBitCountForms) != len(amd64PackedBitCountSpecs) { + t.Fatalf("raw bit-count grammar has %d rows, named lowerer has %d", len(x86RawVectorBitCountForms), len(amd64PackedBitCountSpecs)) + } + seen := make(map[Op]bool) + for _, form := range x86RawVectorBitCountForms { + spec, ok := amd64PackedBitCountSpecs[form.op] + if !ok || seen[form.op] { + t.Fatalf("missing or duplicated bit-count spec for %s", form.op) + } + seen[form.op] = true + for vectorBits, prefix := range [...]string{"X", "Y", "Z"} { + vectorBytes := 16 << vectorBits + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zeroing := range []bool{false, true} { + if zeroing && !masked { + continue + } + for _, broadcast := range []bool{false, true} { + if broadcast && (!memory || !spec.broadcast) { + continue + } + code := encodeX86RawVectorBitCount(form, vectorBits, 21, 20, mask, zeroing, broadcast, memory) + got, length, recognized, err := decodedX86RawVectorBitCountInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s21", prefix) { + t.Fatalf("decode %x = %+v, want %s and %s21", code, got, wantOp, prefix) + } + if masked && (len(got.Args) != 3 || got.Args[1].String() != "K3") || !masked && len(got.Args) != 2 { + t.Fatalf("decode %x = %+v, lost mask", code, got) + } + source := got.Args[0] + if memory { + wantBytes := vectorBytes + if broadcast { + wantBytes = spec.laneBits / 8 + } + if source.Kind != OpMem || source.Mem.Base != AX || source.Mem.Off != int64(wantBytes) { + t.Fatalf("decode %x = %+v, want compressed displacement %d", code, got, wantBytes) + } + } else if source.String() != fmt.Sprintf("%s20", prefix) { + t.Fatalf("decode %x = %+v, want %s20", code, got, prefix) + } + } + } + } + } + } + } +} + +func TestDecodedX86RawVectorBitCountRejectsReservedBits(t *testing.T) { + form := x86RawVectorBitCountForms[5] + base := encodeX86RawVectorBitCount(form, 2, 2, 1, 3, true, false, false) + for _, test := range []struct { + name string + code []byte + }{ + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"vvvv", []byte{base[0], base[1], base[2] &^ 8, base[3], base[4], base[5]}}, + {"V prime", []byte{base[0], base[1], base[2], base[3] &^ 8, base[4], base[5]}}, + {"reserved length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"zero without mask", []byte{base[0], base[1], base[2], base[3] &^ 7, base[4], base[5]}}, + {"register broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawVectorBitCountInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted reserved encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + broadcastWord := encodeX86RawVectorBitCount(x86RawVectorBitCountForms[3], 2, 2, 0, 0, false, true, true) + if _, _, recognized, err := decodedX86RawVectorBitCountInstruction(broadcastWord, 64); !recognized || err == nil { + t.Fatalf("accepted VPOPCNTW broadcast %x", broadcastWord) + } +} + +func TestTranslateRawVectorBitCountLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBitCountFamily(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, form := range x86RawVectorBitCountForms { + for vectorBits := 0; vectorBits < 3; vectorBits++ { + appendBytes(encodeX86RawVectorBitCount(form, vectorBits, 2, 1, 0, false, false, false)) + appendBytes(encodeX86RawVectorBitCount(form, vectorBits, 2, 0, 3, true, false, true)) + if amd64PackedBitCountSpecs[form.op].broadcast { + appendBytes(encodeX86RawVectorBitCount(form, vectorBits, 2, 0, 4, false, true, true)) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawBitCountFamily": {Name: "rawBitCountFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "llvm.ctpop") || !strings.Contains(ir, "llvm.ctlz") { + t.Fatalf("raw vector bit-count family omitted operations:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-vector-bit-count.ll", "raw-vector-bit-count.o", ir) + }) + } +} diff --git a/amd64_raw_blend_imm_literal_test.go b/amd64_raw_blend_imm_literal_test.go new file mode 100644 index 000000000..2b9a1ad49 --- /dev/null +++ b/amd64_raw_blend_imm_literal_test.go @@ -0,0 +1,156 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawImmediatePackedBlendLiteral(spec rawImmediatePackedBlendSpec, vex bool, width int) []byte { + var code []byte + if vex { + code = encodeX86RawVEXImmediatePackedBlend(spec, byte((width-16)/16), 0, 1, 2, 0xa5) + } else { + code = encodeX86RawLegacyImmediatePackedBlend(spec, 0, 2, 0xa5) + } + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x15 // RIP+disp32; destination remains X/Y2. + code = append(code[:modRMIndex+1], append([]byte{1, 0, 0, 0}, code[modRMIndex+1:]...)...) + code = append(code, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*7+3)) + } + return code +} + +func TestDecodeX86RawImmediatePackedBlendRIPDataCompleteFamily(t *testing.T) { + for _, spec := range rawImmediatePackedBlendSpecs { + for _, vex := range []bool{false, true} { + if !vex && spec.legacy == "" { + continue + } + widths := []int{16} + if vex { + widths = append(widths, 32) + } + for _, width := range widths { + name := fmt.Sprintf("%s/vex%t/%d", spec.vector, vex, width) + t.Run(name, func(t *testing.T) { + code := x86RawImmediatePackedBlendLiteral(spec, vex, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := spec.legacy + if vex { + want = spec.vector + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != 0xa5 || + decoded[0].Args[1].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + if !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("source-local data = %x, want %x", decoded[0].x86RIPLiteralData, code[len(code)-width:]) + } + }) + } + } + } +} + +func TestDecodeX86RawImmediatePackedBlendRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { + code[5] = 127 + return code + }, + "overlapping instruction": func(code []byte) []byte { + code[5] = 0 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawImmediatePackedBlendLiteral(rawImmediatePackedBlendSpecs[2], true, 32)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe packed-blend RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawImmediatePackedBlendRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for _, spec := range rawImmediatePackedBlendSpecs { + for _, vex := range []bool{false, true} { + if !vex && spec.legacy == "" { + continue + } + name := string(spec.legacy) + width := 16 + if vex { + name = string(spec.vector) + width = 32 + } + fmt.Fprintf(&source, "TEXT literal%s(SB),$0-0\n", name) + for _, value := range x86RawImmediatePackedBlendLiteral(spec, vex, width) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs["literal"+name] = FuncSig{Name: "literal" + name, Ret: Void} + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(sigs) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(sigs)) + } + for _, fn := range normalized.Funcs { + found := false + for _, instruction := range fn.Instrs { + if !instruction.x86RIPLiteral { + continue + } + if len(instruction.Args) < 2 || !strings.Contains(instruction.Raw, instruction.Args[1].Sym) { + t.Fatalf("%s raw text does not name its materialized source: %+v", fn.Sym, instruction) + } + found = true + } + if !found { + t.Fatalf("%s has no decoded packed-blend literal", fn.Sym) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-blend-imm-literal.ll", "raw-blend-imm-literal.o", ir) + }) + } +} diff --git a/amd64_raw_blend_imm_test.go b/amd64_raw_blend_imm_test.go new file mode 100644 index 000000000..8b980de80 --- /dev/null +++ b/amd64_raw_blend_imm_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +type rawImmediatePackedBlendSpec struct { + opcode byte + legacy Op + vector Op +} + +var rawImmediatePackedBlendSpecs = []rawImmediatePackedBlendSpec{ + {opcode: 0x0e, legacy: "PBLENDW", vector: "VPBLENDW"}, + {opcode: 0x0c, legacy: "BLENDPS", vector: "VBLENDPS"}, + {opcode: 0x0d, legacy: "BLENDPD", vector: "VBLENDPD"}, + {opcode: 0x02, vector: "VPBLENDD"}, +} + +func encodeX86RawLegacyImmediatePackedBlend(spec rawImmediatePackedBlendSpec, source, destination int, immediate byte) []byte { + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + code := []byte{0x66} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x3a, spec.opcode, byte(0xc0|destination&7<<3|source&7), immediate) +} + +func encodeX86RawVEXImmediatePackedBlend(spec rawImmediatePackedBlendSpec, vectorBits byte, first, second, destination int, immediate byte) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | 3 + p1 := byte(^second&15)<<3 | vectorBits<<2 | 1 + return []byte{0xc4, p0, p1, spec.opcode, byte(0xc0 | destination&7<<3 | first&7), immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVBLENDPD(t *testing.T) { + // vblendpd $12, %ymm1, %ymm0, %ymm0; + // vunpcklps %xmm3, %xmm5, %xmm0 + code := []byte{0xc4, 0xe3, 0x7d, 0x0d, 0xc1, 0x0c, 0xc5, 0xd0, 0x14, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VBLENDPD sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VBLENDPD $12, Y1, Y0, Y0 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawImmediatePackedBlendCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawImmediatePackedBlendSpecs { + if spec.legacy != "" { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyImmediatePackedBlend(spec, source, destination, 0xa5) + got, length, ok, err := decodedX86ImmediatePackedBlendInstruction(code, 64) + want := fmt.Sprintf("%s $165, X%d, X%d", spec.legacy, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXImmediatePackedBlend(spec, vectorBits, first, second, destination, 0x5a) + got, length, ok, err := decodedX86ImmediatePackedBlendInstruction(code, 64) + want := fmt.Sprintf("%s $90, %s%d, %s%d, %s%d", spec.vector, vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + if count != 33536 { + t.Fatalf("covered %d immediate packed-blend register encodings, want 33536", count) + } +} + +func TestDecodedX86RawImmediatePackedBlendMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0x66, 0x47, 0x0f, 0x3a, 0x0e, 0x7c, 0x8b, 0x01, 0x7f} + got, length, ok, err := decodedX86ImmediatePackedBlendInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "PBLENDW" || got.Args[0].Imm != 127 || got.Args[2].String() != "X15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x03, 0x75, 0x0d, 0x7c, 0x8b, 0x02, 0xfe} + got, length, ok, err = decodedX86ImmediatePackedBlendInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VBLENDPD" || got.Args[0].Imm != 254 || got.Args[2].String() != "Y1" || got.Args[3].String() != "Y15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + + validLegacy := encodeX86RawLegacyImmediatePackedBlend(rawImmediatePackedBlendSpecs[0], 7, 7, 1) + validVEX := encodeX86RawVEXImmediatePackedBlend(rawImmediatePackedBlendSpecs[0], 0, 7, 7, 7, 1) + for name, invalid := range map[string][]byte{ + "legacy missing immediate": validLegacy[:len(validLegacy)-1], + "vex missing immediate": validVEX[:len(validVEX)-1], + "vex W": {validVEX[0], validVEX[1], validVEX[2] | 0x80, validVEX[3], validVEX[4], validVEX[5]}, + "vex prefix": {validVEX[0], validVEX[1], validVEX[2] &^ 0x01, validVEX[3], validVEX[4], validVEX[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86ImmediatePackedBlendInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + if instruction, _, matched, decodeErr := decodedX86ImmediatePackedBlendInstruction(validVEX, 32); !matched || decodeErr == nil { + t.Fatalf("386 VEX encoding %x decoded as %+v, ok=%v, err=%v", validVEX, instruction, matched, decodeErr) + } +} + +func TestX86ImmediatePackedBlendGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + PBLENDW $7, X6, X7 + BLENDPS $7, X6, X7 + BLENDPD $7, X6, X7 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "VPBLENDW $7, X0, X1, X2", + "VPBLENDD $7, Y0, Y1, Y2", + "VBLENDPS $7, X0, X1, X2", + "VBLENDPD $7, Y0, Y1, Y2", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawImmediatePackedBlendTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_immediate_packed_blend(SB),NOSPLIT,$0-0\n") + for _, spec := range rawImmediatePackedBlendSpecs { + var encodings [][]byte + if spec.legacy != "" { + encodings = append(encodings, encodeX86RawLegacyImmediatePackedBlend(spec, target.registerMax, target.registerMax, 0xa5)) + } + if target.goarch == "amd64" { + encodings = append(encodings, encodeX86RawVEXImmediatePackedBlend(spec, 1, target.registerMax, target.registerMax, target.registerMax, 0x5a)) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "raw_immediate_packed_blend": {Name: "raw_immediate_packed_blend", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-immediate-packed-blend-"+target.name+".ll", "raw-immediate-packed-blend-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_blend_variable.go b/amd64_raw_blend_variable.go new file mode 100644 index 000000000..40fa100a0 --- /dev/null +++ b/amd64_raw_blend_variable.go @@ -0,0 +1,71 @@ +package plan9asm + +import "fmt" + +// VEX variable blends encode the mask register in imm8[7:4], not an ordinary +// immediate. Intel's AVX/AVX2 tables specify 66.0F3A.W0, X/Y widths and no +// EVEX forms. The low nibble is ignored; in 32-bit mode bit 7 is also ignored. +func decodedX86VariableBlendInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 3 { + return Instr{}, 0, false, nil + } + var op Op + for candidate, spec := range amd64VariableBlendSpecs { + if spec.vector && spec.opcode == p.opcode { + op = candidate + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("variable blend: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.w || p.pp != 1 { + return fail("requires VEX.128/256.66.0F3A.W0") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 { + if p.r != 0 || p.x != 0 { + return fail("legacy LES encoding is not a 32-bit VEX instruction") + } + // Three-byte VEX ignores these high register selectors outside + // 64-bit mode, independently of the imm8 mask-register selector. + p.b = 0 + p.upper &= 7 + } + prefix := "X" + if p.vectorLength != 0 { + prefix = "Y" + } + source, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, prefix) + if err != nil { + return Instr{}, 0, true, err + } + immediate := p.modRM + size + if len(code) <= immediate { + return fail("missing mask-register selector byte") + } + mask := int(code[immediate] >> 4) + if mode == 32 { + mask &= 7 + } + reg := func(index int) Operand { + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, index))} + } + args := []Operand{reg(mask), source, reg(p.upper), reg(int(code[p.modRM]>>3&7) + p.r*8)} + return Instr{ + Op: op, Args: args, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, args[0].String(), source.String(), args[2].String(), args[3].String()), + }, immediate + 1, true, nil +} diff --git a/amd64_raw_blend_variable_test.go b/amd64_raw_blend_variable_test.go new file mode 100644 index 000000000..9a85a7613 --- /dev/null +++ b/amd64_raw_blend_variable_test.go @@ -0,0 +1,343 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86VariableBlendTestFamilies = []struct { + legacy, vector string + bits int + legacyCode byte + vectorCode byte +}{ + {"BLENDVPS", "VBLENDVPS", 32, 0x14, 0x4a}, + {"BLENDVPD", "VBLENDVPD", 64, 0x15, 0x4b}, + {"PBLENDVB", "VPBLENDVB", 8, 0x10, 0x4c}, +} + +func TestX86VariableBlendGrammarMatchesGoEncoder(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + legacy, err := os.ReadFile(filepath.Join(root, "asm6.go")) + if err != nil { + t.Fatal(err) + } + vector, err := os.ReadFile(filepath.Join(root, "avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + for _, grammar := range []struct { + text, table string + operands []string + }{ + {string(legacy), "yblendvpd", []string{"Yxr0, Yxm, Yxr"}}, + {string(vector), "_yvblendvpd", []string{"Yxr, Yxm, Yxr, Yxr", "Yyr, Yym, Yyr, Yyr"}}, + } { + pattern := `(?s)var ` + grammar.table + ` = \[\]ytab\{(.*?)\n\}` + rows := regexp.MustCompile(pattern).FindStringSubmatch(grammar.text) + if len(rows) != 2 || strings.Count(rows[1], "argList{") != len(grammar.operands) { + t.Fatalf("Go %s operand grammar changed", grammar.table) + } + for _, operands := range grammar.operands { + if !strings.Contains(rows[1], "argList{"+operands+"}") { + t.Fatalf("Go %s is missing %s", grammar.table, operands) + } + } + } + for _, family := range x86VariableBlendTestFamilies { + for _, vex := range []bool{false, true} { + op, opcode := family.legacy, family.legacyCode + if vex { + op, opcode = family.vector, family.vectorCode + } + spec, ok := amd64VariableBlendSpecs[Op(op)] + if !ok || spec.laneBits != family.bits || spec.vector != vex || spec.opcode != int(opcode) { + t.Fatalf("incorrect typed spec for %s: %+v", op, spec) + } + if !vex { + row := fmt.Sprintf("{A%s, yblendvpd, Pq4, opBytes{0x%02x}}", op, opcode) + if !strings.Contains(string(legacy), row) { + t.Fatalf("Go %s opcode changed", op) + } + continue + } + pattern := `(?s)\{as: A` + op + `, ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(string(vector), -1) + if len(rows) != 1 || rows[0][1] != "_yvblendvpd" { + t.Fatalf("Go %s opcode grammar changed", op) + } + entries := strings.Split(strings.TrimSpace(rows[0][2]), "\n") + if len(entries) != 2 { + t.Fatalf("Go %s must have exactly X/Y forms", op) + } + for i, width := range []int{128, 256} { + want := fmt.Sprintf("avxEscape | vex%d | vex66 | vex0F3A | vexW0, 0x%02X,", width, opcode) + if strings.TrimSpace(entries[i]) != want { + t.Fatalf("Go %s encoding = %s, want %s", op, entries[i], want) + } + } + } + } + if len(amd64VariableBlendSpecs) != 6 { + t.Fatal("variable-blend specs must cover all six Go mnemonics") + } +} + +func TestX86RawVariableBlendEncodingFields(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + for _, length := range []byte{0, 1} { + for selector := 0; selector < 256; selector++ { + code := []byte{0xc4, 0xe3, 0x69 | length<<2, family.vectorCode, 0xd9, byte(selector)} + for _, mode := range []int{32, 64} { + got, size, matched, err := decodedX86VariableBlendInstruction(code, mode) + mask := selector >> 4 + if mode == 32 { + mask &= 7 + } + prefix := "X" + if length != 0 { + prefix = "Y" + } + want := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, mask))}, + {Kind: OpReg, Reg: Reg(prefix + "1")}, + {Kind: OpReg, Reg: Reg(prefix + "2")}, + {Kind: OpReg, Reg: Reg(prefix + "3")}, + } + if err != nil || !matched || size != len(code) || got.Op != Op(family.vector) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %+v", code, mode, got, err, want) + } + } + } + } + } +} + +func TestX86RawVariableBlend386IgnoredRegisterBits(t *testing.T) { + // Intel XED's XMM/YMM_N_32 and XMM/YMM_B_32 tables ignore the high + // vvvv and B selectors. R/X must still distinguish VEX from legacy LES. + for _, family := range x86VariableBlendTestFamilies { + for _, length := range []byte{0, 1} { + for _, modRM := range []byte{0xd9, 0x1b} { + canonical := []byte{0xc4, 0xe3, 0x69 | length<<2, family.vectorCode, modRM, 0x70} + want, _, matched, err := decodedX86VariableBlendInstruction(canonical, 32) + if !matched || err != nil { + t.Fatalf("canonical %x: %v", canonical, err) + } + for bits := 0; bits < 4; bits++ { + code := append([]byte(nil), canonical...) + if bits&1 != 0 { + code[1] &^= 0x20 + } + if bits&2 != 0 { + code[2] &^= 0x40 + } + code[5] = 0xff + got, size, matched, err := decodedX86VariableBlendInstruction(code, 32) + if !matched || err != nil || size != len(code) || !reflect.DeepEqual(got.Args, want.Args) { + t.Fatalf("386 ignored fields %x: %+v %v, want %+v", code, got, err, want) + } + } + } + } + } +} + +func TestX86RawVariableBlendInvalidAndMemoryForms(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + code := []byte{0xc4, 0xe3, 0x69, family.vectorCode, 0xd9, 0x00} + for _, mutate := range []func([]byte) []byte{ + func(b []byte) []byte { b[2] |= 0x80; return b }, // W1 is reserved. + func(b []byte) []byte { b[2] &^= 3; return b }, // No 66 prefix. + func(b []byte) []byte { return b[:4] }, // Missing ModRM. + func(b []byte) []byte { return b[:5] }, // Missing selector. + func(b []byte) []byte { b[4] = 0x04; return b[:5] }, + func(b []byte) []byte { b[4] = 0x80; return b[:5] }, + func(b []byte) []byte { return append([]byte{0x67}, b...) }, + func(b []byte) []byte { return []byte{0x62, 0xf3, 0x6d, 0x08, b[3], b[4], b[5]} }, + } { + invalid := mutate(append([]byte(nil), code...)) + if _, _, matched, err := decodedX86VariableBlendInstruction(invalid, 64); !matched || err == nil { + t.Fatalf("accepted reserved/truncated encoding %x", invalid) + } + } + if _, _, matched, err := decodedX86VariableBlendInstruction(code, 16); !matched || err == nil { + t.Fatal("accepted unsupported mode") + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + memory := []byte{segment.prefix, 0xc4, 0x03, 0x69, family.vectorCode, 0x64, 0x88, 0xf9, 0xf7} + got, size, matched, err := decodedX86VariableBlendInstruction(memory, 64) + want := []Operand{ + {Kind: OpReg, Reg: "X15"}, + {Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, + {Kind: OpReg, Reg: "X2"}, + {Kind: OpReg, Reg: "X12"}, + } + if err != nil || !matched || size != len(memory) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x: %+v %v, want %+v", memory, got, err, want) + } + if _, _, matched, err := decodedX86VariableBlendInstruction(memory, 32); !matched || err == nil { + t.Fatal("accepted legacy LES as a 32-bit VEX instruction") + } + } + rip := []byte{0xc4, 0xe3, 0x69, family.vectorCode, 0x05, 0, 0, 0, 0, 0} + if _, _, matched, err := decodedX86VariableBlendInstruction(rip, 64); !matched || err == nil { + t.Fatal("accepted source-layout-unsafe RIP addressing") + } + if _, size, matched, err := decodedX86VariableBlendInstruction(rip, 32); !matched || err != nil || size != len(rip) { + t.Fatalf("absolute 32-bit addressing: matched=%t size=%d err=%v", matched, size, err) + } + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe2, 0x69, 0x4b, 0xc0, 0}, {0xc4, 0xe3, 0x69, 0x4d, 0xc0, 0}} { + if _, _, matched, _ := decodedX86VariableBlendInstruction(code, 64); matched { + t.Fatalf("matched unrelated instruction %x", code) + } + } +} + +func TestX86RawVariableBlendSourceLocalRIPData(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + for vectorLength, width := range []int{16, 32} { + code := []byte{ + 0xc4, 0xe3, byte(0x69 | vectorLength<<2), family.vectorCode, + 0x05, 0x01, 0, 0, 0, 0x70, 0xc3, + } + for value := 0; value < width; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "variable blend RIP", map[string]bool{}) + if err != nil { + t.Fatalf("%s width %d: %v", family.vector, width, err) + } + if len(decoded) != 2 || decoded[0].Op != Op(family.vector) || + len(decoded[0].x86RIPLiteralData) != width || + decoded[0].Args[1].Kind != OpSym { + t.Fatalf("%s width %d: %#v", family.vector, width, decoded) + } + } + } +} + +func TestX86VariableBlendGoRejectedForms(t *testing.T) { + for _, family := range x86VariableBlendTestFamilies { + for _, form := range []struct { + op string + args []string + }{ + {family.legacy, []string{" X0,X1", " X1,X2,X3", " X0,(AX),(BX)", " X0,Y1,X2", " X0,X1,X16", ".Z X0,X1,X2"}}, + {family.vector, []string{" X0,X1,X2", " (AX),X1,X2,X3", " X0,X1,(AX),X3", " X0,X1,X2,(AX)", + " X0,Y1,X2,X3", " X0,X1,Y2,X3", " X0,X1,X2,Y3", " Z0,Z1,Z2,Z3", " X16,X1,X2,X3", + " X0,X16,X2,X3", " X0,X1,X16,X3", " X0,X1,X2,X16", ".Z X0,X1,X2,X3", ".BCST X0,(AX),X2,X3", ".SAE X0,X1,X2,X3"}}, + } { + for _, args := range form.args { + line := form.op + args + requireX86GoAssemblerResult(t, "amd64", "TEXT bad(SB),4,$0-0\n"+line+"\nRET\n", false) + assertX86VariableBlendRejected(t, "amd64", "x86_64-unknown-linux-gnu", line) + } + } + } +} + +func TestX86RawVariableBlendGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + for _, family := range x86VariableBlendTestFamilies { + for _, width := range []int{0, 16, 32} { + t.Run(fmt.Sprintf("%s/%s/%d", arch, family.vector, width), func(t *testing.T) { + limit := 16 + if arch == "386" { + limit = 8 + } + prefix, op := "X", family.legacy + if width != 0 { + op = family.vector + } + if width == 32 { + prefix = "Y" + } + memories := []string{"(AX)", "-7(BP)(CX*4)", "8192(SP)"} + if arch == "amd64" { + memories = append(memories, "(R8)", "-7(R13)(R15*8)") + } + var source strings.Builder + source.WriteString("TEXT blend(SB),4,$0-0\n") + for reg := 0; reg < limit; reg++ { + if width == 0 { + fmt.Fprintf(&source, "%s X0,X%d,X2\n%s X0,X1,X%d\n", op, reg, op, reg) + } else { + // Vary the four register fields independently, not as a + // redundant Cartesian product of the same grammar. + for field := 0; field < 4; field++ { + args := []string{prefix + "0", prefix + "1", prefix + "2", prefix + "3"} + args[field] = fmt.Sprintf("%s%d", prefix, reg) + fmt.Fprintf(&source, "%s %s\n", op, strings.Join(args, ",")) + } + } + for _, memory := range memories { + if width == 0 { + fmt.Fprintf(&source, "%s X0,%s,X%d\n", op, memory, reg) + } else { + fmt.Fprintf(&source, "%s %s%d,%s,%s2,%s%d\n", op, prefix, reg, memory, prefix, prefix, reg) + } + } + } + source.WriteString("RET\n") + encoderArch := arch + if arch == "386" && width != 0 { + // Go's 386 parser rejects four operands before the shared + // VEX encoder. The same low-register/address encodings + // are valid on i386; decode and compile them there too. + requireX86GoAssemblerResult(t, arch, "TEXT bad(SB),4,$0-0\n"+op+" X0,X1,X2,X3\nRET\n", false) + encoderArch = "amd64" + } + code := assembleX87ControlBytes(t, encoderArch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !x86RawOperandsMatchGoSource(got.Args, want[i].Args) { + t.Fatalf("instruction %d %s operands=%#v, want source %#v", i, got.Op, got.Args, want[i].Args) + } + } + file := x86RawStackObservingFormProbeFile(t, arch, "blend", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{ + Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"blend": {Name: "blend", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-blend.ll", "raw-blend.o", ir) + } + t.Logf("checked %d actual Go encodings", len(want)-1) + }) + } + } + } +} diff --git a/amd64_raw_bmi2_shift.go b/amd64_raw_bmi2_shift.go new file mode 100644 index 000000000..441933038 --- /dev/null +++ b/amd64_raw_bmi2_shift.go @@ -0,0 +1,60 @@ +package plan9asm + +import "fmt" + +// All six Go _ybextrl forms use VEX.L0. Intel XED's hsw-bmi-vex-isa +// table leaves W unconstrained in not64 mode, so Q bytes decode as L on 386. +func decodedX86BMI2ShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.opcode != 0xf7 || p.pp == 0 { + return Instr{}, 0, false, nil + } + + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("BMI2 variable shift: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.vectorLength != 0 { + return fail("requires VEX.L0 encoding") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + + bits := 32 + if p.w && mode == 64 { + bits = 64 + } + var op Op + for name, spec := range amd64BMI2ShiftSpecs { + if spec.bits == bits && spec.prefix == p.pp { + op = Op(name) + break + } + } + + source, size, err := decodedX86VEXRMSource(code[p.modRM:], mode, p.b, p.x, p.segment) + if err != nil { + return Instr{}, 0, true, err + } + count, _ := decodedX86GeneralRegister(p.upper) + destination, _ := decodedX86GeneralRegister(int(code[p.modRM]>>3&7) + p.r*8) + instruction := Instr{ + Op: op, + Args: []Operand{ + {Kind: OpReg, Reg: count}, + source, + {Kind: OpReg, Reg: destination}, + }, + Raw: fmt.Sprintf("%s %s, %s, %s", op, count, source.String(), destination), + } + return instruction, p.modRM + size, true, nil +} diff --git a/amd64_raw_bmi2_shift_test.go b/amd64_raw_bmi2_shift_test.go new file mode 100644 index 000000000..eabee270b --- /dev/null +++ b/amd64_raw_bmi2_shift_test.go @@ -0,0 +1,268 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86BMI2ShiftTestOps = []string{ + "SARXL", "SARXQ", "SHLXL", "SHLXQ", "SHRXL", "SHRXQ", +} + +func TestX86BMI2ShiftGrammarMatchesGoEncoder(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + pattern := `(?s)\{as: A((?:SARX|SHLX|SHRX)[LQ]), ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(string(data), -1) + if len(rows) != 6 || len(amd64BMI2ShiftSpecs) != len(rows) { + t.Fatalf("Go family=%d, typed family=%d", len(rows), len(amd64BMI2ShiftSpecs)) + } + for _, row := range rows { + spec, ok := amd64BMI2ShiftSpecs[row[1]] + prefix := map[int]string{1: "vex66", 2: "vexF3", 3: "vexF2"}[spec.prefix] + wide := strings.HasSuffix(row[1], "Q") + if !ok || row[2] != "_ybextrl" || (spec.bits == 64) != wide { + t.Fatalf("incorrect grammar for %s: %+v", row[1], spec) + } + for _, token := range []string{"vex128", "vex0F38", "0xF7", prefix} { + if !strings.Contains(row[3], token) { + t.Fatalf("Go %s missing %s", row[1], token) + } + } + if strings.Contains(row[3], "vexW1") != wide { + t.Fatalf("incorrect W for %s", row[1]) + } + } + pattern = `(?s)var _ybextrl = \[\]ytab\{(.*?)\n\}` + table := regexp.MustCompile(pattern).FindStringSubmatch(string(data)) + if len(table) != 2 || strings.Count(table[1], "argList{") != 1 || !strings.Contains(table[1], "argList{Yrl, Yml, Yrl}") { + t.Fatal("Go BMI2 shift operand grammar changed") + } +} + +func TestX86RawBMI2ShiftEncodingFields(t *testing.T) { + cases := 0 + for pp, stem := range map[int]string{1: "SHLX", 2: "SARX", 3: "SHRX"} { + for _, wide := range []bool{false, true} { + for source := 0; source < 16; source++ { + for count := 0; count < 16; count++ { + for dst := 0; dst < 16; dst++ { + code := encodeRawScalarMove(false, pp, 0xf7, source, count, dst, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + for _, mode := range []int{32, 64} { + got, n, matched, err := decodedX86BMI2ShiftInstruction(code, mode) + if mode == 32 && (source >= 8 || count >= 8 || dst >= 8) { + if !matched || err == nil { + t.Fatalf("386 accepted %x", code) + } + continue + } + width := "L" + if wide && mode == 64 { + width = "Q" + } + sr, _ := decodedX86GeneralRegister(source) + cr, _ := decodedX86GeneralRegister(count) + dr, _ := decodedX86GeneralRegister(dst) + want := []Operand{ + {Kind: OpReg, Reg: cr}, + {Kind: OpReg, Reg: sr}, + {Kind: OpReg, Reg: dr}, + } + if err != nil || !matched || n != len(code) || got.Op != Op(stem+width) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %s %+v", code, mode, got, err, stem+width, want) + } + } + cases++ + } + } + } + } + } + if cases != 24576 { + t.Fatalf("checked %d encodings, want 24576", cases) + } +} + +func TestX86RawBMI2ShiftInvalidAndSegmentForms(t *testing.T) { + invalid := [][]byte{ + {0xc4, 0xe2, 0x7d, 0xf7, 0xc0}, // VEX.L1. + {0x62, 0xf2, 0x7d, 0x08, 0xf7, 0xc0}, // EVEX. + {0xc4, 0xe2, 0x79, 0xf7}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0xf7, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0xf7, 0x80}, // Missing displacement. + {0xc4, 0xe2, 0x79, 0xf7, 0x05, 0, 0, 0, 0}, // RIP-relative. + {0x67, 0xc4, 0xe2, 0x79, 0xf7, 0x00}, // Address override. + } + for _, code := range invalid { + if _, _, matched, err := decodedX86BMI2ShiftInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x", code) + } + } + if _, _, matched, err := decodedX86BMI2ShiftInstruction([]byte{0xc4, 0xe2, 0x79, 0xf7, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted invalid mode") + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe2, 0x78, 0xf7, 0xc0}, {0xc4, 0xe1, 0x79, 0xf7, 0xc0}} { + if _, _, matched, _ := decodedX86BMI2ShiftInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{segment.prefix, 0xc4, 0x02, 0xb1, 0xf7, 0x64, 0x88, 0xf9} + got, n, matched, err := decodedX86BMI2ShiftInstruction(code, 64) + want := []Operand{ + {Kind: OpReg, Reg: "R9"}, + {Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, + {Kind: OpReg, Reg: "R12"}, + } + if err != nil || !matched || n != len(code) || got.Op != "SHLXQ" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v, want %+v", code, got, err, want) + } + } +} + +func TestX86BMI2ShiftGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86BMI2ShiftTestOps { + args := []string{ + " AX,BX", " X0,AX,BX", " AX,X0,BX", " AX,BX,X0", + " (AX),BX,CX", " AX,BX,(CX)", " $1,BX,CX", + ".Z AX,BX,CX", ".BCST AX,(BX),CX", + } + if arch == "386" { + args = append(args, " R8,BX,CX", " AX,R8,CX", " AX,BX,R8") + } + for _, operands := range args { + source := "TEXT bad(SB),4,$0-0\n" + op + operands + "\nRET\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{ + Goarch: arch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("translator accepted %s%s on %s", op, operands, arch) + } + } + } + } +} + +func TestX86RawBMI2ShiftGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + regs := []string{"AX", "SP", "DI"} + if arch == "amd64" { + regs = append(regs, "R8", "R15") + } + + var source strings.Builder + source.WriteString("TEXT bmi2shift(SB),4,$0-0\n") + for _, op := range x86BMI2ShiftTestOps { + for _, reg := range regs { + for _, input := range []string{reg, "(BX)", "-7(BP)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "%s %s,%s,%s\n", op, reg, input, reg) + } + } + } + source.WriteString("RET\n") + + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if arch == "386" && strings.HasSuffix(string(want[i].Op), "Q") { + want[i].Op = Op(strings.TrimSuffix(string(want[i].Op), "Q") + "L") + } + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + + file := x86RawStackObservingFormProbeFile(t, arch, "bmi2shift", code) + triples := []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{ + Goarch: arch, + TargetTriple: triple, + Sigs: map[string]FuncSig{ + "bmi2shift": {Name: "bmi2shift", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-bmi2-shift.ll", "raw-bmi2-shift.o", ir) + } + t.Logf("checked %d Go encodings", len(want)-1) + }) + } +} + +func TestX86BMI2Shift386IgnoresW(t *testing.T) { + for _, op := range []string{"SARXQ", "SHLXQ", "SHRXQ"} { + t.Run(op, func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT shift(SB),4,$0-0\n"+op+" AX,(BX),CX\nRET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"shift": {Name: "shift", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "load i32, ptr") || !strings.Contains(ir, ", 31\n") || + strings.Contains(ir, "ashr i64") || strings.Contains(ir, "shl i64") || strings.Contains(ir, "lshr i64") { + t.Fatalf("386 %s must read exactly four bytes:\n%s", op, ir) + } + }) + } +} diff --git a/amd64_raw_broadcast_block.go b/amd64_raw_broadcast_block.go new file mode 100644 index 000000000..700bccc9d --- /dev/null +++ b/amd64_raw_broadcast_block.go @@ -0,0 +1,90 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawBlockBroadcastForm struct { + op Op + sourceBytes int + minLength int + allowX bool +} + +// Go 1.27's complete EVEX packed-block broadcast rows. The corresponding +// VEX F128/I128 memory forms have a separate two-form decoder. +var x86RawBlockBroadcastForms = map[[2]int]x86RawBlockBroadcastForm{ + {0x19, 0}: {op: "VBROADCASTF32X2", sourceBytes: 8, minLength: 1, allowX: true}, + {0x59, 0}: {op: "VBROADCASTI32X2", sourceBytes: 8, minLength: 0, allowX: true}, + {0x1a, 0}: {op: "VBROADCASTF32X4", sourceBytes: 16, minLength: 1}, + {0x1a, 1}: {op: "VBROADCASTF64X2", sourceBytes: 16, minLength: 1}, + {0x5a, 0}: {op: "VBROADCASTI32X4", sourceBytes: 16, minLength: 1}, + {0x5a, 1}: {op: "VBROADCASTI64X2", sourceBytes: 16, minLength: 1}, + {0x1b, 0}: {op: "VBROADCASTF32X8", sourceBytes: 32, minLength: 2}, + {0x1b, 1}: {op: "VBROADCASTF64X4", sourceBytes: 32, minLength: 2}, + {0x5b, 0}: {op: "VBROADCASTI32X8", sourceBytes: 32, minLength: 2}, + {0x5b, 1}: {op: "VBROADCASTI64X4", sourceBytes: 32, minLength: 2}, +} + +// decodedX86RawBlockBroadcastInstruction decodes every EVEX block-broadcast +// shape, including register-backed X2 sources and K merge/zero masking. +func decodedX86RawBlockBroadcastInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 { + return Instr{}, 0, false, nil + } + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawBlockBroadcastForms[[2]int{p.opcode, w}] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX block broadcast: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.broadcast || p.upper != 0 || + p.vectorLength < form.minLength || p.vectorLength > 2 { + return fail("invalid EVEX fixed, address, broadcast, vvvv, or vector-length bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + modRM := code[p.modRM] + if modRM>>6 == 3 && !form.allowX { + return fail("this block width requires a memory source") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && p.vectorLength == 2 && destinationNumber >= 8 { + return fail("386 Z-register exceeds the Go assembler frontend's register class") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + source, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, "X", form.sourceBytes, + ) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.zero { + op += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))}) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_broadcast_block_test.go b/amd64_raw_broadcast_block_test.go new file mode 100644 index 000000000..dc5904620 --- /dev/null +++ b/amd64_raw_broadcast_block_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawBlockBroadcast(opcode, widthBit, vectorBits, mask int, zeroing, registerSource bool) []byte { + p1 := byte(0x7d) + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0x40) // disp8(AX), destination 0. + if registerSource { + modRM = 0xc1 // X1, destination 0. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if !registerSource { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawBlockBroadcastSeBiShogunRegression(t *testing.T) { + // Go disassembles the instruction as VBROADCASTI32X4 RIP+1, Z1. + code := []byte{0x62, 0xf2, 0x7d, 0x48, 0x5a, 0x0d, 1, 0, 0, 0, 0xc3} + for index := 0; index < 16; index++ { + code = append(code, byte(index+1)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "onesCountInt32 AVX512 block broadcast", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VBROADCASTI32X4" || + decoded[0].Args[1].Reg != "Z1" || !decoded[0].x86RIPLiteral { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawBlockBroadcastCompleteGoFamily(t *testing.T) { + if len(x86RawBlockBroadcastForms) != 10 { + t.Fatalf("modeled %d EVEX block-broadcast rows, want ten", len(x86RawBlockBroadcastForms)) + } + for encoding, form := range x86RawBlockBroadcastForms { + for vectorBits := form.minLength; vectorBits <= 2; vectorBits++ { + for _, masking := range []struct { + mask, zero int + }{ + {}, {mask: 3}, {mask: 7, zero: 1}, + } { + for _, registerSource := range []bool{false, true} { + if registerSource && !form.allowX { + continue + } + name := fmt.Sprintf("%s/length%d/mask%d/zero%d/reg%t", form.op, vectorBits, masking.mask, masking.zero, registerSource) + t.Run(name, func(t *testing.T) { + code := encodeX86RawBlockBroadcast(encoding[0], encoding[1], vectorBits, masking.mask, masking.zero != 0, registerSource) + got, length, ok, err := decodedX86RawBlockBroadcastInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if masking.zero != 0 { + wantOp += ".Z" + } + wantDestination := fmt.Sprintf("%s0", [...]string{"X", "Y", "Z"}[vectorBits]) + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != wantDestination || + registerSource && got.Args[0].String() != "X1" || + len(got.Args) != 2+map[bool]int{true: 1}[masking.mask != 0] { + t.Fatalf("decode %x = %+v, want %s ... %s", code, got, wantOp, wantDestination) + } + }) + } + } + } + } +} + +func TestDecodeX86RawBlockBroadcastRIPLiteralCompleteFamily(t *testing.T) { + for encoding, form := range x86RawBlockBroadcastForms { + for vectorBits := form.minLength; vectorBits <= 2; vectorBits++ { + code := encodeX86RawBlockBroadcast(encoding[0], encoding[1], vectorBits, 1, false, false) + code = code[:6] + code[5] = 0x05 // RIP-relative source, destination 0. + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < form.sourceBytes; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/length%d", form.op, vectorBits) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != form.sourceBytes || got[0].Op != form.op { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawBlockBroadcastRejectsInvalidForms(t *testing.T) { + base := encodeX86RawBlockBroadcast(0x5a, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x10 }, // no EVEX.b broadcast + func(code []byte) { code[3] |= 0x80 }, // zeroing needs a mask + func(code []byte) { code[3] |= 0x20 }, // vector length 3 reserved + func(code []byte) { code[2] &^= 0x08 }, // reserved vvvv + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawBlockBroadcastInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + registerSource := encodeX86RawBlockBroadcast(0x5a, 0, 2, 0, false, true) + if _, _, ok, err := decodedX86RawBlockBroadcastInstruction(registerSource, 64); !ok || err == nil { + t.Fatalf("accepted m128-only register source %x: ok=%v err=%v", registerSource, ok, err) + } +} + +func TestTranslateX86RawBlockBroadcastLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawBlockBroadcast(SB),$0-0\n") + for encoding, form := range x86RawBlockBroadcastForms { + for vectorBits := form.minLength; vectorBits <= 2; vectorBits++ { + for _, masking := range []struct { + mask int + zero bool + }{ + {}, {mask: 3}, {mask: 7, zero: true}, + } { + code := encodeX86RawBlockBroadcast(encoding[0], encoding[1], vectorBits, masking.mask, masking.zero, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawBlockBroadcast": {Name: "rawBlockBroadcast", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-block-broadcast.ll", "raw-block-broadcast.o", ir) + }) + } +} diff --git a/amd64_raw_broadcast_literal.go b/amd64_raw_broadcast_literal.go new file mode 100644 index 000000000..8f8cc52fc --- /dev/null +++ b/amd64_raw_broadcast_literal.go @@ -0,0 +1,1433 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "strings" + + "golang.org/x/arch/x86/x86asm" +) + +type x86RawLiteralRange struct { + first int + last int + source int +} + +// decodeX86RawPackedBroadcastLiteral resolves a RIP-relative packed scalar +// broadcast only when its source bytes live in the same raw directive group. +// The caller must still prove that those bytes are not reachable instructions. +// A displacement into another TEXT, a missing constant pool, and segment- or +// address-size-relative loads stay on the rejecting decoder path. +func decodeX86RawPackedBroadcastLiteral(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + // A segment or address-size prefix changes the effective source. + switch code[offset] { + case 0x64, 0x65, 0x67: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + modRMIndex := -1 + laneBytes := 0 + if len(code) >= offset+5 && code[offset] == 0xc4 && code[offset+1]&0x1f == 2 { + modRMIndex = offset + 4 + switch code[offset+3] { + case 0x78: + laneBytes = 1 + case 0x79: + laneBytes = 2 + case 0x58: + laneBytes = 4 + case 0x59: + laneBytes = 8 + } + } else if len(code) >= offset+6 && code[offset] == 0x62 && code[offset+1]&0x0f == 2 { + modRMIndex = offset + 5 + switch code[offset+4] { + case 0x78: + laneBytes = 1 + case 0x79: + laneBytes = 2 + case 0x58: + laneBytes = 4 + case 0x59: + laneBytes = 8 + } + } + if laneBytes == 0 || modRMIndex < 0 || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + patched, value, literal, err := x86RawRIPLiteral(code, offset, modRMIndex, laneBytes) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + length := len(patched) + instruction, consumed, ok, err := decodedX86VEXPackedBroadcastInstruction(patched, mode) + if err != nil || !ok || consumed != length { + if err == nil { + err = fmt.Errorf("RIP-relative packed broadcast did not match its VEX/EVEX grammar") + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + + setX86RawRIPLiteral(&instruction, value) + return instruction, length, literal, true, nil +} + +// decodeX86RawVMOVIntegerLiteral covers all memory-to-X VMOVW/VMOVD/VMOVQ VEX and +// EVEX encodings, including VMOVQ's alternate F3 7E encoding. The ordinary +// decoder validates the full grammar after the fixed-length ModRM rewrite. +func decodeX86RawVMOVIntegerLiteral(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + switch code[offset] { + case 0x64, 0x65, 0x67: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 0 + if len(code) >= offset+4 && code[offset] == 0xc5 { + modRMIndex = offset + 3 + width = x86RawVMOVLoadWidth(code[offset+2], code[offset+1]&3, false, false) + } else if len(code) >= offset+5 && code[offset] == 0xc4 && code[offset+1]&0x1f == 1 { + modRMIndex = offset + 4 + width = x86RawVMOVLoadWidth(code[offset+3], code[offset+2]&3, code[offset+2]&0x80 != 0, false) + } else if len(code) >= offset+6 && code[offset] == 0x62 { + modRMIndex = offset + 5 + if code[offset+1]&0x0f == 1 { + width = x86RawVMOVLoadWidth(code[offset+4], code[offset+2]&3, code[offset+2]&0x80 != 0, true) + } else if code[offset+1]&0x0f == 5 && code[offset+4] == 0x6e && code[offset+2]&3 == 1 { + width = 2 + } + } + if width == 0 || modRMIndex < 0 || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + + patched, value, literal, err := x86RawRIPLiteral(code, offset, modRMIndex, width) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, consumed, ok, err := decodedX86VMOVQInstruction(patched, mode) + if err != nil || !ok || consumed != len(patched) || + len(instruction.Args) != 2 || instruction.Args[0].Kind != OpMem { + if err == nil { + err = fmt.Errorf("RIP-relative VMOVW/VMOVD/VMOVQ load did not match its VEX/EVEX grammar") + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + setX86RawRIPLiteral(&instruction, value) + return instruction, len(patched), literal, true, nil +} + +// decodeX86RawScalarFloatBroadcastLiteral resolves VBROADCASTSS/SD loads +// through the same source-local, unreachable constant-pool proof. +func decodeX86RawScalarFloatBroadcastLiteral(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || + (p.opcode != 0x18 && p.opcode != 0x19) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 4 + if p.opcode == 0x19 { + width = 8 + } + patched, value, literal, err := x86RawRIPLiteral(code, offset, modRMIndex, width) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, consumed, ok, err := decodedX86ScalarBroadcastInstruction(patched, mode) + if err != nil || !ok || consumed != len(patched) || + len(instruction.Args) < 2 || instruction.Args[0].Kind != OpMem { + if err == nil { + err = fmt.Errorf("RIP-relative scalar float broadcast did not match its VEX/EVEX grammar") + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + setX86RawRIPLiteral(&instruction, value) + return instruction, len(patched), literal, true, nil +} + +func x86RawVMOVLoadWidth(opcode, pp byte, width64, evex bool) int { + if opcode == 0x6e && pp == 1 { + if width64 { + return 8 + } + return 4 + } + if opcode == 0x7e && pp == 2 && (width64 == evex) { + return 8 + } + return 0 +} + +func x86RawRIPLiteral(code []byte, offset, modRMIndex, width int) ([]byte, int64, x86RawLiteralRange, error) { + patched, data, literal, err := x86RawRIPBytes(code, offset, modRMIndex, width) + if err != nil { + return nil, 0, x86RawLiteralRange{}, err + } + if width > 8 { + return nil, 0, x86RawLiteralRange{}, fmt.Errorf("%d-byte RIP literal does not fit an immediate", width) + } + var value uint64 + for index, b := range data { + value |= uint64(b) << (8 * index) + } + return patched, int64(value), literal, nil +} + +func x86RawRIPBytes(code []byte, offset, modRMIndex, width int) ([]byte, []byte, x86RawLiteralRange, error) { + return x86RawRIPBytesWithSuffix(code, offset, modRMIndex, width, 0) +} + +func x86RawRIPBytesWithSuffix(code []byte, offset, modRMIndex, width, trailingBytes int) ([]byte, []byte, x86RawLiteralRange, error) { + if trailingBytes < 0 || len(code) < modRMIndex+5+trailingBytes { + return nil, nil, x86RawLiteralRange{}, fmt.Errorf("truncated RIP-relative literal") + } + length := modRMIndex + 5 + trailingBytes - offset + displacement := int32(binary.LittleEndian.Uint32(code[modRMIndex+1 : modRMIndex+5])) + first := offset + length + int(displacement) + last := first + width + if width <= 0 || first < 0 || last > len(code) || last < first { + return nil, nil, x86RawLiteralRange{}, fmt.Errorf("RIP-relative literal [%d,%d) is outside directive group", first, last) + } + + // A base-register disp32 has the same byte length as RIP+disp32, allowing + // the existing VEX/EVEX grammar to validate every other encoding field. + patched := append([]byte(nil), code[offset:offset+length]...) + patched[modRMIndex-offset] = patched[modRMIndex-offset]&0x38 | 0x80 + data := append([]byte(nil), code[first:last]...) + return patched, data, x86RawLiteralRange{first: first, last: last}, nil +} + +// decodeX86RawPackedMoveRIPData handles source-local X/Y/Z memory loads for +// the complete packed float and VEX integer move decoder families. +func decodeX86RawPackedMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + packedFloat := (p.opcode == 0x10 || p.opcode == 0x28) && p.pp <= 1 + packedInteger := p.opcode == 0x6f && !p.evex && (p.pp == 1 || p.pp == 2) + if !matched || p.mapNumber != 1 || (!packedFloat && !packedInteger) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedMoveInstruction, "packed move", + ) +} + +// decodeX86RawEVEXPackedIntegerMoveRIPData covers the six EVEX VMOVDQA/DQU +// load opcodes across X/Y/Z widths and optional K masks. +func decodeX86RawEVEXPackedIntegerMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 1 || p.opcode != 0x6f || + (p.pp != 1 && p.pp != 2 && p.pp != 3) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86EVEXPackedIntegerMoveInstruction, "EVEX packed integer move", + ) +} + +// decodeX86RawVBROADCAST128RIPData resolves both VEX.256 F128/I128 +// m128-to-Y forms through their shared source-memory grammar. +func decodeX86RawVBROADCAST128RIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.evex || p.mapNumber != 2 || p.pp != 1 || + p.vectorLength != 1 || (p.opcode != 0x1a && p.opcode != 0x5a) || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, 16, + decodedX86VBROADCAST128Instruction, "VBROADCASTF128/VBROADCASTI128", + ) +} + +// decodeX86RawPackedLogicalRIPData covers VEX VPAND/ANDN/OR/XOR and their +// EVEX D/Q counterparts. EVEX.b reads one scalar lane; other forms read the +// full X/Y/Z-width vector from the same unreachable source-local pool. +func decodeX86RawPackedLogicalRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + switch p.opcode { + case 0xdb, 0xdf, 0xeb, 0xef: + default: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + decoder := x86RawInstructionDecoder(decodedX86VEXPackedIntegerLogicalInstruction) + family := "VEX packed integer logical" + if p.evex { + decoder = decodedX86EVEXPackedLogicalInstruction + family = "EVEX packed logical" + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, decoder, family, + ) +} + +// decodeX86RawFloatLogicalRIPData covers VEX/EVEX VAND/ANDN/OR/XOR PS/PD +// reads from an unreachable source-local X/Y/Z-width constant pool. +func decodeX86RawFloatLogicalRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.pp > 1 || + p.opcode < 0x54 || p.opcode > 0x57 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + decoder := x86RawInstructionDecoder(decodedX86VEXPackedFloatLogicalInstruction) + if p.evex { + decoder = decodedX86RawEVEXFloatLogicalInstruction + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, decoder, "packed floating logical", + ) +} + +// decodeX86RawHorizontalFloatRIPData covers VHADD/VHSUB/VADDSUB PS/PD +// VEX X/Y reads from unreachable source-local vector constants. +func decodeX86RawHorizontalFloatRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.evex || p.mapNumber != 1 || + (p.pp != 1 && p.pp != 3) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + switch p.opcode { + case 0x7c, 0x7d, 0xd0: + default: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 16 + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if matched && !p.evex && p.mapNumber == 3 && p.pp == 1 && + p.segment == "" && !p.addressOverride { + switch p.opcode { + case 0x02, 0x0c, 0x0d, 0x0e: + modRMIndex = offset + p.modRM + width <<= p.vectorLength + } + } + if modRMIndex < 0 && code[offset] == 0x66 { + i := offset + 1 + if i < len(code) && code[i]&0xf0 == 0x40 { + i++ // Optional REX extension for the legacy X registers. + } + if len(code) >= i+4 && code[i] == 0x0f && code[i+1] == 0x3a { + switch code[i+2] { + case 0x0c, 0x0d, 0x0e: + modRMIndex = i + 3 + } + } + } + if modRMIndex < 0 || len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86ImmediatePackedBlendInstruction, "immediate packed blend", + ) +} + +// decodeX86RawCLMULRIPData covers legacy PCLMULQDQ and all VEX/EVEX +// VPCLMULQDQ widths. The imm8 follows the source displacement. +func decodeX86RawCLMULRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 16 + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if matched && p.mapNumber == 3 && p.pp == 1 && p.opcode == 0x44 && + p.segment == "" && !p.addressOverride && p.vectorLength <= 2 { + modRMIndex = offset + p.modRM + width <<= p.vectorLength + } + if modRMIndex < 0 && code[offset] == 0x66 { + i := offset + 1 + if i < len(code) && code[i]&0xf0 == 0x40 { + i++ + } + if len(code) >= i+4 && code[i] == 0x0f && code[i+1] == 0x3a && code[i+2] == 0x44 { + modRMIndex = i + 3 + } + } + if modRMIndex < 0 || len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawCLMULInstruction, "carryless multiply", + ) +} + +// decodeX86RawMaskBlendRIPData folds source-local vector constants, or one +// D/Q broadcast lane, for all six EVEX mask-blend operations. +func decodeX86RawMaskBlendRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + p.opcode < 0x64 || p.opcode > 0x66 || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawMaskBlendInstruction, "EVEX mask blend", + ) +} + +// decodeX86RawScalarMoveRIPData covers memory-to-X VMOVSS/VMOVSD VEX and +// EVEX encodings plus raw EVEX VMOVSH. The typed scalar decoder enforces +// reserved fields and masks. +func decodeX86RawScalarMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + half := p.evex && p.mapNumber == 5 && p.pp == 2 + if !matched || (p.mapNumber != 1 && !half) || p.opcode != 0x10 || + (p.pp != 2 && p.pp != 3) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 4 + if half { + width = 2 + } else if p.pp == 3 { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86ScalarMoveInstruction, "scalar move", + ) +} + +// decodeX86RawMinMaxRIPData covers the complete VEX/EVEX packed integer +// min/max family. EVEX.b reads one D/Q lane; other forms read a full vector. +func decodeX86RawMinMaxRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + for _, spec := range amd64PackedIntegerMinMaxSpecs { + if p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + continue + } + if !p.evex && spec.laneBits == 64 { + continue + } + if p.evex && spec.laneBits >= 32 && p.w != (spec.laneBits == 64) { + continue + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = spec.laneBits / 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedIntegerMinMaxInstruction, "packed integer min/max", + ) + } + return Instr{}, 0, x86RawLiteralRange{}, false, nil +} + +// decodeX86RawPackedCompareRIPData covers all VPCMPEQ/GT B/W/D/Q source +// memory forms. The ordinary decoder checks the EVEX mask-result grammar. +func decodeX86RawPackedCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + for _, spec := range amd64PackedIntegerCompareSpecs { + if p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + continue + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = spec.laneBits / 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedIntegerCompareInstruction, "packed integer compare", + ) + } + return Instr{}, 0, x86RawLiteralRange{}, false, nil +} + +// decodeX86RawVariableShiftRIPData covers the VEX and EVEX per-lane variable +// shift family, including scalar D/Q memory broadcast in EVEX encodings. +func decodeX86RawVariableShiftRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + for _, spec := range amd64PerLaneVariableShiftSpecs { + if p.opcode != int(spec.opcode) || p.w != (spec.laneBits != 32) || + !p.evex && !spec.vex { + continue + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = spec.laneBits / 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PerLaneVariableShiftInstruction, "per-lane variable shift", + ) + } + return Instr{}, 0, x86RawLiteralRange{}, false, nil +} + +// decodeX86RawVPSHUFBRIPData covers every VEX and EVEX vector-width load +// accepted by Go's VPSHUFB table. The instruction has no scalar broadcast. +func decodeX86RawVPSHUFBRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || p.opcode != 0 || + p.w || p.broadcast || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86VPSHUFBInstruction, "VPSHUFB", + ) +} + +// decodeX86RawPackedArithmeticRIPData covers the complete VEX/EVEX VPADD +// and VPSUB families, including their D/Q scalar memory broadcasts. +func decodeX86RawPackedArithmeticRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + properties, recognized := decodedX86PackedIntegerArithmeticOps[byte(p.opcode)] + if !recognized { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = properties.laneBytes + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedIntegerArithmeticInstruction, "packed integer arithmetic", + ) +} + +// decodeX86RawFMA3RIPData handles all packed and scalar FMA3 encodings. +// Scalar operations read one lane; packed EVEX.b memory forms broadcast it. +func decodeX86RawFMA3RIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + half := p.evex && p.mapNumber == 6 + if !matched || p.mapNumber != 2 && !half || p.pp != 1 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + spec, recognized := decodedX86VEXFMA3Ops[byte(p.opcode)] + if !recognized { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if spec.scalar || p.evex && p.broadcast { + width = 4 + if p.w { + width = 8 + } + if half { + width = 2 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86VEXFMA3Instruction, "FMA3", + ) +} + +// decodeX86RawBinaryFloatRIPData covers VADD/MUL/SUB/MIN/DIV/MAX packed and +// scalar forms, including raw AVX-512 FP16, plus scalar VSQRT and packed +// memory broadcast. +func decodeX86RawBinaryFloatRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + half := p.evex && p.mapNumber == 5 && (p.pp == 0 || p.pp == 2) + if !matched || (p.mapNumber != 1 && !half) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + switch p.opcode { + case 0x58, 0x59, 0x5c, 0x5d, 0x5e, 0x5f: + case 0x51: + if p.pp < 2 && !half { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + default: + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if half { + if p.pp == 2 || p.broadcast { + width = 2 + } + } else if p.pp >= 2 || p.evex && p.broadcast { + width = 4 + if p.pp == 1 || p.pp == 3 { + width = 8 + } + } + decoder := decodedX86VEXBinaryFloatInstruction + if half && p.opcode == 0x51 && p.pp == 0 { + decoder = decodedX86SameWidthConversionInstruction + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decoder, "binary floating", + ) +} + +// decodeX86RawScalarFlagCompareRIPData covers VCOMIS{S,D} and +// VUCOMIS{S,D} VEX/EVEX memory sources. EVEX.b remains register-only SAE. +func decodeX86RawScalarFlagCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || (p.opcode != 0x2e && p.opcode != 0x2f) || + (p.pp != 0 && p.pp != 1) || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 4 + if p.pp == 1 { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86ScalarFlagCompareInstruction, "scalar flag compare", + ) +} + +// decodeX86RawVectorFloatCompareRIPData accounts for VCMP's trailing imm8 +// when resolving the RIP displacement and its second Plan 9 operand. +func decodeX86RawVectorFloatCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 1 || p.opcode != 0xc2 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.pp >= 2 || p.evex && p.broadcast { + width = 4 + if p.pp == 1 || p.pp == 3 { + width = 8 + } + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86VectorFloatCompareInstruction, "vector floating compare", + ) +} + +// decodeX86RawPackedMultiplyRIPData covers the five packed word/D/Q multiply +// opcode rows, including EVEX D/Q memory broadcast and masked widths. +func decodeX86RawPackedMultiplyRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.pp != 1 || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + form, recognized := x86PackedMultiplyFormFor(byte(p.mapNumber), byte(p.opcode)) + if !recognized || p.evex && p.broadcast && form.broadcastByte == 0 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = form.broadcastByte + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86PackedMultiplyInstruction, "packed multiply", + ) +} + +// decodeX86RawPackedAbsRIPData resolves source-local X/Y/Z VPABS loads, +// including the D/Q EVEX memory-broadcast lanes. +func decodeX86RawPackedAbsRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 2 || p.pp != 1 || p.opcode < 0x1c || p.opcode > 0x1f || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.evex && p.broadcast { + width = [...]int{1, 2, 4, 8}[p.opcode-0x1c] + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawPackedAbsInstruction, "VPABS", + ) +} + +// decodeX86RawVEXRoundRIPData accounts for VROUND's trailing imm8 while +// resolving packed or scalar source-local constant loads. +func decodeX86RawVEXRoundRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.mapNumber != 3 || p.pp != 1 || p.opcode < 0x08 || p.opcode > 0x0b || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.opcode == 0x0a { + width = 4 + } else if p.opcode == 0x0b { + width = 8 + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawVEXRoundInstruction, "VROUND", + ) +} + +// decodeX86RawDuplicateMoveRIPData covers legacy, VEX and EVEX MOVDDUP, +// MOVSLDUP and MOVSHDUP. The X-width DDUP form reads only one 64-bit lane; +// all other forms read the complete source vector. +func decodeX86RawDuplicateMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := -1 + width := 0 + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if matched && p.mapNumber == 1 && p.segment == "" && !p.addressOverride { + double := p.pp == 3 && p.opcode == 0x12 + single := p.pp == 2 && (p.opcode == 0x12 || p.opcode == 0x16) + if double || single { + modRMIndex = offset + p.modRM + width = 16 << p.vectorLength + if double && p.vectorLength == 0 { + width = 8 + } + } + } else { + i := offset + prefix := code[i] + if prefix == 0xf2 || prefix == 0xf3 { + i++ + if i < len(code) && code[i]&0xf0 == 0x40 { + i++ + } + if len(code) >= i+3 && code[i] == 0x0f && + (prefix == 0xf2 && code[i+1] == 0x12 || + prefix == 0xf3 && (code[i+1] == 0x12 || code[i+1] == 0x16)) { + modRMIndex = i + 2 + width = 16 + if prefix == 0xf2 { + width = 8 + } + } + } + } + if modRMIndex < 0 || len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86DuplicateMoveInstruction, "duplicate move", + ) +} + +// decodeX86RawVectorByteShiftRIPData resolves EVEX X/Y/Z memory forms of +// VPSLLDQ and VPSRLDQ, accounting for the trailing shift-count imm8. +func decodeX86RawVectorByteShiftRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || p.opcode != 0x73 || + p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + group := int(code[modRMIndex]>>3) & 7 + if group != 3 && group != 7 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.pp != 1 || p.segment != "" || p.addressOverride || + p.evex && p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + if !(p.mapNumber == 1 && p.opcode == 0xf5 || + p.mapNumber == 2 && p.opcode == 0x04) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + decode := x86RawInstructionDecoder(decodedX86VEXPackedMADDInstruction) + if p.evex { + decode = decodedX86EVEXPackedMADDInstruction + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || p.evex || p.mapNumber != 3 || p.pp != 1 || p.w || + p.segment != "" || p.addressOverride || p.vectorLength > 1 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + if p.opcode != 0x4a && p.opcode != 0x4b && p.opcode != 0x4c { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, 16<= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 || + p.opcode != int(opcode) || p.segment != "" || p.addressOverride || + p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, decode, family, + ) +} + +func decodeX86RawPackedWidenConversionRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + ops := x86RawPackedWidenConversionByOpcode + if p.w { + ops = x86RawPackedDoubleConversionByOpcode + } + if _, recognized := ops[p.opcode]; !recognized { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 8 << p.vectorLength + if p.w { + width *= 2 + } + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 0, 0, + decodedX86PackedWidenConversionInstruction, "packed widening conversion", + ) +} + +// decodeX86RawLegacySIMDMoveRIPData covers the six 128-bit and two scalar +// legacy MOV loads in Go's yxmov table. The other direction is a store and +// does not read the RIP-relative source bytes. +func decodeX86RawLegacySIMDMoveRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + i := offset + prefix := byte(0) + unsafeAddress := false + for i < len(code) { + switch code[i] { + case 0x64, 0x65, 0x67: + unsafeAddress = true + case 0x66, 0xf2, 0xf3: + if prefix != 0 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + prefix = code[i] + default: + if code[i] >= 0x40 && code[i] <= 0x4f { + i++ + } + goto opcode + } + i++ + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + opcode := code[i+1] + valid := opcode == 0x10 || + opcode == 0x28 && (prefix == 0 || prefix == 0x66) || + opcode == 0x6f && (prefix == 0x66 || prefix == 0xf3) + if !valid || code[i+2]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + if unsafeAddress { + return Instr{}, 0, x86RawLiteralRange{}, true, fmt.Errorf("segment/address override is not source-layout safe") + } + width := 16 + if opcode == 0x10 && prefix == 0xf3 { + width = 4 + } else if opcode == 0x10 && prefix == 0xf2 { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, i+2, width, + decodedX86LegacySIMDMoveInstruction, "legacy SIMD move", + ) +} + +func decodedX86LegacySIMDMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + inst, err := x86asm.Decode(code, mode) + if err != nil || inst.Len <= 0 { + return Instr{}, 0, true, err + } + syntax, err := decodedX86GoSyntax(inst, code[:inst.Len]) + if err != nil { + return Instr{}, 0, true, err + } + instrs, err := parseDecodedX86Instruction(syntax) + if err != nil || len(instrs) != 1 { + if err == nil { + err = fmt.Errorf("legacy SIMD move decoded as %d instructions", len(instrs)) + } + return Instr{}, 0, true, err + } + return instrs[0], inst.Len, true, nil +} + +type x86RawInstructionDecoder func([]byte, int) (Instr, int, bool, error) + +// decodeX86RawIndexedPermuteRIPData preserves local vector/scalar constants +// for the complete VPERMI2/VPERMT2 family. +func decodeX86RawIndexedPermuteRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + (p.opcode < 0x75 || p.opcode > 0x77) && (p.opcode < 0x7d || p.opcode > 0x7f) || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.opcode == 0x76 || p.opcode == 0x77 || p.opcode == 0x7e || p.opcode == 0x7f { + if p.w { + width = 8 + } + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86IndexedPermuteInstruction, "EVEX indexed permute", + ) +} + +// decodeX86RawEVEXVariableDwordPermuteRIPData preserves local Y/Z data or +// scalar broadcast constants for VPERMD and VPERMPS. +func decodeX86RawEVEXVariableDwordPermuteRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || p.w || + (p.opcode != 0x36 && p.opcode != 0x16) || + p.segment != "" || p.addressOverride || p.vectorLength < 1 || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86EVEXVariableDwordPermuteInstruction, "EVEX variable dword permute", + ) +} + +// decodeX86RawMaskCompareRIPData resolves local full-vector or D/Q broadcast +// constants and accounts for the comparison's trailing imm8. +func decodeX86RawMaskCompareRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawMaskCompareForms[[2]int{p.opcode, w}] + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = form.laneBytes + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawMaskCompareInstruction, "EVEX mask compare", + ) +} + +// decodeX86RawScaledRoundRIPData accounts for the trailing imm8 while +// resolving packed vectors, packed broadcasts, and scalar local constants. +func decodeX86RawScaledRoundRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawScaledRoundForms[[2]int{p.opcode, w}] + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if form.scalar || p.broadcast { + width = form.laneBytes + } + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 1, 1, + decodedX86RawScaledRoundInstruction, "EVEX scaled round", + ) +} + +// decodeX86RawEVEXUnpackRIPData preserves source-local full-vector and +// broadcast-lane constants for all eight EVEX integer interleave rows. +func decodeX86RawEVEXUnpackRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + form, recognized := x86RawEVEXUnpackForms[p.opcode] + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = form.broadcastBytes + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86EVEXPackedUnpackInstruction, "EVEX packed unpack", + ) +} + +// decodeX86RawBlockBroadcastRIPData resolves source-local m64/m128/m256 +// blocks for all Go EVEX packed-block broadcasts. +func decodeX86RawBlockBroadcastRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawBlockBroadcastForms[[2]int{p.opcode, w}] + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, form.sourceBytes, + decodedX86RawBlockBroadcastInstruction, "EVEX block broadcast", + ) +} + +// decodeX86RawVariableRotateRIPData preserves local vector or scalar +// constant pools for all four EVEX per-lane variable-rotate forms. +func decodeX86RawVariableRotateRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + (p.opcode != 0x14 && p.opcode != 0x15) || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 4 + if p.w { + width = 8 + } + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawVariableRotateInstruction, "EVEX variable rotate", + ) +} + +// decodeX86RawQQToFloatRIPData resolves source-local literal pools for all +// QWORD-to-PS/PD encodings before the raw-byte decoder treats them as memory. +func decodeX86RawQQToFloatRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, matched := decodeX86RawVectorEncoding(code[offset:]) + _, _, recognized := x86RawQQToFloatForm(p) + if !matched || !p.evex || !p.w || p.mapNumber != 1 || !recognized || + p.segment != "" || p.addressOverride || p.vectorLength > 2 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + p.modRM + if len(code) <= modRMIndex || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 16 << p.vectorLength + if p.broadcast { + width = 8 + } + return x86RawRIPDataThroughDecoder( + code, offset, mode, modRMIndex, width, + decodedX86RawQQToFloatInstruction, "QWORD-to-float conversion", + ) +} + +func x86RawRIPDataThroughDecoder( + code []byte, offset, mode, modRMIndex, width int, + decode x86RawInstructionDecoder, family string, +) (Instr, int, x86RawLiteralRange, bool, error) { + return x86RawRIPDataThroughDecoderOperand( + code, offset, mode, modRMIndex, width, 0, 0, decode, family, + ) +} + +func x86RawRIPDataThroughDecoderOperand( + code []byte, offset, mode, modRMIndex, width, trailingBytes, sourceIndex int, + decode x86RawInstructionDecoder, family string, +) (Instr, int, x86RawLiteralRange, bool, error) { + patched, data, literal, err := x86RawRIPBytesWithSuffix(code, offset, modRMIndex, width, trailingBytes) + if err != nil { + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + instruction, consumed, ok, err := decode(patched, mode) + if err != nil || !ok || consumed != len(patched) || + len(instruction.Args) <= sourceIndex || instruction.Args[sourceIndex].Kind != OpMem { + if err == nil { + err = fmt.Errorf("RIP-relative %s did not match its grammar", family) + } + return Instr{}, 0, x86RawLiteralRange{}, true, err + } + setX86RawRIPDataOperand(&instruction, sourceIndex, data) + return instruction, len(patched), literal, true, nil +} + +func setX86RawRIPDataOperand(instruction *Instr, sourceIndex int, data []byte) { + instruction.Args[sourceIndex] = Operand{Kind: OpSym, Sym: "·__plan9asm_raw_literal_pending(SB)"} + instruction.x86Encoded = true + instruction.x86RIPLiteral = true + instruction.x86RIPLiteralData = data + args := append([]Operand(nil), instruction.Args...) + for index := range args { + args[index].RegList = append([]Reg(nil), args[index].RegList...) + } + instruction.x86RIPMemoryRead = x86RawStaticRead{ + op: instruction.Op, source: sourceIndex, width: int64(len(data)), + symbol: instruction.Args[sourceIndex].Sym, + object: append([]byte(nil), data...), + args: args, + } + rawArgs := make([]string, len(instruction.Args)) + for index, operand := range instruction.Args { + rawArgs[index] = operand.String() + } + instruction.Raw = fmt.Sprintf("%s %s", instruction.Op, strings.Join(rawArgs, ", ")) +} + +func setX86RawRIPLiteral(instruction *Instr, value int64) { + instruction.Args[0] = Operand{Kind: OpImm, Imm: value} + instruction.x86Encoded = true + instruction.x86RIPLiteral = true + rawArgs := make([]string, len(instruction.Args)) + for index, operand := range instruction.Args { + rawArgs[index] = operand.String() + } + instruction.Raw = fmt.Sprintf("%s %s", instruction.Op, strings.Join(rawArgs, ", ")) +} diff --git a/amd64_raw_broadcast_literal_test.go b/amd64_raw_broadcast_literal_test.go new file mode 100644 index 000000000..7e65c1672 --- /dev/null +++ b/amd64_raw_broadcast_literal_test.go @@ -0,0 +1,511 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86RawBroadcastLiteral(op Op, encoding string, vectorBits, mask int, zeroing bool) ([]byte, int64) { + opcode := map[Op]byte{ + "VPBROADCASTB": 0x78, + "VPBROADCASTW": 0x79, + "VPBROADCASTD": 0x58, + "VPBROADCASTQ": 0x59, + }[op] + width := map[Op]int{ + "VPBROADCASTB": 1, + "VPBROADCASTW": 2, + "VPBROADCASTD": 4, + "VPBROADCASTQ": 8, + }[op] + value := int64(0) + data := make([]byte, width) + for i := range data { + data[i] = byte(0x11 + i*0x22) + value |= int64(data[i]) << (8 * i) + } + + var code []byte + if encoding == "VEX" { + p2 := byte(0x79) + if vectorBits == 256 { + p2 |= 0x04 + } + code = []byte{0xc4, 0xe2, p2, opcode, 0x05, 1, 0, 0, 0} + } else { + p1 := byte(0x7d) + if op == "VPBROADCASTQ" { + p1 |= 0x80 + } + lengthBits := map[int]byte{128: 0, 256: 1, 512: 2}[vectorBits] + p2 := lengthBits<<5 | 0x08 | byte(mask) + if zeroing { + p2 |= 0x80 + } + code = []byte{0x62, 0xf2, p1, p2, opcode, 0x05, 1, 0, 0, 0} + } + code = append(code, 0xc3) + code = append(code, data...) + return code, value +} + +func TestDecodeX86RawPackedBroadcastRIPLiteralCompleteFamily(t *testing.T) { + for _, op := range []Op{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + for _, encoding := range []string{"VEX", "EVEX"} { + for _, vectorBits := range []int{128, 256, 512} { + if encoding == "VEX" && vectorBits == 512 { + continue + } + for _, masking := range []struct { + mask int + zeroing bool + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true}, + } { + if encoding == "VEX" && masking.mask != 0 { + continue + } + name := fmt.Sprintf("%s/%s/%d/mask%d/zero%t", op, encoding, vectorBits, masking.mask, masking.zeroing) + t.Run(name, func(t *testing.T) { + code, value := x86RawBroadcastLiteral(op, encoding, vectorBits, masking.mask, masking.zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != op && decoded[0].Op != op+".Z" || + len(decoded[0].Args) < 2 || decoded[0].Args[0].Kind != OpImm || + decoded[0].Args[0].Imm != value || !decoded[0].x86Encoded || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want a literal %s and RET", code, decoded, op) + } + }) + } + } + } + } +} + +func TestTranslateX86RawPackedBroadcastRIPLiteralObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, op := range []Op{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + name := fmt.Sprintf("literalBroadcast%d", index) + code, _ := x86RawBroadcastLiteral(op, "EVEX", 512, 3, false) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-broadcast-literal.ll", "raw-broadcast-literal.o", ir) + }) + } +} + +func TestDecodeX86RawPackedBroadcastRIPLiteralRejectsExecutableAndMissingData(t *testing.T) { + for _, displacement := range []byte{0xf7, 0xf6, 0x20} { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + code[5] = displacement + if displacement != 0x20 { + copy(code[6:9], []byte{0xff, 0xff, 0xff}) + } + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid RIP literal", map[string]bool{}); err == nil { + t.Fatalf("accepted RIP literal displacement %#x", displacement) + } + } + code, _ := x86RawBroadcastLiteral("VPBROADCASTQ", "EVEX", 512, 0, false) + code[6] = 5 // Only the last four bytes remain; a QWORD read is truncated. + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "truncated RIP literal", map[string]bool{}); err == nil { + t.Fatal("accepted truncated RIP-relative QWORD literal") + } + + for _, prefix := range []byte{0x64, 0x65, 0x67} { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + code = append([]byte{prefix}, code...) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "prefixed RIP literal", map[string]bool{}); err == nil { + t.Fatalf("folded a source with prefix %#x", prefix) + } + } +} + +func TestDecodeX86RawPackedBroadcastRIPLiteralDoesNotChange386AbsoluteMemory(t *testing.T) { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + decoded, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 absolute source", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) < 1 || decoded[0].Args[0].Kind != OpMem || decoded[0].x86RIPLiteral { + t.Fatalf("changed a 386 absolute memory operand into a literal: %#v", decoded) + } +} + +func TestTranslateX86RawPackedBroadcastRIPLiteralRejectsAddressObservedText(t *testing.T) { + code, _ := x86RawBroadcastLiteral("VPBROADCASTD", "VEX", 256, 0, false) + var source strings.Builder + source.WriteString("TEXT address(SB),$0-0\n\tLEAQ raw+1(SB), AX\n\tRET\n") + source.WriteString("TEXT raw(SB),$0-0\n\tMOVQ AX, AX\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "address": {Name: "address", Ret: Void}, + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "address-observed RIP-relative raw literal") { + t.Fatalf("address-observed raw TEXT was silently folded: %v", err) + } +} + +func x86RawVMOVIntegerLiteral(encoding string, width int, alternateQ bool) ([]byte, int64) { + var code []byte + switch encoding { + case "VEX2": + if alternateQ { + code = []byte{0xc5, 0xfa, 0x7e, 0x05} + } else { + code = []byte{0xc5, 0xf9, 0x6e, 0x05} + } + case "VEX3": + p1 := byte(0x79) + if width == 8 { + p1 |= 0x80 + } + opcode := byte(0x6e) + if alternateQ { + p1 = 0x7a + opcode = 0x7e + } + code = []byte{0xc4, 0xe1, p1, opcode, 0x05} + case "EVEX": + p1 := byte(0x7d) + if width == 8 { + p1 |= 0x80 + } + opcode := byte(0x6e) + if alternateQ { + p1 = 0xfe + opcode = 0x7e + } + code = []byte{0x62, 0xf1, p1, 0x08, opcode, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + var value int64 + for index := 0; index < width; index++ { + b := byte(0x13 + index*0x11) + code = append(code, b) + value |= int64(b) << (8 * index) + } + return code, value +} + +func TestDecodeX86RawVMOVIntegerRIPLiteralCompleteLoadForms(t *testing.T) { + for _, test := range []struct { + encoding string + width int + alternateQ bool + }{ + {"VEX2", 4, false}, + {"VEX3", 4, false}, + {"EVEX", 4, false}, + {"VEX2", 8, true}, + {"VEX3", 8, false}, + {"VEX3", 8, true}, + {"EVEX", 8, false}, + {"EVEX", 8, true}, + } { + name := fmt.Sprintf("%s/%d/alternate%t", test.encoding, test.width, test.alternateQ) + t.Run(name, func(t *testing.T) { + code, value := x86RawVMOVIntegerLiteral(test.encoding, test.width, test.alternateQ) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + op := Op("VMOVD") + if test.width == 8 { + op = "VMOVQ" + } + if len(decoded) != 2 || decoded[0].Op != op || len(decoded[0].Args) != 2 || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != value || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s literal load and RET", code, decoded, op) + } + }) + } +} + +func TestTranslateX86RawVMOVIntegerRIPLiteralObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, test := range []struct { + encoding string + width int + alternateQ bool + }{ + {"VEX2", 4, false}, {"VEX3", 4, false}, {"EVEX", 4, false}, + {"VEX2", 8, true}, {"VEX3", 8, false}, {"VEX3", 8, true}, + {"EVEX", 8, false}, {"EVEX", 8, true}, + } { + name := fmt.Sprintf("literalVMOV%d", index) + code, _ := x86RawVMOVIntegerLiteral(test.encoding, test.width, test.alternateQ) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-vmov-literal.ll", "raw-vmov-literal.o", ir) + }) + } +} + +func TestDecodeX86RawVMOVIntegerRIPLiteralRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "inside_instruction", edit: func(code []byte) []byte { + copy(code[5:9], []byte{0xfa, 0xff, 0xff, 0xff}) // Target the opcode. + return code + }}, + {name: "missing_pool", edit: func(code []byte) []byte { + code[5] = 0x20 + return code + }}, + {name: "truncated_pool", edit: func(code []byte) []byte { + code[5] = 5 // Only the last four bytes remain. + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x64}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code, _ := x86RawVMOVIntegerLiteral("VEX3", 8, false) + code = test.edit(code) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("folded unsafe raw VMOVQ source: %x", code) + } + }) + } +} + +func TestTranslateX86RawVMOVIntegerRIPLiteralRejectsAddressObservedText(t *testing.T) { + code, _ := x86RawVMOVIntegerLiteral("VEX3", 8, false) + var source strings.Builder + source.WriteString("TEXT address(SB),$0-0\n\tLEAQ raw+1(SB), AX\n\tRET\n") + source.WriteString("TEXT raw(SB),$0-0\n\tMOVQ AX, AX\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "address": {Name: "address", Ret: Void}, + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "address-observed RIP-relative raw literal") { + t.Fatalf("address-observed raw TEXT was silently folded: %v", err) + } +} + +func x86RawScalarFloatBroadcastLiteral(double, evex bool, vectorBits, mask int, zeroing bool) ([]byte, int64) { + opcode := byte(0x18) + width := 4 + if double { + opcode = 0x19 + width = 8 + } + var code []byte + if evex { + p1 := byte(0x7d) + if double { + p1 |= 0x80 + } + p2 := byte(map[int]int{128: 0, 256: 1, 512: 2}[vectorBits]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + code = []byte{0x62, 0xf2, p1, p2, opcode, 0x05} + } else { + p1 := byte(0x79) + if vectorBits == 256 { + p1 |= 0x04 + } + code = []byte{0xc4, 0xe2, p1, opcode, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + var value int64 + for index := 0; index < width; index++ { + b := byte(0x21 + index*0x13) + code = append(code, b) + value |= int64(b) << (8 * index) + } + return code, value +} + +func TestDecodeX86RawScalarFloatBroadcastRIPLiteral(t *testing.T) { + for _, test := range []struct { + double, evex bool + vectorBits int + mask int + zeroing bool + }{ + {vectorBits: 128}, {vectorBits: 256}, {double: true, vectorBits: 256}, + {evex: true, vectorBits: 128}, {evex: true, vectorBits: 256, mask: 3}, + {evex: true, vectorBits: 512, mask: 7, zeroing: true}, + {double: true, evex: true, vectorBits: 256}, + {double: true, evex: true, vectorBits: 512, mask: 7, zeroing: true}, + } { + name := fmt.Sprintf("double%t/evex%t/%d/mask%d/zero%t", test.double, test.evex, test.vectorBits, test.mask, test.zeroing) + t.Run(name, func(t *testing.T) { + code, value := x86RawScalarFloatBroadcastLiteral(test.double, test.evex, test.vectorBits, test.mask, test.zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + op := Op("VBROADCASTSS") + if test.double { + op = "VBROADCASTSD" + } + if test.zeroing { + op += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != op || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != value || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want scalar broadcast of literal", code, decoded) + } + }) + } +} + +func TestTranslateX86RawScalarFloatBroadcastRIPLiteralObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, test := range []struct { + double, evex bool + vectorBits int + mask int + zeroing bool + }{ + {vectorBits: 128}, {vectorBits: 256}, {double: true, vectorBits: 256}, + {evex: true, vectorBits: 128}, {evex: true, vectorBits: 256, mask: 3}, + {evex: true, vectorBits: 512, mask: 7, zeroing: true}, + {double: true, evex: true, vectorBits: 256}, + {double: true, evex: true, vectorBits: 512, mask: 7, zeroing: true}, + } { + name := fmt.Sprintf("scalarFloatLiteral%d", index) + code, _ := x86RawScalarFloatBroadcastLiteral(test.double, test.evex, test.vectorBits, test.mask, test.zeroing) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "scalar-float-literal.ll", "scalar-float-literal.o", ir) + }) + } +} + +func TestDecodeX86RawScalarFloatBroadcastRIPLiteralRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "instruction_overlap", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "outside_group", edit: func(code []byte) []byte { + code[6] = 0x20 + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x65}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code, _ := x86RawScalarFloatBroadcastLiteral(true, true, 512, 0, false) + code = test.edit(code) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("folded unsafe scalar broadcast source: %x", code) + } + }) + } +} diff --git a/amd64_raw_byte_shift.go b/amd64_raw_byte_shift.go new file mode 100644 index 000000000..cf61fc4ba --- /dev/null +++ b/amd64_raw_byte_shift.go @@ -0,0 +1,64 @@ +package plan9asm + +import "fmt" + +// decodedX86RawVectorByteShiftInstruction covers Go's VPSLLDQ/VPSRLDQ +// _yvpslldq table: VEX X/Y register forms and EVEX X/Y/Z register or memory +// forms, all with an imm8. ModRM.reg is the opcode extension (/7 or /3), +// while VEX.vvvv or EVEX.vvvvv names the destination. +func decodedX86RawVectorByteShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.pp != 1 || p.opcode != 0x73 { + return Instr{}, 0, false, nil + } + if len(code) <= p.modRM { + return Instr{}, 0, true, fmt.Errorf("vector byte shift: missing ModRM byte") + } + group := int(code[p.modRM]>>3) & 7 + if group != 3 && group != 7 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector byte shift: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride || p.w || p.r != 0 || p.vectorLength > 2 || + p.evex && !p.fixed || p.mask != 0 || p.zero || p.broadcast { + return fail("invalid prefix, mask or vector length") + } + registerSource := code[p.modRM]>>6 == 3 + if !p.evex && !registerSource { + return fail("VEX byte shift requires a register source") + } + if mode == 32 && (p.upper >= 8 || registerSource && (p.b != 0 || p.x != 0)) { + return fail("extended vector register in 32-bit mode") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<= 8) { + return fail("extended register in 32-bit mode") + } + op := Op("BZHIL") + if p.w && mode == 64 { + op = "BZHIQ" + } + source, size, err := decodedX86VEXRMSource(code[p.modRM:], mode, p.b, p.x, p.segment) + if err != nil { + return Instr{}, 0, true, err + } + index, _ := decodedX86GeneralRegister(p.upper) + destination, _ := decodedX86GeneralRegister(int(code[p.modRM]>>3&7) + p.r*8) + return Instr{ + Op: op, + Args: []Operand{ + {Kind: OpReg, Reg: index}, source, {Kind: OpReg, Reg: destination}, + }, + Raw: fmt.Sprintf("%s %s, %s, %s", op, index, source.String(), destination), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_bzhi_test.go b/amd64_raw_bzhi_test.go new file mode 100644 index 000000000..f51b5594f --- /dev/null +++ b/amd64_raw_bzhi_test.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "reflect" + "testing" +) + +func encodeX86RawBZHI(wide bool, source, index, destination int) []byte { + code := encodeRawScalarMove(false, 0, 0xf5, source, index, destination, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + return code +} + +func TestDecodedX86RawBZHICompleteRegisterFamily(t *testing.T) { + count := 0 + for _, wide := range []bool{false, true} { + for source := 0; source < 16; source++ { + for index := 0; index < 16; index++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawBZHI(wide, source, index, destination) + got, length, ok, err := decodedX86RawBZHIInstruction(code, 64) + wantOp := Op("BZHIL") + if wide { + wantOp = "BZHIQ" + } + sr, _ := decodedX86GeneralRegister(source) + ir, _ := decodedX86GeneralRegister(index) + dr, _ := decodedX86GeneralRegister(destination) + want := []Operand{ + {Kind: OpReg, Reg: ir}, + {Kind: OpReg, Reg: sr}, + {Kind: OpReg, Reg: dr}, + } + if err != nil || !ok || length != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x = %+v length=%d ok=%t err=%v, want %s %+v", code, got, length, ok, err, wantOp, want) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d BZHI register forms, want 8192", count) + } +} + +func TestDecodedX86RawBZHIRealMemoryAndInvalidForms(t *testing.T) { + code := []byte{0xc4, 0x62, 0xf0, 0xf5, 0x04, 0xc6} + got, length, ok, err := decodedX86RawBZHIInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != "BZHIQ" || + len(got.Args) != 3 || got.Args[0].String() != "CX" || + got.Args[1].String() != "0(SI)(AX*8)" || got.Args[2].String() != "R8" { + t.Fatalf("decode real simd BZHI %x = %+v length=%d ok=%t err=%v", code, got, length, ok, err) + } + for name, edit := range map[string]func([]byte) []byte{ + "L1": func(b []byte) []byte { b[2] |= 4; return b }, + "address override": func(b []byte) []byte { return append([]byte{0x67}, b...) }, + "truncated ModRM": func(b []byte) []byte { return b[:4] }, + } { + t.Run(name, func(t *testing.T) { + bad := edit(append([]byte(nil), code...)) + if _, _, matched, err := decodedX86RawBZHIInstruction(bad, 64); !matched || err == nil { + t.Fatalf("accepted invalid BZHI %x, matched=%t err=%v", bad, matched, err) + } + }) + } + badPP := append([]byte(nil), code...) + badPP[2] |= 1 + if _, _, matched, _ := decodedX86RawBZHIInstruction(badPP, 64); matched { + t.Fatalf("accepted non-BZHI pp encoding %x", badPP) + } +} + +func TestTranslateX86RawBZHIRealMemoryObjects(t *testing.T) { + code := []byte{0xc4, 0x62, 0xf0, 0xf5, 0x04, 0xc6, 0xc3} + file := x86RawUnprovedReturnFormProbeFile(t, "amd64", "rawBZHI", code) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawBZHI": {Name: "rawBZHI", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-bzhi.ll", "raw-bzhi.o", ir) + }) + } +} diff --git a/amd64_raw_clmul.go b/amd64_raw_clmul.go new file mode 100644 index 000000000..c51d3120f --- /dev/null +++ b/amd64_raw_clmul.go @@ -0,0 +1,131 @@ +package plan9asm + +import "fmt" + +// decodedX86RawCLMULInstruction covers Go's complete PCLMULQDQ and +// VPCLMULQDQ tables: legacy X, VEX X/Y, and EVEX X/Y/Z, each with imm8. +// Go's vector form has no mask, broadcast, or W1 encoding. +func decodedX86RawCLMULInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if matched && p.mapNumber == 3 && p.pp == 1 && p.opcode == 0x44 { + return decodedX86RawVectorCLMULInstruction(code, mode, p) + } + return decodedX86RawLegacyCLMULInstruction(code, mode) +} + +func decodedX86RawVectorCLMULInstruction(code []byte, mode int, p x86RawVectorEncoding) (Instr, int, bool, error) { + return decodedX86RawVectorCLMULInstructionWithMemory(code, mode, p, x86RawMemoryDecoder{}) +} + +func decodedX86RawVectorCLMULInstructionWithMemory(code []byte, mode int, p x86RawVectorEncoding, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VPCLMULQDQ: %s", message) + } + if mode != 64 { + return fail("Go's four-operand vector form requires amd64") + } + if p.addressOverride || p.w || p.vectorLength > 2 || + p.evex && (!p.fixed || p.mask != 0 || p.zero || p.broadcast) { + return fail("invalid prefix, mask, broadcast or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = memory.evex( + code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<>3&7)+p.r*8))} + imm := Operand{Kind: OpImm, Imm: int64(code[immIndex])} + return Instr{ + Op: "VPCLMULQDQ", + Args: []Operand{imm, source, second, destination}, + Raw: fmt.Sprintf("VPCLMULQDQ %s, %s, %s, %s", imm.String(), source.String(), second.String(), destination.String()), + }, immIndex + 1, true, nil +} + +func decodedX86RawLegacyCLMULInstruction(code []byte, mode int) (Instr, int, bool, error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + case 0x65: + segment = GS + case 0x67: + addressOverride = true + default: + goto encoding + } + i++ + } + +encoding: + if len(code) <= i || code[i] != 0x66 { + return Instr{}, 0, false, nil + } + i++ + rex := byte(0) + if i < len(code) && code[i]&0xf0 == 0x40 { + rex = code[i] + i++ + } + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0x3a || code[i+2] != 0x44 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("PCLMULQDQ: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if addressOverride || mode == 32 && rex != 0 { + return fail("address override or REX prefix is invalid") + } + modRMIndex := i + 3 + if len(code) <= modRMIndex { + return fail("missing ModRM byte") + } + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + int(rex>>2&1)*8 + if mode == 32 && destinationNumber >= 8 { + return fail("high X register is unavailable in 386 mode") + } + source, consumed, err := decodedX86VEXVectorRMOperand( + code[modRMIndex:], mode, int(rex&1), int(rex>>1&1), segment, "X", + ) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := modRMIndex + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + imm := Operand{Kind: OpImm, Imm: int64(code[immIndex])} + return Instr{ + Op: "PCLMULQDQ", + Args: []Operand{imm, source, destination}, + Raw: fmt.Sprintf("PCLMULQDQ %s, %s, %s", imm.String(), source.String(), destination.String()), + }, immIndex + 1, true, nil +} diff --git a/amd64_raw_clmul_test.go b/amd64_raw_clmul_test.go new file mode 100644 index 000000000..b110a4eae --- /dev/null +++ b/amd64_raw_clmul_test.go @@ -0,0 +1,259 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawCLMULCode(encoding string, width int, source, second, destination int, immediate byte) []byte { + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + switch encoding { + case "legacy": + code := []byte{0x66} + rex := byte(0x40 | (destination>>3&1)<<2 | source>>3&1) + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x3a, 0x44, modRM, immediate) + case "vex": + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-source/8)<<5 | 3) + p1 := byte(((^second)&15)<<3 | ((width-16)/16)<<2 | 1) + return []byte{0xc4, p0, p1, 0x44, modRM, immediate} + default: + p0 := byte((1-destination/8&1)<<7 | (1-source/16&1)<<6 | + (1-source/8&1)<<5 | (1-destination/16&1)<<4 | 3) + p1 := byte(((^second)&15)<<3 | 0x05) + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | (1-second/16)<<3) + return []byte{0x62, p0, p1, p2, 0x44, modRM, immediate} + } +} + +func x86RawCLMULLiteral(encoding string, width int) []byte { + code := x86RawCLMULCode(encoding, width, 0, 1, 2, 0x11) + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x15 // RIP+disp32, destination X/Y/Z2. + code = append(code[:modRMIndex+1], append([]byte{1, 0, 0, 0}, code[modRMIndex+1:]...)...) + code = append(code, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*9+7)) + } + return code +} + +func TestDecodeX86RawCLMULCompleteRegisterFamily(t *testing.T) { + for _, encoding := range []string{"legacy", "vex", "evex"} { + widths := []int{16} + if encoding != "legacy" { + widths = append(widths, 32) + } + if encoding == "evex" { + widths = append(widths, 64) + } + for _, width := range widths { + limit := 16 + if encoding == "evex" { + limit = 32 + } + for _, source := range []int{0, 7, 8, limit - 1} { + for _, second := range []int{0, 7, limit - 1} { + if encoding == "legacy" && second != 0 { + continue + } + for _, destination := range []int{0, 7, limit - 1} { + name := fmt.Sprintf("%s/%d/%d/%d/%d", encoding, width, source, second, destination) + t.Run(name, func(t *testing.T) { + code := x86RawCLMULCode(encoding, width, source, second, destination, 0x11) + got, length, ok, err := decodedX86RawCLMULInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + prefix := "X" + if width == 32 { + prefix = "Y" + } else if width == 64 { + prefix = "Z" + } + want := fmt.Sprintf("VPCLMULQDQ $17, %s%d, %s%d, %s%d", prefix, source, prefix, second, prefix, destination) + if encoding == "legacy" { + want = fmt.Sprintf("PCLMULQDQ $17, X%d, X%d", source, destination) + } + if got.Raw != want { + t.Fatalf("decode %x = %q, want %q", code, got.Raw, want) + } + }) + } + } + } + } + } +} + +func TestDecodeX86RawCLMULChecksumAVX2Regression(t *testing.T) { + code := []byte{0xc4, 0xe3, 0x61, 0x44, 0xf4, 0x00, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "checksum crc32c AVX2", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Raw != "VPCLMULQDQ $0, X4, X3, X6 /* decoded from checksum crc32c AVX2 */" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawCLMULMemoryForms(t *testing.T) { + for _, encoding := range []string{"legacy", "vex", "evex"} { + code := x86RawCLMULCode(encoding, 16, 0, 1, 2, 0x11) + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x54 // disp8(SIB), destination X2. + code = append(code[:modRMIndex+1], append([]byte{0x88, 0x20}, code[modRMIndex+1:]...)...) + got, length, ok, err := decodedX86RawCLMULInstruction(code, 64) + wantOffset := int64(32) + if encoding == "evex" { + wantOffset = 512 // EVEX disp8 scales by the 128-bit memory operand. + } + if err != nil || !ok || length != len(code) || got.Args[1].Kind != OpMem || + got.Args[1].Mem.Base != AX || got.Args[1].Mem.Index != CX || + got.Args[1].Mem.Scale != 4 || got.Args[1].Mem.Off != wantOffset { + t.Fatalf("%s memory source %x decoded as %+v, length=%d, ok=%v, err=%v", encoding, code, got, length, ok, err) + } + } +} + +func TestDecodeX86RawCLMULRIPDataCompleteFamily(t *testing.T) { + for _, encoding := range []string{"legacy", "vex", "evex"} { + widths := []int{16} + if encoding != "legacy" { + widths = append(widths, 32) + } + if encoding == "evex" { + widths = append(widths, 64) + } + for _, width := range widths { + name := fmt.Sprintf("%s/%d", encoding, width) + t.Run(name, func(t *testing.T) { + code := x86RawCLMULLiteral(encoding, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Args[0].Imm != 0x11 || + decoded[0].Args[1].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("decoded CLMUL literal %x as %#v", code, decoded) + } + }) + } + } +} + +func TestDecodeX86RawCLMULRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[6] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[6] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawCLMULLiteral("evex", 64)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe CLMUL literal %x", code) + } + }) + } +} + +func TestDecodeX86RawCLMULRejectsReservedForms(t *testing.T) { + valid := x86RawCLMULCode("evex", 64, 0, 1, 2, 0x11) + for name, mutate := range map[string]func([]byte) []byte{ + "mask": func(code []byte) []byte { code[3] |= 1; return code }, + "zeroing": func(code []byte) []byte { code[3] |= 0x80; return code }, + "broadcast": func(code []byte) []byte { code[3] |= 0x10; return code }, + "width": func(code []byte) []byte { code[3] |= 0x60; return code }, + "W": func(code []byte) []byte { code[2] |= 0x80; return code }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + "missing immediate": func(code []byte) []byte { return code[:len(code)-1] }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, _, ok, err := decodedX86RawCLMULInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved CLMUL encoding %x: ok=%v err=%v", code, ok, err) + } + }) + } +} + +func TestTranslateX86RawCLMULObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, encoding := range []string{"legacy", "vex", "evex"} { + width := []int{16, 32, 64}[index] + name := fmt.Sprintf("rawCLMUL%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawCLMULCode(encoding, width, 0, 1, 2, 0x11) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + source.WriteString("\tRET\n") + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-clmul.ll", "raw-clmul.o", ir) + }) + } +} + +func TestTranslateX86RawCLMULRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, encoding := range []string{"legacy", "vex", "evex"} { + width := []int{16, 32, 64}[index] + name := fmt.Sprintf("literalCLMUL%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawCLMULLiteral(encoding, width) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-clmul-literal.ll", "raw-clmul-literal.o", ir) + }) + } +} diff --git a/amd64_raw_compare_float_test.go b/amd64_raw_compare_float_test.go new file mode 100644 index 000000000..1feb031f2 --- /dev/null +++ b/amd64_raw_compare_float_test.go @@ -0,0 +1,380 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Go's yxcmpi table uses X/m, X, signed-imm8 for all four legacy +// floating compares. Intel decoding instead returns destination, source, +// unsigned predicate; a generic operand reversal is not the Go grammar. +func TestX86RawLegacyFloatingCompareCompleteForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var named, raw strings.Builder + named.WriteString("TEXT compareforms(SB),4,$0-0\n") + raw.WriteString("TEXT compareforms(SB),4,$0-0\n") + var expected []byte + for _, family := range []struct { + op string + prefix []byte + }{{"CMPPS", nil}, {"CMPPD", []byte{0x66}}, {"CMPSS", []byte{0xf3}}, {"CMPSD", []byte{0xf2}}} { + forms := []struct { + source, destination string + rex byte + address []byte + }{{"X1", "X0", 0, []byte{0xc1}}, {"-7(BX)(CX*4)", "X7", 0, []byte{0x7c, 0x8b, 0xf9}}} + if arch == "amd64" { + forms = append(forms, struct { + source, destination string + rex byte + address []byte + }{"X15", "X14", 0x45, []byte{0xf7}}) + } + for _, form := range forms { + for predicate := 0; predicate < 256; predicate++ { + code := append([]byte(nil), family.prefix...) + if form.rex != 0 { + code = append(code, form.rex) + } + code = append(code, 0x0f, 0xc2) + code = append(code, form.address...) + code = append(code, byte(predicate)) + instruction := fmt.Sprintf("%s %s, %s, $%d", family.op, form.source, form.destination, int8(predicate)) + fmt.Fprintln(&named, instruction) + expected = append(expected, code...) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + want, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+instruction+"\n") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != want.Funcs[0].Instrs[1].Op || !reflect.DeepEqual(decoded.Instrs[0].Args, want.Funcs[0].Instrs[1].Args) { + t.Fatalf("decoded %x as %+v, want %s", code, decoded.Instrs, instruction) + } + // All encodings are checked against Go above; compiling the + // low predicates and signed boundary cases exercises every + // semantic selector without bloating every target's IR. + if predicate < 8 || predicate == 127 || predicate == 128 || predicate == 255 { + for _, b := range code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", b) + } + } + } + } + } + named.WriteString("RET\n") + raw.WriteString("RET\n") + expected = append(expected, 0xc3) + if got := assembleX87ControlBytes(t, arch, named.String()); !bytes.HasPrefix(got, expected) { + t.Fatalf("Go %s compare encodings disagree: got %d bytes, want prefix of %d bytes", arch, len(got), len(expected)) + } + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"compareforms": {Name: "compareforms", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-compares.ll", "raw-compares.o", ir) + } + }) + } +} + +func TestX86RawLegacyFloatingCompareRejectsMalformedPredicate(t *testing.T) { + for _, predicate := range []x86asm.Arg{nil, x86asm.X0, x86asm.Imm(-1), x86asm.Imm(256)} { + inst := x86asm.Inst{Op: x86asm.CMPPS, Mode: 64, Args: x86asm.Args{x86asm.X0, x86asm.X1, predicate}} + if _, err := decodedX86GoSyntax(inst, nil); err == nil { + t.Fatalf("accepted malformed decoded predicate %v", predicate) + } + } +} + +func TestCrossLinuxRuntimeMatrixX86RawLegacyFloatingCompareRuntime(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, functions, originals, originalFunctions, goOpcodes strings.Builder + goOpcodes.WriteString("TEXT originalopcodes(SB),4,$0-0\n") + var expectedOpcodes []byte + sigs := make(map[string]FuncSig) + for family, prefix := range [][]byte{nil, {0x66}, {0xf3}, {0xf2}} { + for memory := 0; memory < 2; memory++ { + for predicate := 0; predicate < 256; predicate++ { + name := fmt.Sprintf("rawcmp_%d_%d_%d", family, memory, predicate) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ out+0(FP),DI\nMOVQ left+8(FP),AX\nMOVQ right+16(FP),BX\nMOVOU (AX),X0\nMOVOU (BX),X1\n", name) + code := append([]byte(nil), prefix...) + modrm := byte(0xc1) + if memory == 1 { + modrm = 0x03 // X0, (BX) + } + code = append(code, 0x0f, 0xc2, modrm, byte(predicate)) + expectedOpcodes = append(expectedOpcodes, code...) + sourceOperand := "X1" + if memory == 1 { + sourceOperand = "(BX)" + } + opcode := [...]string{"CMPPS", "CMPPD", "CMPSS", "CMPSD"}[family] + fmt.Fprintf(&goOpcodes, "%s %s,X0,$%d\n", opcode, sourceOperand, int8(predicate)) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("MOVOU X0,(DI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void *,const void *,const void *);\n", name) + fmt.Fprintf(&functions, "%s,\n", name) + originals.WriteString(x86RawLegacyFloatingCompareOriginalKernel(name, code)) + fmt.Fprintf(&originalFunctions, "original_%s,\n", name) + } + } + } + goOpcodes.WriteString("RET\n") + expectedOpcodes = append(expectedOpcodes, 0xc3) + if got := assembleX87ControlBytes(t, "amd64", goOpcodes.String()); !bytes.HasPrefix(got, expectedOpcodes) { + t.Fatalf("actual Go full imm8 register/(BX) opcode bytes differ: got=%d expected prefix=%d", len(got), len(expectedOpcodes)) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +#include +#ifndef _WIN32 +#include +#include +#include +#include +#endif +` + declarations.String() + ` +typedef void (*comparison)(void *,const void *,const void *); +static comparison functions[]={ +` + functions.String() + `}; +` + originals.String() + ` +static comparison originals[]={ +` + originalFunctions.String() + `}; + +// Only predicates 0..7 have the legacy IEEE truth table below. Bits 3..7 +// are reserved: an observed machine result is not a mathematical definition. +static int predicate(double a,double b,int p) { + int unordered=isnan(a)||isnan(b); + switch(p) { + case 0: return !unordered&&a==b; + case 1: return !unordered&&a=b; + case 6: return unordered||a>b; + case 7: return !unordered; + default: return -1; + } +} + +static unsigned observe(comparison fn,void *out,const void *left,const void *right) { + unsigned control=0x1f80,observed; + __asm__ volatile("ldmxcsr %0"::"m"(control):"memory"); + fn(out,left,right); + __asm__ volatile("stmxcsr %0":"=m"(observed)::"memory"); + return observed; +} + +#ifndef _WIN32 +// Preserve the original packed memory opcode's unaligned behavior. The +// outcome is an observation of this execution environment, not a new policy. +static int same_unaligned_outcome(comparison actual,comparison original) { + int statuses[2]; + for(int which=0;which<2;which++) { + pid_t child=fork(); + if(child<0) return 1; + if(child==0) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + _Alignas(16) unsigned char left[32]={0},right[32]={0},out[16]; + observe(which?original:actual,out,left,right+1); + _exit(0); + } + if(waitpid(child,&statuses[which],0)!=child) return 1; + if(WIFEXITED(statuses[which])) { + if(WEXITSTATUS(statuses[which])!=0) return 1; + } else if(!WIFSIGNALED(statuses[which]) || + (WTERMSIG(statuses[which])!=SIGSEGV && WTERMSIG(statuses[which])!=SIGBUS)) { + return 1; + } + } + if(WIFSIGNALED(statuses[0])!=WIFSIGNALED(statuses[1]) || + (WIFSIGNALED(statuses[1]) && WTERMSIG(statuses[0])!=WTERMSIG(statuses[1]))) { + fprintf(stderr,"unaligned original opcode outcome differs actual=%x original=%x\n",statuses[0],statuses[1]); + return 1; + } + return 0; +} +#endif + +int main(void) { + uint32_t f[]={0,0x80000000u,0x3f800000u,0xbf800000u,0x7f800000u,0xff800000u,0x7fc12345u}; + uint64_t d[]={0,0x8000000000000000ull,0x3ff0000000000000ull,0xbff0000000000000ull,0x7ff0000000000000ull,0xfff0000000000000ull,0x7ff8123456789abcull}; + for(int family=0;family<4;family++) { + for(int mem=0;mem<2;mem++) { + for(int p=0;p<256;p++) { + for(int a=0;a<7;a++) { + for(int b=0;b<7;b++) { + _Alignas(16) unsigned char left_storage[64],right_storage[64]; + unsigned char actual_storage[48],original_storage[48],expected[16]; + unsigned char left_before[64],right_before[64]; + memset(left_storage,0xa7,sizeof(left_storage)); + memset(right_storage,0xb9,sizeof(right_storage)); + memset(actual_storage,0xc3,sizeof(actual_storage)); + memset(original_storage,0xd5,sizeof(original_storage)); + unsigned char *left=left_storage+16; + // Packed memory remains aligned; scalar memory additionally + // exercises a real unaligned address without inventing a shadow. + unsigned char *right=right_storage+16+(family>=2 && mem?3:0); + unsigned char *got=actual_storage+16,*observed=original_storage+16; + int bytes=family&1?8:4,lanes=16/bytes; + for(int lane=0;lane=2?1:lanes);lane++) { + double x,y; + if(bytes==8) { + memcpy(&x,left+lane*bytes,8); + memcpy(&y,right+lane*bytes,8); + } else { + float xf,yf; + memcpy(&xf,left+lane*bytes,4); + memcpy(&yf,right+lane*bytes,4); + x=xf; + y=yf; + } + memset(expected+lane*bytes,predicate(x,y,p)?255:0,bytes); + } + if(memcmp(got,expected,16)) { + fprintf(stderr,"independent IEEE mismatch family=%d mem=%d predicate=%d a=%d b=%d\n",family,mem,p,a,b); + return 4; + } + } + } + } + } +#ifndef _WIN32 + if(mem && family<2) { + const int predicates[]={0,8,255}; + for(int axis=0;axis<3;axis++) { + int p=predicates[axis]; + int index=(family*2+mem)*256+p; + if(same_unaligned_outcome(functions[index],originals[index])) return 5; + } + } +#endif + } + } + return 0; +} +` + execution.compileAndRunLLVM(t, "raw_compare_float", ir, mainC, "-O2") + t.Logf("2048 exact original imm8 opcodes ×49 inputs: full X bytes, canaries and MXCSR agree; only 0..7 have independent IEEE truth; transport=%v", execution.prefix) +} + +func x86RawLegacyFloatingCompareOriginalKernel(name string, code []byte) string { + var directive strings.Builder + for index, b := range code { + if index != 0 { + directive.WriteByte(',') + } + fmt.Fprintf(&directive, "0x%02x", b) + } + return fmt.Sprintf(` +static __attribute__((noinline)) void original_%s(void *out,const void *left,const void *right) { + __asm__ volatile( + "movdqu (%%1),%%%%xmm0\n\t" + "movdqu (%%%%rbx),%%%%xmm1\n\t" + ".byte %s\n\t" + "movdqu %%%%xmm0,(%%0)" + ::"r"(out),"r"(left),"b"(right) + :"xmm0","xmm1","memory"); +} +`, name, directive.String()) +} + +func TestX86RawLegacyFloatingCompareOriginalKernelRetainsReservedBytes(t *testing.T) { + for _, prefix := range [][]byte{nil, {0x66}, {0xf3}, {0xf2}} { + for _, modrm := range []byte{0xc1, 0x03} { + for predicate := 0; predicate < 256; predicate++ { + code := append([]byte(nil), prefix...) + code = append(code, 0x0f, 0xc2, modrm, byte(predicate)) + kernel := x86RawLegacyFloatingCompareOriginalKernel("byte_identity", code) + var exact strings.Builder + for index, b := range code { + if index != 0 { + exact.WriteByte(',') + } + fmt.Fprintf(&exact, "0x%02x", b) + } + if strings.Count(kernel, ".byte ") != 1 || !strings.Contains(kernel, ".byte "+exact.String()+`\n\t`) { + t.Fatalf("original kernel altered source bytes %x:\n%s", code, kernel) + } + for _, required := range []string{`movdqu (%1),%%xmm0`, `movdqu (%%rbx),%%xmm1`, `movdqu %%xmm0,(%0)`, `"b"(right)`, `"memory"`} { + if !strings.Contains(kernel, required) { + t.Fatalf("original register/memory/output binding lost %q: %x", required, code) + } + } + if predicate >= 8 { + folded := append([]byte(nil), code...) + folded[len(folded)-1] &= 7 + if x86RawLegacyFloatingCompareOriginalKernel("byte_identity", folded) == kernel { + t.Fatalf("reserved imm8 %d folded to a mathematical predicate", predicate) + } + } + } + } + } +} diff --git a/amd64_raw_compare_float_transport_test.go b/amd64_raw_compare_float_transport_test.go new file mode 100644 index 000000000..8adba7148 --- /dev/null +++ b/amd64_raw_compare_float_transport_test.go @@ -0,0 +1,49 @@ +package plan9asm + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" +) + +func TestX86RawLegacyFloatingCompareRuntimeRequiresOwnedGuestTransport(t *testing.T) { + file, err := parser.ParseFile(token.NewFileSet(), "amd64_raw_compare_float_test.go", nil, 0) + if err != nil { + t.Fatal(err) + } + const required = "TestCrossLinuxRuntimeMatrixX86RawLegacyFloatingCompareRuntime" + var body *ast.BlockStmt + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if ok && function.Name.Name == required { + body = function.Body + } + } + if body == nil { + t.Fatal("raw compare runtime is not registered in the required cross-runtime matrix") + } + + tools, consumer := 0, 0 + ast.Inspect(body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok { + return true + } + if function, ok := call.Fun.(*ast.Ident); ok && function.Name == "x86FloatCompareExecutionTools" { + tools++ + } + if method, ok := call.Fun.(*ast.SelectorExpr); ok { + if method.Sel.Name == "Skip" || method.Sel.Name == "Skipf" || method.Sel.Name == "SkipNow" { + t.Fatal("required raw compare execution cannot substitute a host skip for its guest counterpart") + } + if receiver, ok := method.X.(*ast.Ident); ok && receiver.Name == "execution" && method.Sel.Name == "compileAndRunLLVM" { + consumer++ + } + } + return true + }) + if tools != 1 || consumer != 1 { + t.Fatalf("required raw comparison must consume its authenticated tools and execution route exactly once: tools=%d consumer=%d", tools, consumer) + } +} diff --git a/amd64_raw_compress_test.go b/amd64_raw_compress_test.go new file mode 100644 index 000000000..b5cbfbd83 --- /dev/null +++ b/amd64_raw_compress_test.go @@ -0,0 +1,162 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawCompressForms = []struct { + opcode int + widthBit int + op Op + laneBytes int +}{ + {0x63, 0, "VPCOMPRESSB", 1}, + {0x63, 1, "VPCOMPRESSW", 2}, + {0x8a, 0, "VCOMPRESSPS", 4}, + {0x8a, 1, "VCOMPRESSPD", 8}, + {0x8b, 0, "VPCOMPRESSD", 4}, + {0x8b, 1, "VPCOMPRESSQ", 8}, +} + +func encodeX86RawCompress(opcode, widthBit, vectorBits, mask int, zeroing, memory bool) []byte { + p1 := byte(0x7d) // EVEX.vvvv is unused, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xd3) // source register 2, destination register 3. + if memory { + modRM = 0x50 // source register 2, disp8(AX) destination. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawCompressSeBiShogunRegression(t *testing.T) { + // Go disassembles the bytes as VCOMPRESSPS Z0, K1, 0(SI)(R9*4). + code := []byte{0x62, 0xb2, 0x7d, 0x49, 0x8a, 0x04, 0x8e, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "compressBitsFloat32 AVX512 VCOMPRESSPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VCOMPRESSPS" || + decoded[0].Args[0].Reg != "Z0" || decoded[0].Args[1].Reg != "K1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawCompressCompleteGoForms(t *testing.T) { + if len(x86RawCompressForms) != len(amd64PackedCompressLaneBits) { + t.Fatalf("raw/lower compress grammar rows = %d/%d", len(x86RawCompressForms), len(amd64PackedCompressLaneBits)) + } + for _, form := range x86RawCompressForms { + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, variant := range []struct { + name string + mask int + zeroing bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "zeroing register", mask: 7, zeroing: true}, + {name: "memory", memory: true}, + {name: "masked memory", mask: 1, memory: true}, + } { + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawCompress(form.opcode, form.widthBit, vectorBits, variant.mask, variant.zeroing, variant.memory) + got, length, ok, err := decodedX86PackedCompressInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[0].String() != vector+"2" || + !variant.memory && got.Args[len(got.Args)-1].String() != vector+"3" || + variant.memory && got.Args[len(got.Args)-1].String() != fmt.Sprintf("%d(AX)", form.laneBytes) || + len(got.Args) != 2+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawCompressRejectsInvalidForms(t *testing.T) { + base := encodeX86RawCompress(0x8a, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x60 }, // reserved vector length + func(code []byte) { code[3] |= 0x10 }, // broadcast unavailable + func(code []byte) { code[2] &^= 0x08 }, // EVEX.vvvv is reserved + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86PackedCompressInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + if _, _, ok, err := decodedX86PackedCompressInstruction(encodeX86RawCompress(0x8a, 0, 2, 1, false, false), 32); !ok || err == nil { + t.Fatalf("accepted 386 masked compress: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawCompressLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawCompress(SB),$0-0\n") + for _, form := range x86RawCompressForms { + for _, memory := range []bool{false, true} { + mask := 1 + if target.goarch == "386" { + mask = 0 + } + code := encodeX86RawCompress(form.opcode, form.widthBit, 0, mask, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawCompress": {Name: "rawCompress", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-compress.ll", "raw-compress.o", ir) + }) + } +} diff --git a/amd64_raw_convert_half.go b/amd64_raw_convert_half.go new file mode 100644 index 000000000..b290b0e3c --- /dev/null +++ b/amd64_raw_convert_half.go @@ -0,0 +1,189 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawHalfConversionForm struct { + op Op + mapNumber int + opcode int + immediate bool + rawOnly bool +} + +// The complete F16C/AVX-512 half/single conversion rows. The FP16 map5/map6 +// forms are raw-only in Go 1.27. Only legacy VCVTPS2PH reverses ModRM roles +// and appends imm8; FP16 narrowing instead loads and supports broadcasts. +var x86RawHalfConversionForms = [...]x86RawHalfConversionForm{ + {op: "VCVTPH2PS", mapNumber: 2, opcode: 0x13}, + {op: "VCVTPS2PH", mapNumber: 3, opcode: 0x1d, immediate: true}, + {op: "VCVTPH2PSX", mapNumber: 6, opcode: 0x13, rawOnly: true}, + {op: "VCVTPS2PHX", mapNumber: 5, opcode: 0x1d, rawOnly: true}, +} + +func decodedX86PackedHalfConversionInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.pp != 1 { + return Instr{}, 0, false, nil + } + var form x86RawHalfConversionForm + for _, candidate := range x86RawHalfConversionForms { + if p.mapNumber == candidate.mapNumber && p.opcode == candidate.opcode { + form = candidate + break + } + } + if form.op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed half conversion: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if form.rawOnly && !p.evex { + return fail("FP16 conversion requires EVEX") + } + if p.upper != 0 || p.w || p.evex && !p.fixed { + return fail("invalid reserved vvvv/V', W or EVEX fixed bit") + } + if p.evex && p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + registerSource := code[p.modRM]>>6 == 3 + vectorLength := p.vectorLength + op := form.op + if p.broadcast && form.rawOnly { + if registerSource { + vectorLength = 2 + if form.op == "VCVTPS2PHX" { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } else { + // LLVM emits LL=00 for SAE-only; LL does not encode VL here. + op += ".SAE" + } + } else { + op += ".BCST" + } + } + if vectorLength > 2 { + return fail("reserved vector length") + } + if form.rawOnly && mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register or address in 32-bit mode") + } + if !p.evex && mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended VEX register in 32-bit mode") + } + if p.evex && mode == 32 && code[p.modRM]>>6 != 3 && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + if p.broadcast && !form.rawOnly { + if !p.evex || p.vectorLength != 2 || form.op == "VCVTPH2PS" && code[p.modRM]>>6 != 3 { + return fail("SAE requires the EVEX Z-width register form") + } + op += ".SAE" + } + + vector := [...]string{"X", "Y", "Z"}[vectorLength] + shorter := "X" + if vectorLength == 2 { + shorter = "Y" + } + if p.zero { + op += ".Z" + } + modRM := code[p.modRM] + regNumber := int(modRM>>3&7) + p.r*8 + register := func(prefix string) Operand { + return Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regNumber))} + } + decodeRM := func(prefix string) (Operand, int, error) { + if p.evex { + width := 8 << vectorLength + if form.op == "VCVTPS2PHX" { + width *= 2 + } + if form.rawOnly && p.broadcast && !registerSource { + width = amd64HalfConversionSpecs[string(form.op)].inputBits / 8 + } + return decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, prefix, width) + } + return decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, prefix) + } + + args := make([]Operand, 0, 4) + var consumed int + if !form.immediate { + sourcePrefix, destinationPrefix := shorter, vector + if form.op == "VCVTPS2PHX" { + sourcePrefix, destinationPrefix = vector, shorter + } + source, n, err := decodeRM(sourcePrefix) + if err != nil { + return Instr{}, 0, true, err + } + consumed = n + args = append(args, source) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, register(destinationPrefix)) + } else { + destination, n, err := decodeRM(shorter) + if err != nil { + return Instr{}, 0, true, err + } + consumed = n + if len(code) <= p.modRM+consumed { + return fail("missing imm8 rounding control") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[p.modRM+consumed])}) + args = append(args, register(vector)) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + consumed++ + } + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{ + Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: form.rawOnly, x86VectorBytes: 16 << vectorLength, + }, p.modRM + consumed, true, nil +} + +func decodeX86RawFP16ConversionRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, ok := decodeX86RawVectorEncoding(code[offset:]) + if !ok || !p.evex || p.pp != 1 || p.segment != "" || p.addressOverride || + !(p.mapNumber == 6 && p.opcode == 0x13 || p.mapNumber == 5 && p.opcode == 0x1d) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRM := offset + p.modRM + if len(code) <= modRM || code[modRM]&0xc7 != 5 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := 8 << p.vectorLength + if p.broadcast { + width = 2 + } + if p.mapNumber == 5 { + width *= 2 + } + return x86RawRIPDataThroughDecoder(code, offset, mode, modRM, width, decodedX86PackedHalfConversionInstruction, "FP16 conversion") +} diff --git a/amd64_raw_convert_half_test.go b/amd64_raw_convert_half_test.go new file mode 100644 index 000000000..b68004d7f --- /dev/null +++ b/amd64_raw_convert_half_test.go @@ -0,0 +1,264 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVEXHalfConversion(form x86RawHalfConversionForm, width256 bool, reg, rm int, memory bool, immediate byte) []byte { + if memory { + rm = 0 + } + l := byte(0) + if width256 { + l = 1 + } + p0 := byte((1-reg/8)<<7 | 1<<6 | (1-rm/8)<<5 | form.mapNumber) + p1 := byte(0x79 | l<<2) + modRM := byte(0xc0 | (reg&7)<<3 | rm&7) + if memory { + modRM = byte(0x40 | (reg&7)<<3) + } + code := []byte{0xc4, p0, p1, byte(form.opcode), modRM} + if memory { + code = append(code, 7) + } + if form.immediate { + code = append(code, immediate) + } + return code +} + +func encodeX86RawEVEXHalfConversion(form x86RawHalfConversionForm, vectorBits, mask int, zeroing, sae bool, reg, rm int, memory bool, immediate byte) []byte { + if memory { + rm = 0 + } + p0 := byte((1-reg/8&1)<<7 | (1-rm/16)<<6 | (1-rm/8&1)<<5 | (1-reg/16)<<4 | form.mapNumber) + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if sae { + p2 |= 0x10 + } + modRM := byte(0xc0 | (reg&7)<<3 | rm&7) + if memory { + modRM = byte(0x40 | (reg&7)<<3) + } + code := []byte{0x62, p0, 0x7d, p2, byte(form.opcode), modRM} + if memory { + code = append(code, 7) + } + if form.immediate { + code = append(code, immediate) + } + return code +} + +func TestTranslateRawHalfConversionGorseRegression(t *testing.T) { + const source = `TEXT rawHalfToSingle(SB), $0-0 + LONG $0x137da2c4 + WORD $0x474c + BYTE $0xf0 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawHalfToSingle": {Name: "rawHalfToSingle", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "fpext <8 x half>") { + t.Fatalf("raw VCVTPH2PS omitted half-to-single conversion:\n%s", ir) + } +} + +func TestDecodedX86PackedHalfConversionGorseEVEXMemory(t *testing.T) { + cases := []struct { + name string + code []byte + want MemRef + }{ + { + name: "SIB", + code: []byte{0x62, 0xf2, 0x7d, 0x48, 0x13, 0x0c, 0x4f}, + want: MemRef{Base: DI, Index: CX, Scale: 2}, + }, + { + name: "SIB compressed displacement", + code: []byte{0x62, 0xf2, 0x7d, 0x48, 0x13, 0x4c, 0x4f, 0x01}, + want: MemRef{Base: DI, Index: CX, Scale: 2, Off: 32}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, length, matched, err := decodedX86PackedHalfConversionInstruction(tc.code, 64) + if err != nil || !matched || length != len(tc.code) || got.Op != "VCVTPH2PS" { + t.Fatalf("decode %x = %+v length=%d matched=%v err=%v", tc.code, got, length, matched, err) + } + if len(got.Args) != 2 || got.Args[0].Kind != OpMem || got.Args[0].Mem != tc.want || got.Args[1].Reg != "Z1" { + t.Fatalf("decode %x = %+v, want memory %+v and Z1", tc.code, got, tc.want) + } + }) + } +} + +func TestDecodedX86PackedHalfConversionGoRowsAndWidths(t *testing.T) { + if len(x86RawHalfConversionForms) != 4 || + x86RawHalfConversionForms[0] != (x86RawHalfConversionForm{op: "VCVTPH2PS", mapNumber: 2, opcode: 0x13}) || + x86RawHalfConversionForms[1] != (x86RawHalfConversionForm{op: "VCVTPS2PH", mapNumber: 3, opcode: 0x1d, immediate: true}) { + t.Fatalf("packed half conversion grammar does not match the two Go 1.27 opcode rows") + } + for _, form := range x86RawHalfConversionForms[:2] { + for _, width256 := range []bool{false, true} { + vector := "X" + if width256 { + vector = "Y" + } + for _, memory := range []bool{false, true} { + code := encodeX86RawVEXHalfConversion(form, width256, 2, 1, memory, 7) + got, length, matched, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !matched || length != len(code) || got.Op != form.op { + t.Fatalf("VEX %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + if form.immediate { + if len(got.Args) != 3 || got.Args[0].String() != "$7" || got.Args[1].String() != vector+"2" { + t.Fatalf("VEX single-to-half %x = %+v", code, got) + } + } else if len(got.Args) != 2 || got.Args[1].String() != vector+"2" { + t.Fatalf("VEX half-to-single %x = %+v", code, got) + } + } + } + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + code := encodeX86RawEVEXHalfConversion(form, vectorBits, mask, masked && !memory, false, 21, 20, memory, 9) + got, length, matched, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !matched || length != len(code) { + t.Fatalf("EVEX %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + wantOp := form.op + if masked && !memory { + wantOp += ".Z" + } + if got.Op != wantOp { + t.Fatalf("EVEX %x op=%s, want %s", code, got.Op, wantOp) + } + registerIndex := 1 + if !form.immediate { + registerIndex = len(got.Args) - 1 + } + if got.Args[registerIndex].String() != fmt.Sprintf("%s21", vector) { + t.Fatalf("EVEX %x = %+v, lost high register", code, got) + } + } + } + } + } +} + +func TestDecodedX86PackedHalfConversionSAEAndInvalidForms(t *testing.T) { + for _, form := range x86RawHalfConversionForms[:2] { + code := encodeX86RawEVEXHalfConversion(form, 2, 2, true, true, 1, 2, false, 7) + got, length, matched, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !matched || length != len(code) || got.Op != form.op+".SAE.Z" { + t.Fatalf("SAE %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + invalid := append([]byte(nil), code...) + invalid[2] |= 0x80 + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(invalid, 64); !matched || err == nil { + t.Fatalf("W=1 encoding accepted: %x", invalid) + } + invalid = append([]byte(nil), code...) + invalid[3] &^= 7 + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(invalid, 64); !matched || err == nil { + t.Fatalf("zeroing without mask accepted: %x", invalid) + } + } + toHalf := x86RawHalfConversionForms[1] + missingImmediate := encodeX86RawVEXHalfConversion(toHalf, false, 1, 2, false, 7) + missingImmediate = missingImmediate[:len(missingImmediate)-1] + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(missingImmediate, 64); !matched || err == nil { + t.Fatalf("missing imm8 accepted: %x", missingImmediate) + } + fromHalf := x86RawHalfConversionForms[0] + invalidMemorySAE := encodeX86RawEVEXHalfConversion(fromHalf, 2, 0, false, true, 1, 0, true, 0) + if _, _, matched, err := decodedX86PackedHalfConversionInstruction(invalidMemorySAE, 64); !matched || err == nil { + t.Fatalf("half-to-single memory SAE accepted: %x", invalidMemorySAE) + } +} + +func TestTranslateRawPackedHalfConversionLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedHalfConversions(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, form := range x86RawHalfConversionForms[:2] { + for _, width256 := range []bool{false, true} { + appendBytes(encodeX86RawVEXHalfConversion(form, width256, 2, 1, false, 7)) + appendBytes(encodeX86RawVEXHalfConversion(form, width256, 2, 0, true, 7)) + } + for vectorBits := 0; vectorBits < 3; vectorBits++ { + mask := 3 + if target.goarch == "386" && form.immediate { + mask = 0 + } + reg, rm := 21, 20 + if target.goarch == "386" { + reg, rm = 2, 1 + } + appendBytes(encodeX86RawEVEXHalfConversion(form, vectorBits, mask, mask != 0, vectorBits == 2, reg, rm, false, 9)) + appendBytes(encodeX86RawEVEXHalfConversion(form, vectorBits, mask, false, false, 1, 0, true, 9)) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawPackedHalfConversions": {Name: "rawPackedHalfConversions", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "fpext <") || !strings.Contains(ir, "fptrunc <") { + t.Fatalf("raw half-conversion family was not lowered:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-half-conversions.ll", "raw-half-conversions.o", ir) + }) + } +} diff --git a/amd64_raw_decode_guard_test.go b/amd64_raw_decode_guard_test.go new file mode 100644 index 000000000..a9d117865 --- /dev/null +++ b/amd64_raw_decode_guard_test.go @@ -0,0 +1,57 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawDecoderRejectsUnrecognizedVEXInsteadOfLegacyInstructions(t *testing.T) { + // x/arch v0.14 can return SUB AL,imm instead of an error for these + // invalid scalar conversions. Such a successful decode is not semantic support. + for _, code := range [][]byte{{0xc5, 0xf8, 0x2c, 0xc1}, {0xc5, 0xf8, 0x2d, 0xc1}, {0xc4, 0xe1, 0xf8, 0x2c, 0xc1}} { + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"f": {Name: "f", Ret: Void}}}); err == nil { + t.Errorf("%s raw %x must fail closed instead of executing legacy arithmetic: %v", arch, code, err) + } + } + } +} + +func TestX86RawDecoderRecognizesSupportedScalarConversionsBeforeLegacyFallback(t *testing.T) { + for _, tc := range []struct { + code []byte + op Op + }{ + {[]byte{0xc5, 0xfb, 0x2c, 0xc1}, "VCVTTSD2SI"}, + {[]byte{0xc5, 0xfa, 0x2c, 0xc1}, "VCVTTSS2SI"}, + {[]byte{0xc4, 0xe1, 0xfb, 0x2c, 0xc1}, "VCVTTSD2SIQ"}, + } { + decoded, err := decodeX86RawDirectives(rawX86Function(tc.code), "amd64") + if err != nil || len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != tc.op { + t.Errorf("supported VEX %x: %+v, %v; want %s", tc.code, decoded.Instrs, err, tc.op) + } + } +} + +func TestX86RawDecoderRetainsRecognizedVEXFallback(t *testing.T) { + for _, code := range [][]byte{ + {0xc5, 0xf8, 0x77}, {0xc5, 0xf9, 0x6f, 0xc1}, {0xc5, 0xfa, 0x6f, 0xc1}, + {0xc5, 0xf9, 0xe7, 0x00}, {0xc4, 0xe2, 0x79, 0x2a, 0x00}, + } { + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil || len(decoded.Instrs) != 1 { + t.Fatalf("recognized VEX %x: %+v, %v", code, decoded.Instrs, err) + } + } +} diff --git a/amd64_raw_decode_test.go b/amd64_raw_decode_test.go new file mode 100644 index 000000000..0d71f12a6 --- /dev/null +++ b/amd64_raw_decode_test.go @@ -0,0 +1,356 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestAMD64RawDirectiveSequenceUsesSemanticLowering(t *testing.T) { + source := `TEXT rawLoad(SB), NOSPLIT, $256-0 + MOVQ $0, R11 + BYTE $0x4e + BYTE $0x0f + BYTE $0xb7 + BYTE $0x7c + BYTE $0x5c + BYTE $0x78 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "rawLoad": {Name: "rawLoad", Ret: Void}, + }, + }) + if err != nil { + t.Fatalf("Translate() error = %v", err) + } + if strings.Contains(ir, "module asm") { + t.Fatalf("raw instructions bypassed semantic lowering:\n%s", ir) + } + if !strings.Contains(ir, "load i16") || !strings.Contains(ir, "zext i16") { + t.Fatalf("decoded MOVWQZX semantics missing:\n%s", ir) + } +} + +func TestX86RawOnlyFunctionUsesSemanticLowering(t *testing.T) { + for _, test := range []struct { + name string + goarch string + triple string + source string + }{ + { + name: "amd64", + goarch: "amd64", + triple: "x86_64-unknown-linux-gnu", + source: "TEXT raw(SB), NOSPLIT, $0-0\nBYTE $0x90\nBYTE $0xc3\n", + }, + { + name: "386", + goarch: "386", + triple: "i386-unknown-linux-gnu", + source: "TEXT raw(SB), NOSPLIT, $0-0\nBYTE $0x90\nBYTE $0xc3\n", + }, + } { + t.Run(test.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, test.source) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + ir, err := Translate(file, Options{ + Goarch: test.goarch, + TargetTriple: test.triple, + Sigs: map[string]FuncSig{ + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err != nil { + t.Fatalf("Translate() error = %v", err) + } + if strings.Contains(ir, "module asm") { + t.Fatalf("raw-only function bypassed instruction validation:\n%s", ir) + } + if !strings.Contains(ir, "define void @raw()") || !strings.Contains(ir, "ret void") { + t.Fatalf("semantic function body missing:\n%s", ir) + } + }) + } +} + +func TestX86RawPCRelativeInstructionOutsideDirectiveGroupFailsClosed(t *testing.T) { + source := `TEXT rawBranch(SB), NOSPLIT, $0-0 + MOVQ $0, AX + BYTE $0xeb + BYTE $0x7f + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatalf("Parse() error = %v", err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "rawBranch": {Name: "rawBranch", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "PC-relative") { + t.Fatalf("Translate() error = %v, want explicit PC-relative raw-instruction rejection", err) + } +} + +func TestX86RawInternalConditionalBranchCompleteFamily(t *testing.T) { + targets := []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } + forms := []struct { + name string + bytes func(condition byte) []byte + }{ + {name: "short_forward", bytes: func(condition byte) []byte { + return []byte{0x70 + condition, 0x01, 0x90, 0xc3} + }}, + {name: "short_backward", bytes: func(condition byte) []byte { + return []byte{0x90, 0x70 + condition, 0xfd, 0xc3} + }}, + {name: "near_forward", bytes: func(condition byte) []byte { + return []byte{0x0f, 0x80 + condition, 0x01, 0, 0, 0, 0x90, 0xc3} + }}, + {name: "near_backward", bytes: func(condition byte) []byte { + return []byte{0x90, 0x0f, 0x80 + condition, 0xf9, 0xff, 0xff, 0xff, 0xc3} + }}, + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for condition := byte(0); condition < 16; condition++ { + for _, form := range forms { + name := fmt.Sprintf("rawJcc_%02x_%s", condition, form.name) + fmt.Fprintf(&source, "TEXT %s(SB), NOSPLIT, $0-0\n", name) + for _, value := range form.bytes(condition) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, amd64LLVMBlockName("raw_jcc")) { + t.Fatalf("synthetic raw branch labels missing:\n%s", ir) + } + compileLLVMToObject(t, llc, target.triple, "raw-jcc-"+target.goarch+".ll", "raw-jcc-"+target.goarch+".o", ir) + }) + } +} + +func TestX86RawMULXReportedEncoding(t *testing.T) { + const source = `TEXT rawMULX(SB), NOSPLIT, $0-0 + BYTE $0xc4 + BYTE $0x42 + BYTE $0xe3 + BYTE $0xf6 + BYTE $0xf6 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawMULX": {Name: "rawMULX", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-mulx.ll", "raw-mulx.o", ir) +} + +func TestX86RawMULXCompleteGoAssemblerForms(t *testing.T) { + // Go's x86 table defines both register and Yml memory sources for MULXL + // and (on amd64) MULXQ. Exercise both widths plus direct, SIB, disp8 and + // base-less disp32 ModRM forms, including extended registers. + targets := []struct { + goarch string + triple string + forms [][]byte + }{ + { + goarch: "amd64", + triple: "x86_64-unknown-linux-gnu", + forms: [][]byte{ + {0xc4, 0x42, 0xe3, 0xf6, 0xf6}, // MULXQ R14, BX, R14 + {0xc4, 0x42, 0x3b, 0xf6, 0xfe}, // MULXL R14, R8, R15 + {0xc4, 0x22, 0xab, 0xf6, 0x5c, 0x88, 0x10}, // MULXQ 16(AX)(R9*4), R10, R11 + {0xc4, 0xe2, 0x6b, 0xf6, 0x5c, 0x48, 0x08}, // MULXL 8(AX)(CX*2), DX, BX + {0xc4, 0xe2, 0xfb, 0xf6, 0x0c, 0x25, 0x78, 0x56, 0x34, 0x12}, // MULXQ 0x12345678, AX, CX + }, + }, + { + goarch: "386", + triple: "i386-unknown-linux-gnu", + forms: [][]byte{ + {0xc4, 0xe2, 0x73, 0xf6, 0xd0}, // MULXL AX, CX, DX + {0xc4, 0xe2, 0x6b, 0xf6, 0x5c, 0x48, 0x08}, // MULXL 8(AX)(CX*2), DX, BX + {0xc4, 0xe2, 0x7b, 0xf6, 0x0d, 0x78, 0x56, 0x34, 0x12}, // MULXL 0x12345678, AX, CX + }, + }, + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range targets { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for i, encoding := range target.forms { + name := fmt.Sprintf("rawMULX_%d", i) + fmt.Fprintf(&source, "TEXT %s(SB), NOSPLIT, $0-0\n", name) + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + source.WriteString("\tRET\n") + sigs[name] = FuncSig{Name: name, Ret: Void} + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.goarch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-mulx-forms.ll", "raw-mulx-forms.o", ir) + }) + } +} + +func TestX86RawMULXRejectsLayoutDependentAddressForms(t *testing.T) { + tests := []struct { + name string + encoding []byte + }{ + {name: "address_size_override", encoding: []byte{0x67, 0xc4, 0xe2, 0x6b, 0xf6, 0x5c, 0x48, 0x08}}, + {name: "rip_relative", encoding: []byte{0xc4, 0xe2, 0xfb, 0xf6, 0x0d, 0, 0, 0, 0}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawMULX(SB), NOSPLIT, $0-0\n") + for _, value := range test.encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawMULX": {Name: "rawMULX", Ret: Void}}}) + if err == nil || !strings.Contains(err.Error(), "source-layout safe") { + t.Fatalf("Translate() error = %v, want source-layout safety rejection", err) + } + }) + } +} + +func TestX86RawForwardJumpSkipsEmbeddedDataButStillLowersReachableCode(t *testing.T) { + const source = `TEXT rawSignature(SB), NOSPLIT, $0-0 + BYTE $0xeb + BYTE $0x04 + LONG $0xe77da1c4 + BYTE $0xc3 +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawSignature": {Name: "rawSignature", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "module asm") || !strings.Contains(ir, "ret void") { + t.Fatalf("reachable raw control flow was not lowered semantically:\n%s", ir) + } +} + +func encodeX86RawRetpoline(register int) []byte { + return []byte{0xe8, 0x04, 0, 0, 0, 0xf3, 0x90, 0xeb, 0xfc, byte(0x48 | (register&8)>>1), 0x89, byte(0x04 | (register&7)<<3), 0x24, 0xc3} +} + +func TestX86RawRetpolineFamilyLowersToIndirectTailCalls(t *testing.T) { + names := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + var source strings.Builder + sigs := map[string]FuncSig{} + for register, name := range names { + if register == 4 { + continue + } + function := "retpoline" + name + fmt.Fprintf(&source, "TEXT %s(SB),NOSPLIT,$0-0\n", function) + for _, value := range encodeX86RawRetpoline(register) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[function] = FuncSig{Name: function, Ret: Void} + if got, ok := x86RawRetpolineTarget(encodeX86RawRetpoline(register)); !ok || got != Reg(name) { + t.Fatalf("decoded retpoline register %d as %s, ok=%v", register, got, ok) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "module asm") || !strings.Contains(ir, "call i64 %") || !strings.Contains(ir, "ret void") { + t.Fatalf("raw retpolines were not lowered as semantic indirect tail calls:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-retpoline.ll", "amd64-raw-retpoline.o", ir) +} + +func TestX86RawRetpolineDecoderRejectsNearMisses(t *testing.T) { + for _, offset := range []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 13} { + nearMiss := append([]byte(nil), encodeX86RawRetpoline(0)...) + nearMiss[offset] ^= 1 + if _, ok := x86RawRetpolineTarget(nearMiss); ok { + t.Fatalf("retpoline decoder accepted byte %d near miss %#v", offset, nearMiss) + } + } + if _, ok := x86RawRetpolineTarget(encodeX86RawRetpoline(4)); ok { + t.Fatal("retpoline decoder accepted impossible SP target") + } +} diff --git a/amd64_raw_descriptor_alias_test.go b/amd64_raw_descriptor_alias_test.go new file mode 100644 index 000000000..7a5938e4d --- /dev/null +++ b/amd64_raw_descriptor_alias_test.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawDescriptorStoreAliases(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + want string + }{ + {name: "sldt-memory-amd64", code: []byte{0x0f, 0x00, 0x07}, mode: 64, want: "SLDTW"}, + {name: "sldt-memory-386", code: []byte{0x0f, 0x00, 0x07}, mode: 32, want: "SLDTW"}, + {name: "sldt-register-word", code: []byte{0x66, 0x0f, 0x00, 0xc0}, mode: 64, want: "SLDTW"}, + {name: "sldt-register-long", code: []byte{0x0f, 0x00, 0xc0}, mode: 64, want: "SLDTL"}, + {name: "sldt-register-quad", code: []byte{0x48, 0x0f, 0x00, 0xc0}, mode: 64, want: "SLDTQ"}, + {name: "str-memory-amd64", code: []byte{0x0f, 0x00, 0x0f}, mode: 64, want: "STRW"}, + {name: "str-memory-386", code: []byte{0x0f, 0x00, 0x0f}, mode: 32, want: "STRW"}, + {name: "str-register-word", code: []byte{0x66, 0x0f, 0x00, 0xc8}, mode: 64, want: "STRW"}, + {name: "str-register-long", code: []byte{0x0f, 0x00, 0xc8}, mode: 64, want: "STRL"}, + {name: "str-register-quad", code: []byte{0x48, 0x0f, 0x00, 0xc8}, mode: 64, want: "STRQ"}, + {name: "smsw-memory-amd64", code: []byte{0x0f, 0x01, 0x27}, mode: 64, want: "SMSWW"}, + {name: "smsw-memory-386", code: []byte{0x0f, 0x01, 0x27}, mode: 32, want: "SMSWW"}, + {name: "smsw-register-word", code: []byte{0x66, 0x0f, 0x01, 0xe0}, mode: 64, want: "SMSWW"}, + {name: "smsw-register-long", code: []byte{0x0f, 0x01, 0xe0}, mode: 64, want: "SMSWL"}, + {name: "smsw-register-quad", code: []byte{0x48, 0x0f, 0x01, 0xe0}, mode: 64, want: "SMSWQ"}, + } { + t.Run(tc.name, func(t *testing.T) { + inst, err := x86asm.Decode(tc.code, tc.mode) + if err != nil { + t.Fatal(err) + } + syntax, err := decodedX86GoSyntax(inst, tc.code) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(syntax, tc.want+" ") { + t.Fatalf("raw %x decoded as %q, want %s", tc.code, syntax, tc.want) + } + }) + } +} + +func TestX86RawDescriptorStoreCompiles(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawDescriptorStore(SB),$0-0\n") + codes := [][]byte{ + {0x0f, 0x00, 0x07}, {0x0f, 0x00, 0x0f}, {0x0f, 0x01, 0x27}, + {0x66, 0x0f, 0x00, 0xc0}, {0x0f, 0x00, 0xc0}, + {0x66, 0x0f, 0x00, 0xc8}, {0x0f, 0x00, 0xc8}, + {0x66, 0x0f, 0x01, 0xe0}, {0x0f, 0x01, 0xe0}, + } + if target.goarch == "amd64" { + codes = append(codes, + []byte{0x48, 0x0f, 0x00, 0xc0}, + []byte{0x48, 0x0f, 0x00, 0xc8}, + []byte{0x48, 0x0f, 0x01, 0xe0}, + ) + } + for _, code := range codes { + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawDescriptorStore": {Name: "rawDescriptorStore", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-descriptor-store.ll", "raw-descriptor-store.o", ir) + }) + } +} diff --git a/amd64_raw_displacement_test.go b/amd64_raw_displacement_test.go new file mode 100644 index 000000000..d3e18ebe8 --- /dev/null +++ b/amd64_raw_displacement_test.go @@ -0,0 +1,180 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func x86RawLEADisplacementEncoding(mode, width, addressBits, shape, scale int, displacement int32) []byte { + var encoding []byte + if width == 16 { + encoding = append(encoding, 0x66) + } + if addressBits != mode { + encoding = append(encoding, 0x67) + } + if width == 64 { + encoding = append(encoding, 0x48) + } + encoding = append(encoding, 0x8d) + if addressBits == 16 { + // mod=10, reg=CX, r/m=BX+SI; the displacement is physical disp16. + encoding = append(encoding, 0x88) + return binary.LittleEndian.AppendUint16(encoding, uint16(displacement)) + } + switch shape { + case 0: // AX + disp32 + encoding = append(encoding, 0x88) + case 1: // AX + BX*scale + disp32 + encoding = append(encoding, 0x8c, byte(scale<<6|3<<3)) + case 2: // AX*scale + disp32; no base is encoded. + encoding = append(encoding, 0x0c, byte(scale<<6|5)) + case 3: // disp32; SIB has neither a base nor an index. + encoding = append(encoding, 0x0c, 0x25) + default: + panic("unknown raw LEA displacement fixture shape") + } + return binary.LittleEndian.AppendUint32(encoding, uint32(displacement)) +} + +func x86RawDisplacementDirectives(encoding []byte) string { + var source strings.Builder + for _, value := range encoding { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func TestX86RawDisplacementRetainsPhysicalSignedField(t *testing.T) { + // Intel SDM Vol. 2, section 2.2.1: ModR/M/SIB disp32 is sign-extended + // in a 64-bit effective address. x/arch v0.14 exposes it as uint32. + for _, mode := range []int{32, 64} { + for _, shape := range []int{0, 1, 2, 3} { + for _, displacement := range []int32{-1 << 31, -129, -128, -64, -1, 0, 127, 128, 1<<31 - 1} { + t.Run(fmt.Sprintf("mode%d/shape%d/%d", mode, shape, displacement), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, mode, mode, shape, 2, displacement) + inst, err := x86asm.Decode(encoding, mode) + if err != nil || inst.Len != len(encoding) || inst.Op != x86asm.LEA { + t.Fatalf("physical LEA decode: %+v, %v", inst, err) + } + syntax, err := decodedX86GoSyntax(inst, encoding) + if err != nil { + t.Fatal(err) + } + instruction := x86RawDisplacementDecodedLEA(t, mode, encoding) + if got := instruction.Args[0].Mem.Off; got != int64(displacement) { + t.Fatalf("physical signed disp32 became %d, want %d: %q", got, displacement, syntax) + } + }) + } + } + } + // The independently discovered go-cv-simd bytes use REX.R/X and no base. + encoding := []byte{0x4e, 0x8d, 0x04, 0x9d, 0xc0, 0xff, 0xff, 0xff} + inst, err := x86asm.Decode(encoding, 64) + if err != nil { + t.Fatal(err) + } + if syntax, err := decodedX86GoSyntax(inst, encoding); err != nil || syntax != "LEAQ -0x40(R11*4), R8" { + t.Fatalf("original no-base extended-index LEA: %q, %v", syntax, err) + } +} + +func x86RawDisplacementDecodedLEA(t *testing.T, mode int, encoding []byte) Instr { + t.Helper() + arch := "amd64" + if mode == 32 { + arch = "386" + } + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+x86RawDisplacementDirectives(encoding)+"RET\n") + if err != nil { + t.Fatal(err) + } + function, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatal(err) + } + for _, instruction := range function.Instrs { + if strings.HasPrefix(string(instruction.Op), "LEA") { + if len(instruction.Args) != 2 || instruction.Args[0].Kind != OpMem { + t.Fatalf("lost physical LEA memory operand: %+v", instruction) + } + return instruction + } + } + t.Fatal("decoded raw source lost its LEA instruction") + return Instr{} +} + +func TestX86RawDisplacementDoesNotSignExtendMOVAbsoluteOffset(t *testing.T) { + for _, opcode := range []byte{0xa0, 0xa1, 0xa2, 0xa3} { + for _, offset := range []uint64{0x80000000, 0xffffffc0, 0x1122334480000000} { + t.Run(fmt.Sprintf("%x/%x", opcode, offset), func(t *testing.T) { + encoding := binary.LittleEndian.AppendUint64([]byte{0x48, opcode}, offset) + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Len != len(encoding) { + t.Fatalf("physical MOV moffs decode: %+v, %v", inst, err) + } + syntax, err := decodedX86GoSyntax(inst, encoding) + if err != nil { + t.Fatal(err) + } + if syntax != x86asm.GoSyntax(inst, 0, nil) { + t.Fatalf("absolute MOV spelling changed: %q", syntax) + } + found := false + normalized := normalizeDecodedX86MemoryDisplacements(inst) + for _, operand := range normalized.Args { + if memory, ok := operand.(x86asm.Mem); ok { + found = true + if uint64(memory.Disp) != offset { + t.Fatalf("absolute native offset narrowed: %q", syntax) + } + } + } + if !found { + t.Fatalf("missing absolute memory operand: %q", syntax) + } + }) + } + } +} + +func TestX86RawDisplacementLEAAllWidthsAndFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + for _, shape := range []int{0, 1, 2, 3} { + t.Run(fmt.Sprintf("%s/width%d/shape%d", target.triple, width, shape), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, width, mode, shape, 2, -64) + body := "@MOV x+0(FP),AX\n@MOV $3,BX\n@MOV $0,CX\n" + x86RawDisplacementDirectives(encoding) + "@MOV CX,ret+@WORD(FP)\nRET" + source, sig, _ := x86SourceStackNumericalFixture(target.arch, body) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-disp.ll", "raw-disp.o", ir) + }) + } + } + } +} diff --git a/amd64_raw_double_dword.go b/amd64_raw_double_dword.go new file mode 100644 index 000000000..a28a552ca --- /dev/null +++ b/amd64_raw_double_dword.go @@ -0,0 +1,156 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Opcode E6 has three Go 1.27 conversion grammars. VEX.W is ignored; EVEX.W +// distinguishes the dword/double family from qword/double instructions. LL +// selects the vector width except when EVEX.b enables rounding or SAE on a +// register-source 512-bit double-to-dword form. +func decodedX86PackedDoubleDwordInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedDoubleDwordInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedDoubleDwordInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.opcode != 0xe6 || (p.pp != 1 && p.pp != 2 && p.pp != 3) { + return Instr{}, 0, false, nil + } + if p.pp != 2 && (!p.evex || p.w) { + return decodedX86PackedNumericConversionInstructionWithMemory(code, mode, memory) + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed double/dword conversion: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("reserved vvvv/V' field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.evex { + if p.w != (p.pp != 2) { + // EVEX.W selects another conversion family, such as VCVTQQ2PD. + return Instr{}, 0, false, nil + } + if !p.fixed || p.zero && p.mask == 0 { + return fail("invalid EVEX fixed, mask or zeroing field") + } + } else if p.vectorLength > 1 { + return fail("invalid VEX vector length") + } + registerSource := code[p.modRM]>>6 == 3 + if mode == 32 { + if p.pp == 2 && (p.r != 0 || registerSource && (p.b != 0 || p.x != 0)) { + return fail("signed dword-to-double has no extended physical vector register in 32-bit mode") + } + if !p.evex && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended VEX register in 32-bit mode") + } + if p.evex && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + } + rounding := p.evex && p.broadcast && registerSource + broadcast := p.evex && p.broadcast && !registerSource + if p.pp == 2 && rounding { + return fail("dword-to-double has no SAE or rounding form") + } + if p.evex && !rounding && p.vectorLength > 2 { + return fail("reserved EVEX vector length") + } + if p.pp != 2 && rounding && !p.evex { + return fail("rounding or SAE requires EVEX") + } + + vectorLength := p.vectorLength + if rounding { + vectorLength = 2 // Register-source SAE/rounding implies 512 bits. + } + var op Op + var sourcePrefix, destinationPrefix string + var memoryScale int + if p.pp == 2 { + op = "VCVTDQ2PD" + destinationPrefix = [...]string{"X", "Y", "Z"}[vectorLength] + sourcePrefix = "X" + if vectorLength == 2 { + sourcePrefix = "Y" + } + memoryScale = 8 << vectorLength + } else { + stem := "VCVTPD2DQ" + if p.pp == 1 { + stem = "VCVTTPD2DQ" + } + sourcePrefix = [...]string{"X", "Y", "Z"}[vectorLength] + destinationPrefix = "X" + if vectorLength == 2 { + destinationPrefix = "Y" + } + op = Op(stem) + if vectorLength < 2 { + op = Op(stem + [...]string{"X", "Y"}[vectorLength]) + } + memoryScale = 16 << vectorLength + } + if mode == 32 { + if sourcePrefix == "Z" && registerSource && (p.b != 0 || p.x != 0) { + return fail("Z8-Z31 are not Go assembler source registers in 32-bit mode") + } + if destinationPrefix == "Z" && p.r != 0 { + return fail("Z8-Z31 are not Go assembler destination registers in 32-bit mode") + } + } + if _, exists := amd64PackedDoubleDwordOps[string(op)]; !exists { + return fail("encoding has no Go packed conversion form") + } + if rounding { + if p.pp == 1 { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } else if broadcast { + op += ".BCST" + if p.pp == 2 { + memoryScale = 4 + } else { + memoryScale = 8 + } + } + if p.zero { + op += ".Z" + } + + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix, memoryScale) + } else { + source, consumed, err = memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix) + } + if err != nil { + return Instr{}, 0, true, err + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationPrefix, int(code[p.modRM]>>3&7)+p.r*8))} + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_double_dword_test.go b/amd64_raw_double_dword_test.go new file mode 100644 index 000000000..72e802428 --- /dev/null +++ b/amd64_raw_double_dword_test.go @@ -0,0 +1,234 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawPackedDoubleDwordBaseForms = []struct { + op Op + pp byte + length byte + evex bool + src Reg + dst Reg +}{ + {"VCVTDQ2PD", 2, 0, false, "X1", "X0"}, + {"VCVTDQ2PD", 2, 1, false, "X1", "Y0"}, + {"VCVTDQ2PD", 2, 0, true, "X1", "X0"}, + {"VCVTDQ2PD", 2, 1, true, "X1", "Y0"}, + {"VCVTDQ2PD", 2, 2, true, "Y1", "Z0"}, + {"VCVTPD2DQX", 3, 0, false, "X1", "X0"}, + {"VCVTPD2DQY", 3, 1, false, "Y1", "X0"}, + {"VCVTPD2DQX", 3, 0, true, "X1", "X0"}, + {"VCVTPD2DQY", 3, 1, true, "Y1", "X0"}, + {"VCVTPD2DQ", 3, 2, true, "Z1", "Y0"}, + {"VCVTTPD2DQX", 1, 0, false, "X1", "X0"}, + {"VCVTTPD2DQY", 1, 1, false, "Y1", "X0"}, + {"VCVTTPD2DQX", 1, 0, true, "X1", "X0"}, + {"VCVTTPD2DQY", 1, 1, true, "Y1", "X0"}, + {"VCVTTPD2DQ", 1, 2, true, "Z1", "Y0"}, +} + +func TestX86RawPackedDoubleDwordCompleteGo127BaseForms(t *testing.T) { + if len(amd64PackedDoubleDwordOps) != 9 { + t.Fatalf("packed double/dword grammar has %d entries, want nine", len(amd64PackedDoubleDwordOps)) + } + for _, tc := range x86RawPackedDoubleDwordBaseForms { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s/LL=%d/EVEX=%t/memory=%t", tc.op, tc.length, tc.evex, memory) + t.Run(name, func(t *testing.T) { + modRM := byte(0xc1) + if memory { + modRM = 0x08 + } + var code []byte + if tc.evex { + w := byte(0) + if tc.pp != 2 { + w = 0x80 + } + code = []byte{0x62, 0xf1, 0x7c | tc.pp | w, 0x08 | tc.length<<5, 0xe6, modRM} + } else { + code = []byte{0xc5, 0xf8 | tc.pp | tc.length<<2, 0xe6, modRM} + } + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != tc.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", code, got.Instrs, tc.op) + } + args := got.Instrs[0].Args + if memory { + if args[0].Kind != OpMem || args[0].Mem.Base != AX { + t.Fatalf("memory source %#x = %+v", code, args[0]) + } + } else if args[0].Kind != OpReg || args[0].Reg != tc.src { + t.Fatalf("register source %#x = %+v, want %s", code, args[0], tc.src) + } + wantDestination := tc.dst + if memory { + wantDestination = Reg(string(tc.dst[:1]) + "1") + } + if args[1].Reg != wantDestination { + t.Fatalf("destination %#x = %s, want %s", code, args[1].Reg, wantDestination) + } + }) + } + } +} + +func TestX86RawPackedDoubleDwordEncodingAxes(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + op Op + args []string + }{ + {"VEX-W-ignored", []byte{0xc4, 0xe1, 0xfe, 0xe6, 0xc1}, "VCVTDQ2PD", []string{"X1", "Y0"}}, + {"masked", []byte{0x62, 0xf1, 0x7e, 0x09, 0xe6, 0xc1}, "VCVTDQ2PD", []string{"X1", "K1", "X0"}}, + {"masked-broadcast-zero", []byte{0x62, 0xf1, 0x7e, 0x99, 0xe6, 0x00}, "VCVTDQ2PD.BCST.Z", []string{"0(AX)", "K1", "X0"}}, + {"round-down-zero", []byte{0x62, 0xf1, 0xff, 0xb9, 0xe6, 0xd8}, "VCVTPD2DQ.RD_SAE.Z", []string{"Z0", "K1", "Y3"}}, + {"truncate-SAE", []byte{0x62, 0xf1, 0xfd, 0x18, 0xe6, 0xc1}, "VCVTTPD2DQ.SAE", []string{"Z1", "Y0"}}, + {"extended-source-and-destination", []byte{0x62, 0xa1, 0x7e, 0x08, 0xe6, 0xc1}, "VCVTDQ2PD", []string{"X17", "X16"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86PackedDoubleDwordInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) || got.Op != tc.op { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s", tc.code, got, length, recognized, err, tc.op) + } + if len(got.Args) != len(tc.args) { + t.Fatalf("decode %#x operands = %+v, want %v", tc.code, got.Args, tc.args) + } + for index, arg := range got.Args { + if arg.String() != tc.args[index] { + t.Fatalf("decode %#x operand %d = %s, want %s", tc.code, index, arg.String(), tc.args[index]) + } + } + }) + } + for _, tc := range []struct { + name string + code []byte + off int64 + }{ + {"dword-Z-disp8", []byte{0x62, 0xf1, 0x7e, 0x48, 0xe6, 0x40, 0x02}, 64}, + {"double-Y-disp8", []byte{0x62, 0xf1, 0xff, 0x28, 0xe6, 0x40, 0x02}, 64}, + {"double-broadcast-disp8", []byte{0x62, 0xf1, 0xff, 0x38, 0xe6, 0x40, 0x02}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + got, _, recognized, err := decodedX86PackedDoubleDwordInstruction(tc.code, 64) + if !recognized || err != nil || got.Args[0].Kind != OpMem || got.Args[0].Mem.Off != tc.off { + t.Fatalf("decode %#x = %+v, %v, %v; want offset %d", tc.code, got, recognized, err, tc.off) + } + }) + } + for _, code := range [][]byte{ + {0x62, 0xa1, 0x7e, 0x08, 0xe6, 0xc1}, // X17 -> X16. + {0x62, 0xa1, 0xff, 0x28, 0xe6, 0xc1}, // Y17 -> X16. + } { + got, _, recognized, err := decodedX86PackedDoubleDwordInstruction(code, 32) + // Both directions now preserve native state. Go accepts these + // spellings, but neither can invent physical high mode32 banks. + if !recognized || err == nil { + t.Fatal("mode32 native conversion fabricated a high physical bank", got, err) + } + } + requireX86GoAssemblerResult(t, "386", "TEXT high(SB),4,$0-0\n\tVCVTDQ2PD X17, K1, X16\n\tVCVTPD2DQY Y17, K1, X16\n\tRET\n", true) +} + +func TestX86RawPackedDoubleDwordRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"VEX-vvvv", []byte{0xc5, 0xee, 0xe6, 0xc1}, 64}, + {"EVEX-vvvv", []byte{0x62, 0xf1, 0x6e, 0x08, 0xe6, 0xc1}, 64}, + {"EVEX-zero-without-mask", []byte{0x62, 0xf1, 0x7e, 0x88, 0xe6, 0xc1}, 64}, + {"EVEX-LL3-without-rounding", []byte{0x62, 0xf1, 0x7e, 0x68, 0xe6, 0xc1}, 64}, + {"dword-register-broadcast", []byte{0x62, 0xf1, 0x7e, 0x18, 0xe6, 0xc1}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfe, 0xe6, 0xc1}, 64}, + {"missing-ModRM", []byte{0xc5, 0xfe, 0xe6}, 64}, + {"extended-386", []byte{0xc4, 0x61, 0x7e, 0xe6, 0xc1}, 32}, + {"Z-destination-386", []byte{0x62, 0xe1, 0x7e, 0x48, 0xe6, 0xc1}, 32}, + {"Z-source-386", []byte{0x62, 0xb1, 0xff, 0x48, 0xe6, 0xc1}, 32}, + {"extended-memory-386", []byte{0x62, 0xb1, 0x7e, 0x08, 0xe6, 0x01}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86PackedDoubleDwordInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } +} + +func TestX86RawPackedDoubleDwordCompilesEveryBaseForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawdoubleconversion(SB),4,$0-0\n") + encodings := make([][]byte, 0, len(x86RawPackedDoubleDwordBaseForms)+4) + for _, form := range x86RawPackedDoubleDwordBaseForms { + modRM := byte(0xc1) + var code []byte + if form.evex { + w := byte(0) + if form.pp != 2 { + w = 0x80 + } + code = []byte{0x62, 0xf1, 0x7c | form.pp | w, 0x08 | form.length<<5, 0xe6, modRM} + } else { + code = []byte{0xc5, 0xf8 | form.pp | form.length<<2, 0xe6, modRM} + } + encodings = append(encodings, code) + } + encodings = append(encodings, + []byte{0x62, 0xf1, 0x7e, 0x99, 0xe6, 0x00}, // Dword broadcast, mask, zero. + []byte{0x62, 0xf1, 0xff, 0xb9, 0xe6, 0xd8}, // Double embedded rounding. + []byte{0x62, 0xf1, 0xff, 0x38, 0xe6, 0x40, 0x02}, // Double broadcast disp8. + ) + if target.goarch == "amd64" { + encodings = append(encodings, []byte{0x62, 0xa1, 0x7e, 0x08, 0xe6, 0xc1}) // Physical high X. + } + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawdoubleconversion": {Name: "rawdoubleconversion", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-double-conversion.ll", "raw-double-conversion.o", ir) + }) + } +} diff --git a/amd64_raw_duplicate_literal_test.go b/amd64_raw_duplicate_literal_test.go new file mode 100644 index 000000000..854638eef --- /dev/null +++ b/amd64_raw_duplicate_literal_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86RawDuplicateMoveConstant(spec rawDuplicateMoveSpec, encoding string, width int, mask int, zeroing bool) []byte { + var code []byte + switch encoding { + case "legacy": + code = encodeX86RawLegacyDuplicateMove(spec, 0, 0) + case "vex": + code = encodeX86RawVEXDuplicateMove(spec, byte(map[int]int{16: 0, 32: 1}[width]), 0, 0) + case "evex": + code = encodeX86RawEVEXDuplicateMove(spec, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), mask, zeroing, 0, 0) + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if spec.double && width == 16 { + dataWidth = 8 + } + for i := 0; i < dataWidth; i++ { + code = append(code, byte(i*37+3)) + } + return code +} + +func TestDecodeX86RawDuplicateMoveConstantCompleteFamily(t *testing.T) { + for _, spec := range rawDuplicateMoveSpecs { + for _, encoding := range []string{"legacy", "vex", "evex"} { + widths := []int{16} + if encoding != "legacy" { + widths = append(widths, 32) + } + if encoding == "evex" { + widths = append(widths, 64) + } + for _, width := range widths { + mask := 0 + zeroing := false + if encoding == "evex" && width == 64 { + mask, zeroing = 3, true + } + name := fmt.Sprintf("%s/%s/%d", spec.vector, encoding, width) + t.Run(name, func(t *testing.T) { + code := x86RawDuplicateMoveConstant(spec, encoding, width, mask, zeroing) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := spec.vector + if encoding == "legacy" { + want = spec.legacy + } + if zeroing { + want += ".Z" + } + if len(got) != 2 || got[0].Op != want || + got[0].Args[0].Kind != OpSym || !got[0].x86RIPLiteral || + got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, got, want) + } + }) + } + } + } +} + +func TestDecodeX86RawDuplicateMoveConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawDuplicateMoveConstant(rawDuplicateMoveSpecs[0], "vex", 32, 0, false) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[5] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[5] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe duplicate-move RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawDuplicateMoveConstantObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, spec := range rawDuplicateMoveSpecs { + code := x86RawDuplicateMoveConstant(spec, "evex", 64, 3, true) + name := fmt.Sprintf("rawDuplicateConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-duplicate-literal.ll", "raw-duplicate-literal.o", ir) + }) + } +} diff --git a/amd64_raw_duplicate_test.go b/amd64_raw_duplicate_test.go new file mode 100644 index 000000000..92093b1b4 --- /dev/null +++ b/amd64_raw_duplicate_test.go @@ -0,0 +1,343 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" +) + +type rawDuplicateMoveSpec struct { + legacyPrefix byte + opcode byte + legacy Op + vector Op + double bool +} + +var rawDuplicateMoveSpecs = []rawDuplicateMoveSpec{ + {legacyPrefix: 0xf2, opcode: 0x12, legacy: "MOVDDUP", vector: "VMOVDDUP", double: true}, + {legacyPrefix: 0xf3, opcode: 0x16, legacy: "MOVSHDUP", vector: "VMOVSHDUP"}, + {legacyPrefix: 0xf3, opcode: 0x12, legacy: "MOVSLDUP", vector: "VMOVSLDUP"}, +} + +func encodeX86RawLegacyDuplicateMove(spec rawDuplicateMoveSpec, source, destination int) []byte { + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + code := []byte{spec.legacyPrefix} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, spec.opcode, byte(0xc0|destination&7<<3|source&7)) +} + +func encodeX86RawVEXDuplicateMove(spec rawDuplicateMoveSpec, vectorBits byte, source, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^source>>3)&1)<<5 | 1 + pp := byte(2) + if spec.double { + pp = 3 + } + p1 := byte(0x78 | vectorBits<<2 | pp) + return []byte{0xc4, p0, p1, spec.opcode, byte(0xc0 | destination&7<<3 | source&7)} +} + +func encodeX86RawEVEXDuplicateMove(spec rawDuplicateMoveSpec, vectorBits byte, mask int, zeroing bool, source, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^source>>4)&1)<<6 | + byte((^source>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + pp := byte(2) + if spec.double { + pp = 3 + } + p1 := byte(0x7c | pp) + if spec.double { + p1 |= 0x80 + } + p2 := vectorBits<<5 | 0x08 | byte(mask) + if zeroing { + p2 |= 0x80 + } + return []byte{0x62, p0, p1, p2, spec.opcode, byte(0xc0 | destination&7<<3 | source&7)} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVMOVSHDUP(t *testing.T) { + // vmovshdup %xmm0, %xmm1; + // vpermilps $208, %xmm7, %xmm0 + code := []byte{0xc5, 0xfa, 0x16, 0xc8, 0xc4, 0xe3, 0x79, 0x04, 0xc7, 0xd0} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VMOVSHDUP sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VMOVSHDUP X0, X1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawDuplicateMoveCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawDuplicateMoveSpecs { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyDuplicateMove(spec, source, destination) + got, length, ok, err := decodedX86DuplicateMoveInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d", spec.legacy, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXDuplicateMove(spec, vectorBits, source, destination) + got, length, ok, err := decodedX86DuplicateMoveInstruction(code, 64) + want := fmt.Sprintf("%s %s%d, %s%d", spec.vector, vectorName, source, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{{}, {mask: 3}, {mask: 7, zeroing: true, suffix: ".Z"}} { + for source := 0; source < 32; source++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXDuplicateMove(spec, vectorBits, masking.mask, masking.zeroing, source, destination) + got, length, ok, err := decodedX86DuplicateMoveInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", vectorName, source)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(spec.vector) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + if count != 29952 { + t.Fatalf("covered %d duplicate-move register encodings, want 29952", count) + } +} + +func TestDecodedX86RawDuplicateMoveMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0xf3, 0x47, 0x0f, 0x16, 0x7c, 0x8b, 0x01} + got, length, ok, err := decodedX86DuplicateMoveInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "MOVSHDUP" || got.Args[1].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x01, 0x7e, 0x16, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86DuplicateMoveInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VMOVSHDUP" || got.Args[1].String() != "Y15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0xf1, 0xff, 0x8b, 0x12, 0x50, 0x02} + got, length, ok, err = decodedX86DuplicateMoveInstruction(evexMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 16} + if err != nil || !ok || length != len(evexMemory) || got.Op != "VMOVDDUP.Z" || got.Args[1].String() != "K3" || got.Args[2].String() != "X2" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXDuplicateMove(rawDuplicateMoveSpecs[1], 0, 7, 7) + validEVEX := encodeX86RawEVEXDuplicateMove(rawDuplicateMoveSpecs[1], 2, 0, false, 7, 5) + for name, invalid := range map[string][]byte{ + "vex W": {validVEX[0], validVEX[1], validVEX[2] | 0x80, validVEX[3], validVEX[4]}, + "vex vvvv": {validVEX[0], validVEX[1], validVEX[2] &^ 0x08, validVEX[3], validVEX[4]}, + "evex fixed bit": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x04, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex wrong W": {validEVEX[0], validEVEX[1], validEVEX[2] | 0x80, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex vvvvv": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x08, validEVEX[3] &^ 0x08, validEVEX[4], validEVEX[5]}, + "evex broadcast": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x10, validEVEX[4], validEVEX[5]}, + "evex reserved length": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x60, validEVEX[4], validEVEX[5]}, + "zero without mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x80, validEVEX[4], validEVEX[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86DuplicateMoveInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + highZ386 := encodeX86RawEVEXDuplicateMove(rawDuplicateMoveSpecs[1], 2, 0, false, 8, 2) + if instruction, _, matched, decodeErr := decodedX86DuplicateMoveInstruction(highZ386, 32); !matched || decodeErr == nil { + t.Fatalf("386 high-Z encoding %x decoded as %+v, ok=%v, err=%v", highZ386, instruction, matched, decodeErr) + } +} + +func TestX86DuplicateMoveRawGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + MOVDDUP X6, X7 + MOVSHDUP X6, X7 + MOVSLDUP X6, X7 + VMOVDDUP Y6, Y7 + VMOVSHDUP X20, K1, X21 + VMOVSLDUP.Z Y20, K2, Y21 + VMOVDDUP Z6, K3, Z7 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "MOVSHDUP X8, X0", + "VMOVDDUP Z8, Z0", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawDuplicateMoveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_duplicate_move(SB),NOSPLIT,$0-0\n") + for _, spec := range rawDuplicateMoveSpecs { + var encodings [][]byte + if target.goarch == "amd64" { + encodings = append(encodings, + encodeX86RawLegacyDuplicateMove(spec, 15, 14), + encodeX86RawVEXDuplicateMove(spec, 1, 15, 14), + encodeX86RawEVEXDuplicateMove(spec, 2, 7, true, 31, 30), + ) + } else { + encodings = append(encodings, + encodeX86RawLegacyDuplicateMove(spec, 7, 6), + encodeX86RawVEXDuplicateMove(spec, 1, 7, 6), + encodeX86RawEVEXDuplicateMove(spec, 0, 1, false, 20, 21), + encodeX86RawEVEXDuplicateMove(spec, 2, 2, true, 6, 7), + ) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "raw_duplicate_move": {Name: "raw_duplicate_move", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-duplicate-move-"+target.name+".ll", "raw-duplicate-move-"+target.name+".o", ll) + }) + } +} + +func TestAMD64DuplicateMoveRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = ` +TEXT duplicateMoveSemantics(SB),NOSPLIT,$0-32 + MOVQ out+0(FP), AX + MOVQ source+8(FP), BX + MOVQ old+16(FP), CX + KMOVQ mask+24(FP), K1 + VMOVUPS (BX), Y0 + VMOVSHDUP Y0, Y1 + VMOVUPS Y1, 0(AX) + VMOVSLDUP Y0, Y2 + VMOVUPS Y2, 32(AX) + VMOVDDUP (BX), X3 + VMOVUPS X3, 64(AX) + VMOVUPS (CX), Y4 + VMOVSHDUP Y0, K1, Y4 + VMOVUPS Y4, 80(AX) + VMOVUPS (CX), Y4 + VMOVSLDUP.Z Y0, K1, Y4 + VMOVUPS Y4, 112(AX) + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + frame := FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }} + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ll, err := Translate(file, Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "duplicateMoveSemantics": {Name: "duplicateMoveSemantics", Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: frame}, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void duplicateMoveSemantics(uint32_t *, const uint32_t *, const uint32_t *, uint64_t); +int main(void) { + uint32_t source[8] = {10, 11, 20, 21, 30, 31, 40, 41}; + uint32_t old[8] = {100, 101, 102, 103, 104, 105, 106, 107}; + uint32_t out[36] = {0}; + uint64_t mask = 0x55; + duplicateMoveSemantics(out, source, old, mask); + uint32_t sh[8] = {11, 11, 21, 21, 31, 31, 41, 41}; + uint32_t sl[8] = {10, 10, 20, 20, 30, 30, 40, 40}; + uint32_t dd[4] = {10, 11, 10, 11}; + for (int i = 0; i < 8; i++) if (out[i] != sh[i]) return 1 + i; + for (int i = 0; i < 8; i++) if (out[8+i] != sl[i]) return 20 + i; + for (int i = 0; i < 4; i++) if (out[16+i] != dd[i]) return 40 + i; + for (int i = 0; i < 8; i++) { + uint32_t want = (mask >> i) & 1 ? sh[i] : old[i]; + if (out[20+i] != want) return 50 + i; + } + for (int i = 0; i < 8; i++) { + uint32_t want = (mask >> i) & 1 ? sl[i] : 0; + if (out[28+i] != want) return 70 + i; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "duplicate_move", triple, ll, mainC, runPrefix) +} diff --git a/amd64_raw_endbranch.go b/amd64_raw_endbranch.go new file mode 100644 index 000000000..9f58ee7ca --- /dev/null +++ b/amd64_raw_endbranch.go @@ -0,0 +1,21 @@ +package plan9asm + +// ENDBR32/ENDBR64 have fixed, operand-free encodings (Intel SDM, volume 2). +// x86asm does not decode them yet. Preserve the actual instruction rather than +// replacing the CET state transition with a NOP. Go names only ENDBR64, so +// x86Encoded also keeps ENDBR32 confined to validated raw input. +func decodedX86EndBranchInstruction(code []byte) (Instr, int, bool) { + if len(code) < 4 || code[0] != 0xf3 || code[1] != 0x0f || code[2] != 0x1e { + return Instr{}, 0, false + } + var op Op + switch code[3] { + case 0xfa: + op = "ENDBR64" + case 0xfb: + op = "ENDBR32" + default: + return Instr{}, 0, false + } + return Instr{Op: op, Raw: string(op), x86Encoded: true}, 4, true +} diff --git a/amd64_raw_endbranch_test.go b/amd64_raw_endbranch_test.go new file mode 100644 index 000000000..a71c04ae1 --- /dev/null +++ b/amd64_raw_endbranch_test.go @@ -0,0 +1,171 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Intel SDM lists the operand-free F3 0F 1E FA/FB ENDBR64/ENDBR32 +// encodings. Go 1.27 names only ENDBR64; either encoding is valid BYTE data. +func TestX86RawEndBranchFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + for _, form := range []struct { + op Op + last byte + }{ + {"ENDBR64", 0xfa}, + {"ENDBR32", 0xfb}, + } { + t.Run(string(form.op)+"/"+target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT endbranch(SB),$0-0 + MOVL $17, AX + CMPL AX, AX + BYTE $0xf3; BYTE $0x0f; BYTE $0x1e; BYTE $0x%02x + SETEQ BL + ADDL $3, AX + RET +`, form.last) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 7 || decoded.Instrs[3].Op != form.op { + t.Fatalf("wrong instruction boundaries: %+v", decoded.Instrs) + } + ir, err := Translate(file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"endbranch": {Name: "endbranch", Ret: I32}}, + }) + if err != nil { + t.Fatal(err) + } + want := fmt.Sprintf(".byte 0xf3, 0x0f, 0x1e, 0x%02x", form.last) + if !strings.Contains(ir, want) { + t.Fatalf("IR lost ENDBR encoding %q", want) + } + compileLLVMToObject(t, llc, target.triple, "endbranch.ll", "endbranch.o", ir) + }) + } + } +} + +func TestX86RawEndBranchPreservesBoundaries(t *testing.T) { + // JE targets the second ENDBR after a first four-byte ENDBR and NOP. + code := []byte{0x74, 5, 0xf3, 0x0f, 0x1e, 0xfa, 0x90, 0xf3, 0x0f, 0x1e, 0xfb, 0xc3} + for _, mode := range []int{32, 64} { + instructions, err := decodeX86RawDirectiveGroup(code, mode, 0, "endbranch fixture", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + var ops []Op + for _, instruction := range instructions { + if instruction.Op == "ENDBR32" || instruction.Op == "ENDBR64" { + ops = append(ops, instruction.Op) + } + } + if len(ops) != 2 || ops[0] != "ENDBR64" || ops[1] != "ENDBR32" { + t.Fatalf("mode %d: ENDBR instructions = %v", mode, ops) + } + for _, incomplete := range [][]byte{{0xf3}, {0xf3, 0x0f}, {0xf3, 0x0f, 0x1e}} { + if _, err := decodeX86RawDirectiveGroup(incomplete, mode, 0, "truncated ENDBR", map[string]bool{}); err == nil { + t.Fatalf("accepted truncated encoding %x", incomplete) + } + } + } +} + +func TestX86EndBranch32RemainsRawOnly(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, instruction := range []string{"ENDBR32", "ENDBR32 AX", "ENDBR32.Z"} { + source := "TEXT invalid(SB),$0-0\n" + instruction + "\nRET\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, err := Translate(file, Options{ + Goarch: arch, + Sigs: map[string]FuncSig{"invalid": {Name: "invalid", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted Go-undefined text form %q on %s", instruction, arch) + } + } + } +} + +func TestX86RawEndBranchRuntimePreservesState(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64; required amd64 CI covers this host-inapplicable test") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + // This checks GP/flags preservation, not operating-system CET enforcement. + // Both instructions remain executable bytes, even on pre-CET CPUs. + for _, last := range []byte{0xfa, 0xfb} { + t.Run(fmt.Sprintf("%02x", last), func(t *testing.T) { + source := fmt.Sprintf(`TEXT endbranchstate(SB),NOSPLIT,$0-8 + MOVQ out+0(FP), DI + MOVL $17, AX + CMPL AX, AX + BYTE $0xf3; BYTE $0x0f; BYTE $0x1e; BYTE $0x%02x + SETEQ 0(DI) + SETCS 1(DI) + MOVL AX, 4(DI) + RET +`, last) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{ + "endbranchstate": { + Name: "endbranchstate", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +struct result { uint8_t zf, cf, padding[2]; uint32_t value; }; +extern void endbranchstate(struct result *); +int main(void) { + struct result out = {0}; + endbranchstate(&out); + return out.zf != 1 || out.cf != 0 || out.value != 17; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_endbranch_state", triple, ir, mainC, runPrefix) + }) + } +} diff --git a/amd64_raw_enqueue.go b/amd64_raw_enqueue.go new file mode 100644 index 000000000..c2f3336c9 --- /dev/null +++ b/amd64_raw_enqueue.go @@ -0,0 +1,121 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Decode the shared ModRM/SIB grammar using an equivalent LEA understood by +// x/arch. ENQCMD's register width follows address size, not REX.W or 0x66. +func decodedX86EnqueueInstruction(code []byte, mode int) (Instr, int, bool, error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + var mandatory, segment, rex byte + addressOverride, locked := false, false + for i < len(code) { + switch code[i] { + case 0xf2, 0xf3: + mandatory, rex = code[i], 0 + case 0x64, 0x65: + segment, rex = code[i], 0 + case 0x26, 0x2e, 0x36, 0x3e: + segment, rex = 0, 0 + case 0x66: + rex = 0 + case 0x67: + addressOverride, rex = true, 0 + case 0xf0: + locked, rex = true, 0 + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + if len(code) < i+3 || code[i] != 0x0f || code[i+1] != 0x38 || code[i+2] != 0xf8 { + return Instr{}, 0, false, nil + } + var op Op + for name, prefix := range x86EnqueueOps { + if prefix == mandatory { + op = Op(name) + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(reason string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s: %s", op, reason) + } + if locked { + return fail("LOCK is not permitted") + } + if len(code) <= i+3 { + return fail("missing ModRM byte") + } + if code[i+3]>>6 == 3 { + return fail("the command source must be memory") + } + var lea []byte + if segment != 0 { + lea = append(lea, segment) + } + if addressOverride { + lea = append(lea, 0x67) + } + if mode == 64 { + lea = append(lea, rex|0x48) + } + lea = append(lea, 0x8d) + leaHeader := len(lea) + lea = append(lea, code[i+3:]...) + decoded, err := x86asm.Decode(lea, mode) + if err != nil || decoded.Op != x86asm.LEA { + return fail("truncated or invalid memory address") + } + memory, ok := decoded.Args[1].(x86asm.Mem) + if !ok { + return fail("decoded source is not memory") + } + if memory.Base == x86asm.RIP || memory.Base == x86asm.EIP { + return fail("PC-relative command source requires source-layout context") + } + // x/arch prints segment overrides as FS:..., while the Plan 9 parser + // represents them as (...)(FS). Carry the segment structurally instead. + memory.Segment = 0 + decoded.Args[1] = memory + parsed, err := parseDecodedX86Instruction(x86asm.GoSyntax(decoded, 0, nil)) + if err != nil || len(parsed) != 1 || len(parsed[0].Args) != 2 { + return fail("cannot represent decoded memory address") + } + length := i + 3 + decoded.Len - leaHeader + if length > 15 { + return fail("encoding exceeds the 15-byte instruction limit") + } + ins := parsed[0] + if memory.Base == 0 && memory.Index == 0 { + ins.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Off: memory.Disp}} + } + if ins.Args[0].Kind != OpMem { + return fail("cannot represent decoded source as memory") + } + switch segment { + case 0x64: + ins.Args[0].Mem.Segment = FS + case 0x65: + ins.Args[0].Mem.Segment = GS + } + ins.Op = op + ins.x86AddressBits = decoded.AddrSize + ins.Raw = fmt.Sprintf("%s %s, %s", op, ins.Args[0], ins.Args[1]) + return ins, length, true, nil +} diff --git a/amd64_raw_enqueue_test.go b/amd64_raw_enqueue_test.go new file mode 100644 index 000000000..586efd7e7 --- /dev/null +++ b/amd64_raw_enqueue_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +// Intel's ENQCMD family uses F2/F3 0F 38 F8 /r, a 64-byte memory +// source, a GP destination address, and ZF to report retry. Neither +// spelling is present in Go 1.27's x86 opcode tables; ixl-go uses BYTE. +// https://www.intel.com/content/dam/develop/external/us/en/documents/architecture-instruction-set-extensions-programming-reference-737410.pdf +func TestX86RawEnqueueFamilyObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, family := range []struct { + name string + prefix byte + }{ + {"ENQCMD", 0xf2}, + {"ENQCMDS", 0xf3}, + } { + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(family.name+"/"+target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT enqueue(SB),$0-0 + MOVL $0, AX + MOVL $0, BX + BYTE $0x%02x; BYTE $0x0f; BYTE $0x38; BYTE $0xf8; BYTE $0x03 + SETEQ AX + RET +`, family.prefix) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + instruction := decoded.Instrs[len(decoded.Instrs)-3] + if instruction.Op != Op(family.name) || len(instruction.Args) != 2 || + instruction.Args[0].Kind != OpMem || instruction.Args[0].Mem.Base != BX || + instruction.Args[1].Kind != OpReg || instruction.Args[1].Reg != AX { + t.Fatalf("wrong command source/destination: %+v", instruction) + } + ir, err := Translate(file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"enqueue": {Name: "enqueue", Ret: I32}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, strings.ToLower(family.name)) { + t.Fatalf("lost device command %s", family.name) + } + compileLLVMToObject(t, llc, target.triple, "enqueue.ll", "enqueue.o", ir) + }) + } + } +} + +func TestX86RawEnqueueAddressGrammar(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, test := range []struct { + name string + mode int + code []byte + address string + destination Reg + bits int + }{ + {"base64", 64, []byte{0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 64}, + {"base32", 32, []byte{0xf3, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 32}, + {"sib_disp8", 64, []byte{0xf2, 0x44, 0x0f, 0x38, 0xf8, 0x54, 0x8b, 0x80}, "-128(BX)(CX*4)", "R10", 64}, + {"extended_base_index", 64, []byte{0xf2, 0x47, 0x0f, 0x38, 0xf8, 0x14, 0x8b}, "0(R11)(R9*4)", "R10", 64}, + {"disp32", 32, []byte{0xf2, 0x0f, 0x38, 0xf8, 0xb0, 0x78, 0x56, 0x34, 0x12}, "305419896(AX)", SI, 32}, + {"absolute32", 32, []byte{0xf2, 0x0f, 0x38, 0xf8, 0x05, 0x00, 0x10, 0x00, 0x00}, "4096()", AX, 32}, + {"override32", 64, []byte{0x67, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 32}, + {"override16", 32, []byte{0x67, 0xf2, 0x0f, 0x38, 0xf8, 0x00}, "0(BX)(SI*1)", AX, 16}, + {"fs", 64, []byte{0x64, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)(FS)", AX, 64}, + {"gs", 32, []byte{0x65, 0xf3, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)(GS)", AX, 32}, + {"rex_reset", 64, []byte{0x44, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, "0(BX)", AX, 64}, + } { + t.Run(test.name, func(t *testing.T) { + ins, length, ok, err := decodedX86EnqueueInstruction(test.code, test.mode) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode = %+v, %d, %v, %v", ins, length, ok, err) + } + if ins.Args[0].String() != test.address || ins.Args[1].Reg != test.destination || ins.x86AddressBits != test.bits { + t.Fatalf("decoded %+v address=%s, want %s, %s, %d bits", ins, ins.Args[0], test.address, test.destination, test.bits) + } + arch, triple := "amd64", "x86_64-unknown-linux-gnu" + if test.mode == 32 { + arch, triple = "386", "i386-unknown-linux-gnu" + } + file, err := Parse(ArchAMD64, "TEXT enqueue(SB),$0-0\nRET\n") + if err != nil { + t.Fatal(err) + } + instructions := file.Funcs[0].Instrs + file.Funcs[0].Instrs = append([]Instr{instructions[0], ins}, instructions[1:]...) + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"enqueue": {Name: "enqueue", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, flag := range []string{"cf", "of", "slt", "pf"} { + if strings.Count(ir, "store i1 false, ptr %flags_"+flag) < 2 { + t.Fatalf("missing cleared %s status flag", flag) + } + } + if !strings.Contains(ir, "elementtype([64 x i8])") || !strings.Contains(ir, "; sete $0") { + t.Fatal("lost command width or device retry status") + } + compileLLVMToObject(t, llc, triple, "address.ll", "address.o", ir) + if segment := ins.Args[0].Mem.Segment; segment != "" { + cmd := exec.Command(llc, "-O0", "-mtriple="+triple, "-filetype=asm", "-o", "-", "-") + cmd.Stdin = strings.NewReader(ir) + assembly, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("emit assembly: %v\n%s", err, assembly) + } + if !strings.Contains(string(assembly), "%"+strings.ToLower(string(segment))+":") { + t.Fatalf("source segment override was lost:\n%s", assembly) + } + } + }) + } +} + +func TestX86RawEnqueueRejectsInvalidEncodings(t *testing.T) { + for _, code := range [][]byte{ + {0xf2, 0x0f, 0x38, 0xf8}, + {0xf2, 0x0f, 0x38, 0xf8, 0xc3}, + {0xf3, 0x0f, 0x38, 0xf8, 0x04}, + {0xf2, 0x0f, 0x38, 0xf8, 0x43}, + {0xf2, 0x0f, 0x38, 0xf8, 0x83, 0x01}, + {0xf0, 0xf2, 0x0f, 0x38, 0xf8, 0x03}, + } { + for _, mode := range []int{32, 64} { + if _, _, ok, err := decodedX86EnqueueInstruction(code, mode); !ok || err == nil { + t.Fatalf("accepted malformed %x in mode %d: recognized=%v err=%v", code, mode, ok, err) + } + } + } + for _, code := range [][]byte{ + {0x0f, 0x38, 0xf8, 0x03}, + {0xf2, 0x0f, 0x38, 0xf9, 0x03}, + {0xb8, 0xf2, 0x0f, 0x38, 0xf8}, + } { + if _, _, ok, _ := decodedX86EnqueueInstruction(code, 64); ok { + t.Fatalf("claimed another instruction: %x", code) + } + } +} + +func TestX86EnqueueRejectsInvalidOperands(t *testing.T) { + for _, op := range []string{"ENQCMD", "ENQCMDS"} { + for _, tail := range []string{" AX, BX", " (AX), X0", " (AX), (BX)", " (AX)", ".Z (AX), BX"} { + file, err := Parse(ArchAMD64, "TEXT invalid(SB),$0-0\n"+op+tail+"\nRET\n") + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{ + "invalid": {Name: "invalid", Ret: Void}, + }}); err == nil { + t.Fatalf("accepted %s%s", op, tail) + } + } + } +} diff --git a/amd64_raw_evex_float_logical.go b/amd64_raw_evex_float_logical.go new file mode 100644 index 000000000..d702be118 --- /dev/null +++ b/amd64_raw_evex_float_logical.go @@ -0,0 +1,83 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawEVEXFloatLogicalInstruction covers Go 1.27's EVEX +// VAND/ANDN/OR/XOR PS/PD rows across X/Y/Z, masks, and scalar broadcast. +func decodedX86RawEVEXFloatLogicalInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86RawEVEXFloatLogicalInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86RawEVEXFloatLogicalInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 1 || p.opcode < 0x54 || p.opcode > 0x57 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX floating logical: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 || p.pp > 1 || p.w != (p.pp == 1) { + return fail("invalid prefix or vector length") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires K1-K7") + } + if mode == 32 && p.mask != 0 { + return fail("386 masked form exceeds Go's three-operand frontend") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.broadcast && code[p.modRM]>>6 == 3 { + return fail("broadcast requires a memory source") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + scale := 16 << p.vectorLength + if p.broadcast { + scale = 4 + if p.w { + scale = 8 + } + } + source, consumed, err := memory.evex( + code[p.modRM:], mode, p.b, p.x, p.segment, width, scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + stem := [...]string{"VAND", "VANDN", "VOR", "VXOR"}[p.opcode-0x54] + suffix := "PS" + if p.w { + suffix = "PD" + } + op := Op(stem + suffix) + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + destNumber := int(code[p.modRM]>>3&7) + p.r*8 + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destNumber))} + args := []Operand{source, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{ + Op: op, Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: true, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_evex_float_logical_test.go b/amd64_raw_evex_float_logical_test.go new file mode 100644 index 000000000..8ae798431 --- /dev/null +++ b/amd64_raw_evex_float_logical_test.go @@ -0,0 +1,202 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawEVEXFloatLogicalEncoding(opcode, pp byte, width, mask int, zeroing, broadcast, rip bool) []byte { + p1 := byte(0x7c | pp) // EVEX.vvvv names Z0, fixed bit set. + if pp == 1 { + p1 |= 0x80 // PD requires EVEX.W1. + } + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // Z3 first source, Z2 destination. + if broadcast { + modRM = 0x13 // 0(BX) first source. + } + if rip { + modRM = 0x15 // RIP+disp32 first source. + } + code := []byte{0x62, 0xf1, p1, p2, opcode, modRM} + if rip { + code = append(code, 1, 0, 0, 0) + } + return code +} + +func x86RawEVEXFloatLogicalLiteral(opcode, pp byte, width, mask int, zeroing, broadcast bool) []byte { + code := x86RawEVEXFloatLogicalEncoding(opcode, pp, width, mask, zeroing, broadcast, true) + code = append(code, 0xc3) + dataWidth := width + if broadcast { + dataWidth = 4 + if pp == 1 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*13+3)) + } + return code +} + +func TestDecodeX86RawEVEXFloatLogicalArithAVX512Regression(t *testing.T) { + // Go's assembler decodes these bytes as VANDPS 0(SI)(R8*4), Z0, Z1. + code := []byte{0x62, 0xb1, 0x7c, 0x48, 0x54, 0x0c, 0x86, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "arith AVX512 VANDPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VANDPS" || + decoded[0].Args[0].Kind != OpMem || + decoded[0].Args[0].Mem.Base != SI || + decoded[0].Args[0].Mem.Index != "R8" || + decoded[0].Args[0].Mem.Scale != 4 || + decoded[0].Args[1].Reg != "Z0" || + decoded[0].Args[2].Reg != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawEVEXFloatLogicalCompleteGo127Family(t *testing.T) { + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, width := range []int{16, 32, 64} { + for _, masking := range []struct { + mask int + zero bool + }{ + {}, {mask: 3}, {mask: 7, zero: true}, + } { + for _, broadcast := range []bool{false, true} { + code := x86RawEVEXFloatLogicalEncoding(opcode, pp, width, masking.mask, masking.zero, broadcast, false) + got, length, ok, err := decodedX86RawEVEXFloatLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + want := Op(stem + suffix) + if broadcast { + want += ".BCST" + } + if masking.zero { + want += ".Z" + } + if got.Op != want || len(got.Args) != 3+map[bool]int{false: 0, true: 1}[masking.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, want) + } + if broadcast && got.Args[0].Kind != OpMem || !broadcast && got.Args[0].Reg == "" { + t.Fatalf("decode %x has wrong first source: %+v", code, got) + } + } + } + } + } + } + high := []byte{0x62, 0x01, 0x04, 0x40, 0x54, 0xff} + got, length, ok, err := decodedX86RawEVEXFloatLogicalInstruction(high, 64) + if err != nil || !ok || length != len(high) || got.Raw != "VANDPS Z31, Z31, Z31" { + t.Fatalf("high EVEX registers %x decoded as %+v, length=%d, ok=%v, err=%v", high, got, length, ok, err) + } +} + +func TestDecodeX86RawEVEXFloatLogicalRIPDataCompleteFamily(t *testing.T) { + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + name := fmt.Sprintf("%s%s/%d/bcst%t", stem, suffix, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawEVEXFloatLogicalLiteral(opcode, pp, width, 7, true, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + dataWidth := width + if broadcast { + dataWidth = 4 + if pp == 1 { + dataWidth = 8 + } + } + if len(decoded) != 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-dataWidth:]) { + t.Fatalf("decoded floating logical literal %x as %#v", code, decoded) + } + }) + } + } + } + } +} + +func TestDecodeX86RawEVEXFloatLogicalRejectsUnsafeForms(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated literal": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[6] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[6] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawEVEXFloatLogicalLiteral(0x54, 0, 64, 7, true, true)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe floating logical literal %x", code) + } + }) + } + for _, code := range [][]byte{ + x86RawEVEXFloatLogicalEncoding(0x54, 0, 16, 0, true, false, false), + {0x62, 0xf1, 0x7c, 0x19, 0x54, 0xd3}, // EVEX.b with a register source. + } { + if _, _, ok, err := decodedX86RawEVEXFloatLogicalInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved floating logical form %x: ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawEVEXFloatLogicalRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + name := "literal" + stem + suffix + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawEVEXFloatLogicalLiteral(opcode, pp, 64, 7, true, true) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-evex-float-logical-literal.ll", "raw-evex-float-logical-literal.o", ir) + }) + } +} diff --git a/amd64_raw_evex_insert_test.go b/amd64_raw_evex_insert_test.go new file mode 100644 index 000000000..8f9b80ced --- /dev/null +++ b/amd64_raw_evex_insert_test.go @@ -0,0 +1,216 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedInsert(op Op, vectorBits byte, mask int, zeroing bool, first, second, destination int, immediate byte) []byte { + opcode := map[Op]byte{ + "VINSERTF32X4": 0x18, + "VINSERTF64X2": 0x18, + "VINSERTF32X8": 0x1a, + "VINSERTF64X4": 0x1a, + "VINSERTI32X4": 0x38, + "VINSERTI64X2": 0x38, + "VINSERTI32X8": 0x3a, + "VINSERTI64X4": 0x3a, + }[op] + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := byte(^second&15)<<3 | 5 + if strings.Contains(string(op), "64") { + p1 |= 0x80 + } + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0x62, p0, p1, p2, opcode, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedHasteEVEXPackedInsert(t *testing.T) { + code := []byte{0x62, 0xf3, 0xfd, 0x48, 0x38, 0x47, 0x01, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "haste VINSERTI64X2 sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VINSERTI64X2 $1, 16(DI), Z0, Z0" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawEVEXPackedInsertCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, family := range []struct { + ops []Op + vectorBits []byte + }{ + { + ops: []Op{"VINSERTF32X4", "VINSERTF64X2", "VINSERTI32X4", "VINSERTI64X2"}, + vectorBits: []byte{1, 2}, + }, + { + ops: []Op{"VINSERTF32X8", "VINSERTF64X4", "VINSERTI32X8", "VINSERTI64X4"}, + vectorBits: []byte{2}, + }, + } { + for _, op := range family.ops { + for _, vectorBits := range family.vectorBits { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + firstName := "X" + if strings.HasSuffix(string(op), "X8") || strings.HasSuffix(string(op), "X4") && strings.Contains(string(op), "64X4") { + firstName = "Y" + } + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(first*8 + destination) + code := encodeX86RawEVEXPackedInsert(op, vectorBits, masking.mask, masking.zeroing, first, second, destination, immediate) + got, length, ok, err := decodedX86EVEXPackedInsertInstruction(code, 64) + args := []string{ + fmt.Sprintf("$%d", immediate), + fmt.Sprintf("%s%d", firstName, first), + fmt.Sprintf("%s%d", vectorName, second), + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(op) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + } + if count != 1179648 { + t.Fatalf("covered %d EVEX packed-insert register encodings, want 1179648", count) + } +} + +func TestDecodedX86RawEVEXPackedInsertMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x62, 0x03, 0x15, 0xc3, 0x3a, 0x64, 0x8b, 0x02, 0xff} + got, length, ok, err := decodedX86EVEXPackedInsertInstruction(code, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 64} + if err != nil || !ok || length != len(code) || got.Op != "VINSERTI32X8.Z" || len(got.Args) != 5 || got.Args[0].Imm != 255 || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) || got.Args[2].String() != "Z29" || got.Args[3].String() != "K3" || got.Args[4].String() != "Z28" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86RawEVEXPackedInsert("VINSERTI64X2", 2, 0, false, 0, 1, 2, 3) + for name, mutate := range map[string]func([]byte){ + "missing EVEX fixed bit": func(code []byte) { code[2] &^= 0x04 }, + "EVEX.b": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "zero without mask": func(code []byte) { code[3] |= 0x80 }, + "x4 with X output": func(code []byte) { code[3] &^= 0x60 }, + "x8 with Y output": func(code []byte) { + code[3] = code[3]&^0x60 | 0x20 + code[4] = 0x3a + }, + } { + invalid := append([]byte(nil), valid...) + mutate(invalid) + if instruction, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {0x62}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5], valid[6]}, + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5], valid[6]}, + {valid[0], valid[1], valid[2], valid[3], 0x19, valid[5], valid[6]}, + } { + if instruction, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("unrelated encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + truncatedImmediate := valid[:len(valid)-1] + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(truncatedImmediate, 64); !matched || decodeErr == nil { + t.Fatalf("truncated immediate returned ok=%v err=%v", matched, decodeErr) + } + ripRelative := []byte{0x62, 0xf3, 0xfd, 0x48, 0x38, 0x05, 0, 0, 0, 0, 1} + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + addressOverride := append([]byte{0x67}, valid...) + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(addressOverride, 64); !matched || decodeErr == nil { + t.Fatalf("address-override encoding returned ok=%v err=%v", matched, decodeErr) + } + for _, op := range []Op{"VINSERTF32X4", "VINSERTF64X2", "VINSERTF32X8", "VINSERTF64X4", "VINSERTI32X4", "VINSERTI64X2", "VINSERTI32X8", "VINSERTI64X4"} { + vectorBits := byte(1) + if strings.HasSuffix(string(op), "X8") || strings.Contains(string(op), "64X4") { + vectorBits = 2 + } + invalid := encodeX86RawEVEXPackedInsert(op, vectorBits, 0, false, 0, 0, 1, 0) + if _, _, matched, decodeErr := decodedX86EVEXPackedInsertInstruction(invalid, 32); !matched || decodeErr == nil { + t.Fatalf("386 %s encoding %x returned ok=%v err=%v", op, invalid, matched, decodeErr) + } + } +} + +func TestTranslateX86RawEVEXPackedInsertTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + code := [][]byte{ + encodeX86RawEVEXPackedInsert("VINSERTI64X2", 1, 3, true, 20, 20, 21, 1), + } + code = append(code, + []byte{0x62, 0xf3, 0xfd, 0x48, 0x38, 0x47, 0x01, 0x01}, + []byte{0x62, 0x03, 0x15, 0xc3, 0x3a, 0x64, 0x8b, 0x02, 0xff}, + ) + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_insert(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_evex_packed_insert": {Name: "raw_evex_packed_insert", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-packed-insert-"+target.name+".ll", "raw-evex-packed-insert-"+target.name+".o", ir) + }) + } +} diff --git a/amd64_raw_evex_madd.go b/amd64_raw_evex_madd.go new file mode 100644 index 000000000..0bec29b3b --- /dev/null +++ b/amd64_raw_evex_madd.go @@ -0,0 +1,34 @@ +package plan9asm + +import ( + "fmt" +) + +// decodedX86EVEXPackedMADDInstruction covers the EVEX rows of Go's +// _yvandnpd optab shared by VPMADDUBSW and VPMADDWD. Neither row permits +// EVEX.b broadcasting, but both permit K masks and zeroing at X/Y/Z widths. +func decodedX86EVEXPackedMADDInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.pp != 1 { + return Instr{}, 0, false, nil + } + op := map[[2]int]Op{ + {1, 0xf5}: "VPMADDWD", + {2, 0x04}: "VPMADDUBSW", + }[[2]int{p.mapNumber, p.opcode}] + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX packed multiply-add: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.broadcast || p.addressOverride || p.vectorLength > 2 { + return fail("invalid EVEX fixed, width, broadcast, address, or vector-length bits") + } + // Both opcode-specific ISA rows are WIG. Full validated physical bytes, + // including mode32 masks, use the common private decoder producer. + return decodedX86BinaryVectorOperands(code, p, mode, op, 16) +} diff --git a/amd64_raw_evex_madd_test.go b/amd64_raw_evex_madd_test.go new file mode 100644 index 000000000..d183a6462 --- /dev/null +++ b/amd64_raw_evex_madd_test.go @@ -0,0 +1,175 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedMADD(op Op, width, mask, destination, first, second int, zeroing, memory bool) []byte { + opcode, opcodeMap := byte(0xf5), byte(1) + if op == "VPMADDUBSW" { + opcode, opcodeMap = 0x04, 2 + } + p0 := byte((1-(destination>>3)&1)<<7 | (1-(first>>4)&1)<<6 | + (1-(first>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | int(opcodeMap)) + p1 := byte(((^second)&15)<<3 | 0x05) + p2 := byte(width<<5 | (1-((second>>4)&1))<<3 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | first&7) + if memory { + modRM = byte(0x40 | (destination&7)<<3) + p0 = byte((1-(destination>>3)&1)<<7 | 0x60 | (1-(destination>>4)&1)<<4 | int(opcodeMap)) + } + code := []byte{0x62, p0, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawEVEXMADDSeBiShogunRegression(t *testing.T) { + // Go's assembler disassembles these bytes as VPMADDUBSW Z1, Z5, Z1. + code := []byte{0x62, 0xf2, 0x55, 0x48, 0x04, 0xc9, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "mulInt8 AVX512 VPMADDUBSW", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPMADDUBSW" || + decoded[0].Args[0].Reg != "Z1" || decoded[0].Args[1].Reg != "Z5" || + decoded[0].Args[2].Reg != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXPackedMADDCompleteGoForms(t *testing.T) { + for _, op := range []Op{"VPMADDWD", "VPMADDUBSW"} { + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + memory bool + destination int + first int + second int + }{ + {name: "register", destination: 2, first: 3, second: 4}, + {name: "high registers", destination: 18, first: 19, second: 20}, + {name: "masked", mask: 3, destination: 2, first: 3, second: 4}, + {name: "zeroing", mask: 7, zeroing: true, destination: 2, first: 3, second: 4}, + {name: "memory", memory: true, destination: 2, second: 4}, + {name: "zeroing memory", memory: true, mask: 5, zeroing: true, destination: 2, second: 4}, + } { + name := fmt.Sprintf("%s/%s/%s", op, vector, form.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawEVEXPackedMADD( + op, width, form.mask, form.destination, form.first, form.second, form.zeroing, form.memory, + ) + got, length, ok, err := decodedX86EVEXPackedMADDInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := op + if form.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != fmt.Sprintf("%s%d", vector, form.second) || + got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s%d", vector, form.destination) { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + if !form.memory && got.Args[0].String() != fmt.Sprintf("%s%d", vector, form.first) { + t.Fatalf("decode %x first source = %s", code, got.Args[0]) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXPackedMADDRIPLiteral(t *testing.T) { + for _, op := range []Op{"VPMADDWD", "VPMADDUBSW"} { + for width := 0; width < 3; width++ { + code := encodeX86RawEVEXPackedMADD(op, width, 1, 2, 0, 4, false, false) + code[5] = 0x15 // RIP-relative first source, destination 2. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d", op, width) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != op { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawEVEXPackedMADDRejectsInvalidForms(t *testing.T) { + base := encodeX86RawEVEXPackedMADD("VPMADDUBSW", 2, 0, 2, 3, 4, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x10 }, // unsupported broadcast + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86EVEXPackedMADDInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawEVEXPackedMADDLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawEVEXPackedMADD(SB),$0-0\n") + for _, op := range []Op{"VPMADDWD", "VPMADDUBSW"} { + for width := 0; width < 3; width++ { + for _, memory := range []bool{false, true} { + code := encodeX86RawEVEXPackedMADD(op, width, 0, 2, 3, 4, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawEVEXPackedMADD": {Name: "rawEVEXPackedMADD", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-madd.ll", "raw-evex-madd.o", ir) + }) + } +} diff --git a/amd64_raw_evex_packed_broadcast_test.go b/amd64_raw_evex_packed_broadcast_test.go new file mode 100644 index 000000000..4d7d857cf --- /dev/null +++ b/amd64_raw_evex_packed_broadcast_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedBroadcast(op Op, vectorBits byte, mask int, zeroing, vectorSource bool, source, destination int) []byte { + opcode := byte(0) + width64 := false + if vectorSource { + switch op { + case "VPBROADCASTB": + opcode = 0x78 + case "VPBROADCASTW": + opcode = 0x79 + case "VPBROADCASTD": + opcode = 0x58 + case "VPBROADCASTQ": + opcode = 0x59 + width64 = true + } + } else { + switch op { + case "VPBROADCASTB": + opcode = 0x7a + case "VPBROADCASTW": + opcode = 0x7b + case "VPBROADCASTD": + opcode = 0x7c + case "VPBROADCASTQ": + opcode = 0x7c + width64 = true + } + } + p0 := byte((^destination>>3)&1)<<7 | + byte((^source>>4)&1)<<6 | + byte((^source>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 2 + p1 := byte(0x7d) + if width64 { + p1 |= 0x80 + } + p2 := vectorBits<<5 | 0x08 | byte(mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | destination&7<<3 | source&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestDecodeX86RawDirectiveGroupReportedHasteEVEXBroadcast(t *testing.T) { + code := []byte{0x62, 0x52, 0xfd, 0x48, 0x7c, 0xf2} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "haste VPBROADCASTQ sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPBROADCASTQ R10, Z14" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawEVEXPackedBroadcastCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, op := range []Op{"VPBROADCASTB", "VPBROADCASTW", "VPBROADCASTD", "VPBROADCASTQ"} { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for _, sourceKind := range []struct { + vector bool + count int + prefix string + }{ + {count: 16, prefix: "R"}, + {vector: true, count: 32, prefix: "X"}, + } { + for source := 0; source < sourceKind.count; source++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXPackedBroadcast(op, byte(vectorBits), masking.mask, masking.zeroing, sourceKind.vector, source, destination) + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", sourceKind.prefix, source)} + if !sourceKind.vector { + args[0] = decodedX86RegisterNameForTest(source) + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(op) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 55296 { + t.Fatalf("covered %d EVEX packed-broadcast register encodings, want 55296", count) + } +} + +func TestDecodedX86RawEVEXPackedBroadcastMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x65, 0x62, 0x12, 0x7d, 0x2b, 0x58, 0x64, 0x8b, 0x20} + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + wantMemory := MemRef{Segment: GS, Base: "R11", Index: "R9", Scale: 4, Off: 128} + if err != nil || !ok || length != len(code) || got.Op != "VPBROADCASTD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) || got.Args[1].String() != "K3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 0, 0) + for name, mutate := range map[string]func([]byte){ + "W on byte": func(code []byte) { code[2] |= 0x80; code[4] = 0x7a }, + "used vvvv": func(code []byte) { code[2] &^= 0x08 }, + "used high vvvv": func(code []byte) { code[3] &^= 0x08 }, + "EVEX.b": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "zero without mask": func(code []byte) { code[3] |= 0x80 }, + "GP high extension": func(code []byte) { code[1] &^= 0x40 }, + "GP opcode on memory": func(code []byte) { code[5] &^= 0xc0 }, + } { + invalid := append([]byte(nil), valid...) + mutate(invalid) + if instruction, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {0x62}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2], valid[3], 0x77, valid[5]}, + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("unrelated encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + + ripRelative := []byte{0x62, 0xf2, 0x7d, 0x08, 0x58, 0x05, 0, 0, 0, 0} + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + addressOverride := append([]byte{0x67}, valid...) + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(addressOverride, 64); !matched || decodeErr == nil { + t.Fatalf("address-override encoding returned ok=%v err=%v", matched, decodeErr) + } + for name, invalid := range map[string][]byte{ + "extended destination on 386": encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 0, 8), + "extended source on 386": encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, true, 8, 0), + } { + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 32); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, invalid, matched, decodeErr) + } + } + valid386 := encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 7, 7) + got, length, ok, err = decodedX86VEXPackedBroadcastInstruction(valid386, 32) + if err != nil || !ok || length != len(valid386) || got.Raw != "VPBROADCASTD DI, X7" { + t.Fatalf("386 base-register form decoded as %+v, length=%d ok=%v err=%v", got, length, ok, err) + } +} + +func TestTranslateX86RawEVEXPackedBroadcastTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + code := [][]byte{ + encodeX86RawEVEXPackedBroadcast("VPBROADCASTD", 0, 0, false, false, 0, 1), + encodeX86RawEVEXPackedBroadcast("VPBROADCASTB", 1, 1, true, true, 1, 2), + } + if target.goarch == "amd64" { + code = append(code, []byte{0x62, 0x52, 0xfd, 0x48, 0x7c, 0xf2}) + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_broadcast(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_evex_packed_broadcast": {Name: "raw_evex_packed_broadcast", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-packed-broadcast-"+target.name+".ll", "raw-evex-packed-broadcast-"+target.name+".o", ir) + }) + } +} diff --git a/amd64_raw_evex_packed_float_move_test.go b/amd64_raw_evex_packed_float_move_test.go new file mode 100644 index 000000000..4cfa2ebc8 --- /dev/null +++ b/amd64_raw_evex_packed_float_move_test.go @@ -0,0 +1,80 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawEVEXPackedFloatMoveCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvmovapd table gives VMOVAPS/APD/UPS/UPD the same + // bidirectional X/Y/Z and K-masked EVEX forms in addition to the four + // legacy VEX rows. These LLVM-verified encodings cover every EVEX opcode, + // direction, vector width, mask mode, high-register field, and compressed + // disp8 addressing shape used by that table. + tests := []struct { + name string + code []byte + want string + }{ + {name: "aps x load masked", code: []byte{0x62, 0xf1, 0x7c, 0x09, 0x28, 0xc1}, want: "VMOVAPS X1, K1, X0"}, + {name: "aps x store compressed disp8", code: []byte{0x62, 0xf1, 0x7c, 0x0a, 0x29, 0x50, 0x01}, want: "VMOVAPS X2, K2, 16(AX)"}, + {name: "apd y high load masked", code: []byte{0x62, 0xa1, 0xfd, 0x29, 0x28, 0xe5}, want: "VMOVAPD Y21, K1, Y20"}, + {name: "apd y store masked sib", code: []byte{0x62, 0x91, 0xfd, 0x2a, 0x29, 0x64, 0x88, 0x03}, want: "VMOVAPD Y4, K2, 96(R8)(R9*4)"}, + {name: "apd z high load zeroing", code: []byte{0x62, 0xa1, 0xfd, 0xcb, 0x28, 0xe5}, want: "VMOVAPD.Z Z21, K3, Z20"}, + {name: "ups z high load zeroing", code: []byte{0x62, 0x01, 0x7c, 0xcf, 0x10, 0xfe}, want: "VMOVUPS.Z Z30, K7, Z31"}, + {name: "ups z store compressed disp8", code: []byte{0x62, 0xf1, 0x7c, 0x4e, 0x11, 0x7f, 0x07}, want: "VMOVUPS Z7, K6, 448(DI)"}, + {name: "upd x load masked", code: []byte{0x62, 0xf1, 0xfd, 0x09, 0x10, 0xd3}, want: "VMOVUPD X3, K1, X2"}, + {name: "upd x store compressed disp8", code: []byte{0x62, 0xf1, 0xfd, 0x0a, 0x11, 0x60, 0x02}, want: "VMOVUPD X4, K2, 32(AX)"}, + {name: "ups y memory load zeroing sib", code: []byte{0x62, 0x91, 0x7c, 0xaa, 0x10, 0x64, 0x88, 0x03}, want: "VMOVUPS.Z 96(R8)(R9*4), K2, Y4"}, + {name: "ups y high store masked", code: []byte{0x62, 0x41, 0x7c, 0x2b, 0x11, 0x4c, 0x24, 0x02}, want: "VMOVUPS Y25, K3, 64(R12)"}, + {name: "upd z high store masked sib", code: []byte{0x62, 0x01, 0xfd, 0x4b, 0x11, 0x4c, 0x6c, 0x02}, want: "VMOVUPD Z25, K3, 128(R12)(R13*2)"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := decodeX86RawDirectives(rawX86Function(test.code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, test.want+" ") { + t.Fatalf("decoded %#x as %#v, want %q", test.code, got.Instrs, test.want) + } + }) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_float_move(SB),NOSPLIT,$0-0\n") + for _, test := range tests { + for _, value := range test.code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"raw_evex_packed_float_move": {Name: "raw_evex_packed_float_move", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-evex-packed-float-move.ll", "raw-evex-packed-float-move.o", ir) +} + +func TestX86RawEVEXPackedFloatMoveReportedMinioInstruction(t *testing.T) { + code := []byte{0x62, 0xc1, 0x7c, 0x48, 0x10, 0x04, 0x09} + got, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, "VMOVUPS 0(R9)(CX*1), Z16 ") { + t.Fatalf("decoded reported minio bytes as %#v", got.Instrs) + } +} diff --git a/amd64_raw_evex_packed_integer_move_test.go b/amd64_raw_evex_packed_integer_move_test.go new file mode 100644 index 000000000..06683c97c --- /dev/null +++ b/amd64_raw_evex_packed_integer_move_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawEVEXPackedIntegerMoveCompleteGo127Family(t *testing.T) { + // Go 1.27 gives VMOVDQA32/64 and VMOVDQU8/16/32/64 the same + // 12-row _yvmovdqa32 table: X/Y/Z, load/store, with and without a + // nonzero K mask. These LLVM-verified encodings exercise every opcode, + // both directions, every vector width, high registers, SIB addressing, + // zeroing, and EVEX compressed disp8 scaling. + tests := []struct { + name string + code []byte + want string + }{ + {name: "dqa32 x load", code: []byte{0x62, 0xf1, 0x7d, 0x08, 0x6f, 0xc1}, want: "VMOVDQA32 X1, X0"}, + {name: "dqa32 x store compressed disp8", code: []byte{0x62, 0xf1, 0x7d, 0x08, 0x7f, 0x50, 0x01}, want: "VMOVDQA32 X2, 16(AX)"}, + {name: "dqa64 y high load masked", code: []byte{0x62, 0xa1, 0xfd, 0x29, 0x6f, 0xe5}, want: "VMOVDQA64 Y21, K1, Y20"}, + {name: "dqa64 y store masked sib", code: []byte{0x62, 0x91, 0xfd, 0x2a, 0x7f, 0x64, 0x88, 0x03}, want: "VMOVDQA64 Y4, K2, 96(R8)(R9*4)"}, + {name: "dqu8 z high load zeroing", code: []byte{0x62, 0x01, 0x7f, 0xcf, 0x6f, 0xfe}, want: "VMOVDQU8.Z Z30, K7, Z31"}, + {name: "dqu8 z store compressed disp8", code: []byte{0x62, 0xf1, 0x7f, 0x4e, 0x7f, 0x7f, 0x07}, want: "VMOVDQU8 Z7, K6, 448(DI)"}, + {name: "dqu16 x load masked", code: []byte{0x62, 0xf1, 0xff, 0x09, 0x6f, 0xd3}, want: "VMOVDQU16 X3, K1, X2"}, + {name: "dqu16 x store compressed disp8", code: []byte{0x62, 0xf1, 0xff, 0x0a, 0x7f, 0x60, 0x02}, want: "VMOVDQU16 X4, K2, 32(AX)"}, + {name: "dqu32 y memory load zeroing sib", code: []byte{0x62, 0x91, 0x7e, 0xaa, 0x6f, 0x64, 0x88, 0x03}, want: "VMOVDQU32.Z 96(R8)(R9*4), K2, Y4"}, + {name: "dqu32 y high store masked", code: []byte{0x62, 0x41, 0x7e, 0x2b, 0x7f, 0x4c, 0x24, 0x02}, want: "VMOVDQU32 Y25, K3, 64(R12)"}, + {name: "dqu64 z high load masked", code: []byte{0x62, 0x01, 0xfe, 0x4f, 0x6f, 0xf7}, want: "VMOVDQU64 Z31, K7, Z30"}, + {name: "dqu64 z high store masked sib", code: []byte{0x62, 0x01, 0xfe, 0x4b, 0x7f, 0x4c, 0x6c, 0x02}, want: "VMOVDQU64 Z25, K3, 128(R12)(R13*2)"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := decodeX86RawDirectives(rawX86Function(test.code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, test.want+" ") { + t.Fatalf("decoded %#x as %#v, want %q", test.code, got.Instrs, test.want) + } + }) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_integer_move(SB),NOSPLIT,$0-0\n") + for _, test := range tests { + for _, value := range test.code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"raw_evex_packed_integer_move": {Name: "raw_evex_packed_integer_move", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-evex-packed-integer-move.ll", "raw-evex-packed-integer-move.o", ir) +} + +func TestX86RawEVEXPackedIntegerMoveReportedMinioSequence(t *testing.T) { + var code []byte + for register := byte(0); register < 8; register++ { + modRM := byte(0x07 | register<<3) + code = append(code, 0x62, 0xf1, 0x7e, 0x48, 0x6f, modRM) + if register != 0 { + code[len(code)-1] |= 0x40 + code = append(code, register) + } + } + got, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 8 { + t.Fatalf("decoded minio sequence as %d instructions: %#v", len(got.Instrs), got.Instrs) + } + for index, instruction := range got.Instrs { + want := fmt.Sprintf("VMOVDQU32 %d(DI), Z%d", index*64, index) + if !strings.HasPrefix(instruction.Raw, want+" ") { + t.Fatalf("instruction %d = %q, want %q", index, instruction.Raw, want) + } + } +} + +func TestX86RawEVEXPackedIntegerMoveZeroingRegisterStoreEncoding(t *testing.T) { + for _, spec := range []struct { + op Op + pp int + w byte + }{{"VMOVDQA32", 1, 0}, {"VMOVDQA64", 1, 0x80}, {"VMOVDQU8", 3, 0}, {"VMOVDQU16", 3, 0x80}, {"VMOVDQU32", 2, 0}, {"VMOVDQU64", 2, 0x80}} { + for width, prefix := range []string{"X", "Y", "Z"} { + for src := 0; src < 32; src++ { + for dst := 0; dst < 32; dst++ { + code := encodeRawScalarMove(true, spec.pp, 0x11, src, 0, dst, 7, true) + code[2] = code[2]&0x7f | spec.w + code[3] |= byte(width) << 5 + code[4] = 0x7f + got, size, matched, err := decodedX86EVEXPackedIntegerMoveInstruction(code, 64) + want := fmt.Sprintf("%s.Z %s%d, K7, %s%d", spec.op, prefix, src, prefix, dst) + if err != nil || !matched || size != len(code) || got.Raw != want { + t.Fatalf("decode %x=%q size=%d matched=%v err=%v, want %q", code, got.Raw, size, matched, err, want) + } + } + } + } + } +} + +func TestX86RawEVEXPackedIntegerMoveRejectsInvalidEVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + }{ + {name: "broadcast bit", code: []byte{0x62, 0xf1, 0x7e, 0x58, 0x6f, 0xc1}}, + {name: "zeroing without mask", code: []byte{0x62, 0xf1, 0x7e, 0xc8, 0x6f, 0xc1}}, + {name: "zeroing store", code: []byte{0x62, 0xf1, 0x7e, 0x89, 0x7f, 0x08}}, + {name: "reserved vector length", code: []byte{0x62, 0xf1, 0x7e, 0x68, 0x6f, 0xc1}}, + {name: "address override", code: []byte{0x67, 0x62, 0xf1, 0x7e, 0x48, 0x6f, 0x07}}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, ok, err := decodedX86EVEXPackedIntegerMoveInstruction(test.code, 64); !ok || err == nil { + t.Fatalf("decoded invalid encoding %#x with ok=%v err=%v", test.code, ok, err) + } + }) + } +} + +func rawX86Function(code []byte) Func { + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + return fn +} diff --git a/amd64_raw_evex_packed_logical_test.go b/amd64_raw_evex_packed_logical_test.go new file mode 100644 index 000000000..edc2252c5 --- /dev/null +++ b/amd64_raw_evex_packed_logical_test.go @@ -0,0 +1,242 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawEVEXPackedLogical(op Op, vectorBits byte, mask int, zeroing, broadcast bool, first, second, destination int) []byte { + opcode := map[Op]byte{ + "VPANDD": 0xdb, + "VPANDQ": 0xdb, + "VPANDND": 0xdf, + "VPANDNQ": 0xdf, + "VPORD": 0xeb, + "VPORQ": 0xeb, + "VPXORD": 0xef, + "VPXORQ": 0xef, + }[op] + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + p1 := byte(^second&15)<<3 | 5 + if strings.HasSuffix(string(op), "Q") { + p1 |= 0x80 + } + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestDecodeX86RawDirectiveGroupReportedHasteEVEXPackedLogical(t *testing.T) { + code := []byte{0x62, 0xf1, 0xf5, 0x48, 0xef, 0xc9} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "haste VPXORQ sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPXORQ Z1, Z1, Z1" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawEVEXPackedLogicalCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, op := range []Op{"VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", "VPORD", "VPORQ", "VPXORD", "VPXORQ"} { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXPackedLogical(op, byte(vectorBits), masking.mask, masking.zeroing, false, first, second, destination) + got, length, ok, err := decodedX86EVEXPackedLogicalInstruction(code, 64) + args := []string{ + fmt.Sprintf("%s%d", vectorName, first), + fmt.Sprintf("%s%d", vectorName, second), + } + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(op) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 2359296 { + t.Fatalf("covered %d EVEX packed-logical register encodings, want 2359296", count) + } +} + +func TestDecodedX86RawEVEXPackedLogicalMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x62, 0x01, 0x95, 0xd3, 0xdb, 0x64, 0x8b, 0x20} + got, length, ok, err := decodedX86EVEXPackedLogicalInstruction(code, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 256} + if err != nil || !ok || length != len(code) || got.Op != "VPANDQ.BCST.Z" || len(got.Args) != 4 || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) || got.Args[1].String() != "Z29" || got.Args[2].String() != "K3" || got.Args[3].String() != "Z28" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86RawEVEXPackedLogical("VPXORD", 0, 0, false, false, 0, 1, 2) + for name, mutate := range map[string]func([]byte){ + "missing EVEX fixed bit": func(code []byte) { code[2] &^= 0x04 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "zero without mask": func(code []byte) { code[3] |= 0x80 }, + "broadcast register": func(code []byte) { code[3] |= 0x10 }, + } { + invalid := append([]byte(nil), valid...) + mutate(invalid) + if instruction, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {0x62}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2], valid[3], 0xee, valid[5]}, + } { + if instruction, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("unrelated encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + ripRelative := []byte{0x62, 0xf1, 0x75, 0x08, 0xef, 0x15, 0, 0, 0, 0} + if _, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + addressOverride := append([]byte{0x67}, valid...) + if _, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(addressOverride, 64); !matched || decodeErr == nil { + t.Fatalf("address-override encoding returned ok=%v err=%v", matched, decodeErr) + } + for name, invalid := range map[string][]byte{ + "masked on 386": encodeX86RawEVEXPackedLogical("VPXORD", 0, 1, false, false, 0, 1, 2), + "high Z first on 386": encodeX86RawEVEXPackedLogical("VPXORD", 2, 0, false, false, 8, 1, 2), + "high Z second on 386": encodeX86RawEVEXPackedLogical("VPXORD", 2, 0, false, false, 0, 8, 2), + "high Z destination 386": encodeX86RawEVEXPackedLogical("VPXORD", 2, 0, false, false, 0, 1, 8), + } { + if _, _, matched, decodeErr := decodedX86EVEXPackedLogicalInstruction(invalid, 32); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, invalid, matched, decodeErr) + } + } + valid386 := encodeX86RawEVEXPackedLogical("VPXORD", 1, 0, false, false, 29, 30, 31) + got, length, ok, err = decodedX86EVEXPackedLogicalInstruction(valid386, 32) + if err != nil || !ok || length != len(valid386) || got.Raw != "VPXORD Y29, Y30, Y31" { + t.Fatalf("386 high X/Y register form decoded as %+v, length=%d ok=%v err=%v", got, length, ok, err) + } +} + +func TestTranslateX86RawEVEXPackedLogicalTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + first, second, destination := 29, 30, 31 + if target.goarch == "386" { + // Only the physical low bank belongs to this object fixture. + // The original high-bank bytes remain a separate Context case. + first, second, destination = 5, 6, 7 + } + code := [][]byte{ + encodeX86RawEVEXPackedLogical("VPXORD", 1, 0, false, false, first, second, destination), + } + if target.goarch == "amd64" { + code = append(code, + []byte{0x62, 0xf1, 0xf5, 0x48, 0xef, 0xc9}, + []byte{0x62, 0x01, 0x95, 0xd3, 0xdb, 0x64, 0x8b, 0x20}, + ) + } + var source strings.Builder + source.WriteString("TEXT raw_evex_packed_logical(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_evex_packed_logical": {Name: "raw_evex_packed_logical", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-packed-logical-"+target.name+".ll", "raw-evex-packed-logical-"+target.name+".o", ir) + }) + } +} + +func TestX86RawEVEXPackedLogicalMode32HighRegistersNeedContext(t *testing.T) { + checked := 0 + for _, op := range []Op{"VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", "VPORD", "VPORQ", "VPXORD", "VPXORQ"} { + for _, vectorBits := range []byte{0, 1} { + for _, position := range []int{0, 1, 2} { + for _, high := range []int{8, 15, 16, 31} { + registers := [3]int{0, 1, 2} + registers[position] = high + code := encodeX86RawEVEXPackedLogical(op, vectorBits, 0, false, false, registers[0], registers[1], registers[2]) + var source strings.Builder + source.WriteString("TEXT raw_mode32_high(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"raw_mode32_high": {Name: "raw_mode32_high", Ret: Void}}, + }) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("raw mode32 high-bank bytes %x acquired an unproved physical binding: %v", code, err) + } + checked++ + } + } + } + } + if checked != 192 { + t.Fatalf("checked %d raw mode32 high-bank cases, want 192", checked) + } +} diff --git a/amd64_raw_evex_unpack.go b/amd64_raw_evex_unpack.go new file mode 100644 index 000000000..d3444054f --- /dev/null +++ b/amd64_raw_evex_unpack.go @@ -0,0 +1,94 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawEVEXUnpackForm struct { + op Op + w bool + broadcastBytes int +} + +// Go 1.27's eight EVEX integer interleave rows. Dword and QDQ sources may +// use scalar-memory broadcast; byte and word rows may not. +var x86RawEVEXUnpackForms = map[int]x86RawEVEXUnpackForm{ + 0x60: {op: "VPUNPCKLBW"}, + 0x61: {op: "VPUNPCKLWD"}, + 0x62: {op: "VPUNPCKLDQ", broadcastBytes: 4}, + 0x6c: {op: "VPUNPCKLQDQ", w: true, broadcastBytes: 8}, + 0x68: {op: "VPUNPCKHBW"}, + 0x69: {op: "VPUNPCKHWD"}, + 0x6a: {op: "VPUNPCKHDQ", broadcastBytes: 4}, + 0x6d: {op: "VPUNPCKHQDQ", w: true, broadcastBytes: 8}, +} + +func decodedX86EVEXPackedUnpackInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 { + return Instr{}, 0, false, nil + } + form, recognized := x86RawEVEXUnpackForms[p.opcode] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX packed unpack: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.w != form.w || p.addressOverride || p.vectorLength > 2 { + return fail("invalid EVEX fixed, element width, address, or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if mode == 32 && p.mask != 0 { + return fail("386 mask form exceeds the Go assembler frontend's operand limit") + } + modRM := code[p.modRM] + if p.broadcast && (form.broadcastBytes == 0 || modRM>>6 == 3) { + return fail("broadcast requires an enabled scalar-memory row") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && + (destinationNumber >= 8 || p.upper >= 8 || + modRM>>6 == 3 && int(modRM&7)+p.b*8+p.x*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("%w: EVEX unpack high register has no proved physical mode32 binding", ErrProbeNeedsContext) + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + disp8Scale := 16 << p.vectorLength + if p.broadcast { + disp8Scale = form.broadcastBytes + } + first, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))} + args := []Operand{first, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_evex_unpack_test.go b/amd64_raw_evex_unpack_test.go new file mode 100644 index 000000000..e5b11cafb --- /dev/null +++ b/amd64_raw_evex_unpack_test.go @@ -0,0 +1,203 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawEVEXUnpack(opcode, width, mask int, zeroing, broadcast, memory bool) []byte { + form := x86RawEVEXUnpackForms[opcode] + p1 := byte(0x5d) // EVEX.vvvv = source register 4, pp = 66. + if form.w { + p1 |= 0x80 + } + p2 := byte(width<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // first register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX), destination register 2. + } + code := []byte{0x62, 0xf1, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawEVEXUnpackSeBiShogunRegression(t *testing.T) { + // Go's assembler disassembles these bytes as VPUNPCKHDQ Z2, Z3, Z7. + code := []byte{0x62, 0xf1, 0x65, 0x48, 0x6a, 0xfa, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "onesCountInt32 AVX512 VPUNPCKHDQ", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPUNPCKHDQ" || + decoded[0].Args[0].Reg != "Z2" || decoded[0].Args[1].Reg != "Z3" || + decoded[0].Args[2].Reg != "Z7" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXUnpackCompleteGoForms(t *testing.T) { + if len(x86RawEVEXUnpackForms) != 8 { + t.Fatalf("modeled %d EVEX unpack rows, want eight", len(x86RawEVEXUnpackForms)) + } + for opcode, form := range x86RawEVEXUnpackForms { + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, variant := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "memory", memory: true}, + {name: "zeroing memory", mask: 7, zeroing: true, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + {name: "broadcast zero memory", broadcast: true, mask: 7, zeroing: true, memory: true}, + } { + if variant.broadcast && form.broadcastBytes == 0 { + continue + } + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawEVEXUnpack(opcode, width, variant.mask, variant.zeroing, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86EVEXPackedUnpackInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.broadcast { + wantOp += ".BCST" + } + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != vector+"4" || + got.Args[len(got.Args)-1].String() != vector+"2" || + !variant.memory && got.Args[0].String() != vector+"3" { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXUnpackRIPLiteral(t *testing.T) { + for opcode, form := range x86RawEVEXUnpackForms { + for width := 0; width < 3; width++ { + for _, broadcast := range []bool{false, true} { + if broadcast && form.broadcastBytes == 0 { + continue + } + code := encodeX86RawEVEXUnpack(opcode, width, 1, false, broadcast, false) + code[5] = 0x15 // RIP-relative first source, destination 2. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + if broadcast { + literalWidth = form.broadcastBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d/broadcast%t", form.op, width, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != wantOp { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXUnpackRejectsInvalidForms(t *testing.T) { + base := encodeX86RawEVEXUnpack(0x6a, 2, 0, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[2] |= 0x80 }, // HDQ requires EVEX.W0 + func(code []byte) { code[3] |= 0x80 }, // zeroing needs a mask + func(code []byte) { code[3] |= 0x10 }, // broadcast requires memory + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86EVEXPackedUnpackInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + byteBroadcast := encodeX86RawEVEXUnpack(0x60, 2, 0, false, true, true) + if _, _, ok, err := decodedX86EVEXPackedUnpackInstruction(byteBroadcast, 64); !ok || err == nil { + t.Fatalf("accepted VPUNPCKLBW broadcast %x: ok=%v err=%v", byteBroadcast, ok, err) + } +} + +func TestTranslateX86RawEVEXUnpackLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawEVEXUnpack(SB),$0-0\n") + for opcode, form := range x86RawEVEXUnpackForms { + for width := 0; width < 3; width++ { + for _, variant := range []struct { + memory, broadcast bool + }{ + {}, {memory: true}, {memory: true, broadcast: true}, + } { + if variant.broadcast && form.broadcastBytes == 0 { + continue + } + code := encodeX86RawEVEXUnpack(opcode, width, 0, false, variant.broadcast, variant.memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawEVEXUnpack": {Name: "rawEVEXUnpack", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-evex-unpack.ll", "raw-evex-unpack.o", ir) + }) + } +} diff --git a/amd64_raw_expand_test.go b/amd64_raw_expand_test.go new file mode 100644 index 000000000..90de94c79 --- /dev/null +++ b/amd64_raw_expand_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawExpandForms = []struct { + opcode int + widthBit int + op Op + laneBytes int +}{ + {0x62, 0, "VPEXPANDB", 1}, + {0x62, 1, "VPEXPANDW", 2}, + {0x88, 0, "VEXPANDPS", 4}, + {0x88, 1, "VEXPANDPD", 8}, + {0x89, 0, "VPEXPANDD", 4}, + {0x89, 1, "VPEXPANDQ", 8}, +} + +func encodeX86RawExpand(opcode, widthBit, vectorBits, mask int, zeroing, memory bool) []byte { + p1 := byte(0x7d) // EVEX.vvvv is unused, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xd3) // source register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX) source, destination register 2. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawExpandSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VEXPANDPS.Z Z2, K1, Z3. + code := []byte{0x62, 0xf2, 0x7d, 0xc9, 0x88, 0xda, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "fastCumSumFloat32 VEXPANDPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VEXPANDPS.Z" || + decoded[0].Args[0].Reg != "Z2" || decoded[0].Args[1].Reg != "K1" || + decoded[0].Args[2].Reg != "Z3" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawExpandCompleteGoForms(t *testing.T) { + if len(x86RawExpandForms) != len(amd64PackedExpandSpecs) { + t.Fatalf("raw/lower expand grammar rows = %d/%d", len(x86RawExpandForms), len(amd64PackedExpandSpecs)) + } + for _, form := range x86RawExpandForms { + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, variant := range []struct { + name string + mask int + zeroing bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "zeroing register", mask: 7, zeroing: true}, + {name: "memory", memory: true}, + {name: "masked memory", mask: 1, memory: true}, + {name: "zeroing memory", mask: 4, zeroing: true, memory: true}, + } { + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawExpand(form.opcode, form.widthBit, vectorBits, variant.mask, variant.zeroing, variant.memory) + got, length, ok, err := decodedX86PackedExpandInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != vector+"2" || + !variant.memory && got.Args[0].String() != vector+"3" || + variant.memory && got.Args[0].String() != fmt.Sprintf("%d(AX)", form.laneBytes) || + len(got.Args) != 2+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawExpandRejectsInvalidForms(t *testing.T) { + base := encodeX86RawExpand(0x88, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x60 }, // reserved vector length + func(code []byte) { code[3] |= 0x10 }, // broadcast unavailable + func(code []byte) { code[2] &^= 0x08 }, // EVEX.vvvv is reserved + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86PackedExpandInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + if _, _, ok, err := decodedX86PackedExpandInstruction(encodeX86RawExpand(0x88, 0, 2, 1, false, false), 32); !ok || err == nil { + t.Fatalf("accepted 386 masked expand: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawExpandLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawExpand(SB),$0-0\n") + for _, form := range x86RawExpandForms { + for _, memory := range []bool{false, true} { + mask := 1 + if target.goarch == "386" { + mask = 0 + } + code := encodeX86RawExpand(form.opcode, form.widthBit, 0, mask, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawExpand": {Name: "rawExpand", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-expand.ll", "raw-expand.o", ir) + }) + } +} diff --git a/amd64_raw_explicit_mask_move.go b/amd64_raw_explicit_mask_move.go new file mode 100644 index 000000000..1f089ad56 --- /dev/null +++ b/amd64_raw_explicit_mask_move.go @@ -0,0 +1,66 @@ +package plan9asm + +import ( + "fmt" +) + +// Go 1.27's _yvmaskmovpd table describes the four VEX X/Y masked load/store +// families. Opcode bit 1 selects store; W selects the D/Q integer lane. +var x86RawExplicitMaskMoveOps = map[int]Op{ + 0x2c: "VMASKMOVPS", + 0x2d: "VMASKMOVPD", + 0x8c: "VPMASKMOVD", +} + +func decodedX86ExplicitMaskMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.evex || p.mapNumber != 2 { + return Instr{}, 0, false, nil + } + // The floating store opcodes are 2E/2F, integer stores 8E/8F. + base := p.opcode &^ 2 + op, known := x86RawExplicitMaskMoveOps[base] + if !known { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("explicit mask move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.vectorLength > 1 || p.w && base != 0x8c { + return fail("invalid VEX prefix or width") + } + if base == 0x8c && p.w { + op = "VPMASKMOVQ" + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 == 3 { + return fail("explicit mask move requires a memory operand") + } + width := "X" + if p.vectorLength == 1 { + width = "Y" + } + memory, consumed, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8))} + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + args := []Operand{memory, mask, data} + if p.opcode&2 != 0 { + args = []Operand{data, mask, memory} + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s, %s", op, args[0].String(), args[1].String(), args[2].String()), + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_explicit_mask_move_test.go b/amd64_raw_explicit_mask_move_test.go new file mode 100644 index 000000000..847cf39e4 --- /dev/null +++ b/amd64_raw_explicit_mask_move_test.go @@ -0,0 +1,102 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawExplicitMaskMoveGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawmaskmoveforms(SB),4,$0-0\n") + for _, op := range []string{"VMASKMOVPS", "VMASKMOVPD", "VPMASKMOVD", "VPMASKMOVQ"} { + for _, width := range []string{"X", "Y"} { + for _, reg := range []int{1, 7, 15} { + fmt.Fprintf(&source, "%s 8(AX), %s2, %s%d\n", op, width, width, reg) + fmt.Fprintf(&source, "%s %s%d, %s2, 16(BX)\n", op, width, reg, width) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d = %s %v, want %s %v", index, got.Op, got.Args, want[index].Op, want[index].Args) + } + } + if len(code) == 0 || code[len(code)-1] != 0xc3 { + t.Fatal("Go assembler did not end the fixture with RET") + } + var raw strings.Builder + raw.WriteString("TEXT rawmaskmoveforms(SB),4,$0-0\n") + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + raw.WriteString("RET\n") + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{ + Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"rawmaskmoveforms": {Name: "rawmaskmoveforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-explicit-mask-move.ll", "raw-explicit-mask-move.o", ir) + } + }) + } +} + +func TestX86RawExplicitMaskMoveRejectsInvalidEncodings(t *testing.T) { + for _, test := range []struct { + name string + code []byte + }{ + {name: "missing ModRM", code: []byte{0xc4, 0xa2, 0x69, 0x8e}}, + {name: "register destination", code: []byte{0xc4, 0xe2, 0x69, 0x8e, 0xc1}}, + {name: "wrong pp", code: []byte{0xc4, 0xa2, 0x68, 0x8e, 0x0c, 0x82}}, + {name: "floating W1", code: []byte{0xc4, 0xe2, 0xe9, 0x2c, 0x01}}, + {name: "address override", code: []byte{0x67, 0xc4, 0xa2, 0x69, 0x8e, 0x0c, 0x82}}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86ExplicitMaskMoveInstruction(test.code, 64); !matched || err == nil { + t.Fatalf("invalid %x matched=%v error=%v", test.code, matched, err) + } + }) + } + for _, code := range [][]byte{ + {0xc4, 0xe3, 0x69, 0x8e, 0xc1}, + {0x62, 0xe2, 0x69, 0x08, 0x8e, 0xc1}, + } { + if _, _, matched, err := decodedX86ExplicitMaskMoveInstruction(code, 64); matched || err != nil { + t.Fatalf("other opcode %x matched=%v error=%v", code, matched, err) + } + } +} diff --git a/amd64_raw_far_return.go b/amd64_raw_far_return.go new file mode 100644 index 000000000..1d07064c4 --- /dev/null +++ b/amd64_raw_far_return.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// decodedX86FarReturn preserves the actual operand-size attribute before +// x86asm.GoSyntax erases LRET's width or emits Intel's IRET/IRETD aliases. +// In both 32- and 64-bit modes a bare far return has a 32-bit operand size; +// only an effective REX.W promotes it to 64 bits. See Intel SDM volume 2, +// RET and IRET/IRETD/IRETQ, and Go's shared x86 ynone/yret encoder rows. +func decodedX86FarReturn(inst x86asm.Inst) (syntax string, recognized bool, err error) { + kind := amd64InterruptReturn + switch inst.Op { + case x86asm.LRET: + kind = amd64FarReturn + case x86asm.IRET: + if inst.DataSize != 16 { + return "", true, fmt.Errorf("raw IRET has inconsistent operand size %d", inst.DataSize) + } + case x86asm.IRETD: + if inst.DataSize != 32 { + return "", true, fmt.Errorf("raw IRETD has inconsistent operand size %d", inst.DataSize) + } + case x86asm.IRETQ: + if inst.DataSize != 64 { + return "", true, fmt.Errorf("raw IRETQ has inconsistent operand size %d", inst.DataSize) + } + default: + return "", false, nil + } + if (inst.Mode != 32 && inst.Mode != 64) || (inst.Mode == 32 && inst.DataSize == 64) { + return "", true, fmt.Errorf("raw %s operand size %d is not supported in mode %d", inst.Op, inst.DataSize, inst.Mode) + } + for _, prefix := range inst.Prefix { + if prefix == 0 { + continue + } + // Register only the size prefixes represented by the typed family. + // Never silently drop LOCK, REP, segment or address-size prefixes. + // Decoder-rejected prefix orderings remain explicit decode failures. + if prefix&x86asm.PrefixInvalid != 0 || (prefix&0xff != x86asm.PrefixDataSize && !prefix.IsREX()) { + return "", true, fmt.Errorf("raw %s has an invalid or unregistered prefix %s", inst.Op, prefix) + } + if prefix.IsREX() && inst.Mode != 64 { + return "", true, fmt.Errorf("raw %s has a REX prefix outside 64-bit mode", inst.Op) + } + } + for _, extra := range inst.Args[1:] { + if extra != nil { + return "", true, fmt.Errorf("raw %s has extra operands", inst.Op) + } + } + argument := "" + if inst.Args[0] != nil { + immediate, ok := inst.Args[0].(x86asm.Imm) + if kind != amd64FarReturn || !ok || immediate < 0 || immediate > 65535 { + return "", true, fmt.Errorf("raw %s requires no operand or an encoded unsigned imm16", inst.Op) + } + argument = fmt.Sprintf(" $%d", immediate) + } + for op, spec := range amd64FarReturnSpecs { + if spec.kind == kind && spec.bits == inst.DataSize { + return string(op) + argument, true, nil + } + } + return "", true, fmt.Errorf("raw %s has unregistered operand size %d", inst.Op, inst.DataSize) +} diff --git a/amd64_raw_far_return_test.go b/amd64_raw_far_return_test.go new file mode 100644 index 000000000..0b3e61679 --- /dev/null +++ b/amd64_raw_far_return_test.go @@ -0,0 +1,408 @@ +package plan9asm + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86FarReturnGrammarMatchesActualGoRows(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src", "cmd", "internal", "obj", "x86", "asm6.go") + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Fatal(err) + } + want := make(map[Op]amd64FarReturnSpec) + ast.Inspect(file, func(node ast.Node) bool { + row, ok := node.(*ast.CompositeLit) + if !ok || len(row.Elts) == 0 { + return true + } + name, ok := row.Elts[0].(*ast.Ident) + if !ok || (!strings.HasPrefix(name.Name, "AIRET") && !strings.HasPrefix(name.Name, "ARETF")) { + return true + } + if len(row.Elts) != 4 { + t.Fatalf("unrecognized actual Go far-return opcode row %s", name.Name) + } + operands, ok := row.Elts[1].(*ast.Ident) + if !ok || (operands.Name != "ynone" && operands.Name != "yret") { + t.Fatalf("unrecognized actual Go far-return operand row: %#v", row.Elts[1]) + } + prefix, ok := row.Elts[2].(*ast.Ident) + if !ok { + t.Fatalf("unrecognized actual Go far-return prefix: %#v", row.Elts[2]) + } + bits := map[string]int{"Pe": 16, "Px": 32, "Pw": 64}[prefix.Name] + if bits == 0 { + t.Fatalf("unrecognized actual Go far-return width prefix %s", prefix.Name) + } + kind := amd64InterruptReturn + if operands.Name == "yret" { + kind = amd64FarReturn + } + want[Op(strings.TrimPrefix(name.Name, "A"))] = amd64FarReturnSpec{ + kind: kind, bits: bits, allowImmediate: operands.Name == "yret", + } + return false + }) + if len(want) != len(amd64FarReturnSpecs) { + t.Fatalf("actual Go rows %v do not match typed inventory %v", want, amd64FarReturnSpecs) + } + for op, expected := range want { + spec, ok := amd64FarReturnSpecs[op] + if !ok || spec != expected { + t.Fatalf("actual Go %s spec=%+v is missing or differs", op, expected) + } + } + // The row names alone are not operand completeness: audit both entire + // referenced ytabs, including the operand-free RETF and imm16 truncation. + seen := make(map[string]bool) + ast.Inspect(file, func(node ast.Node) bool { + value, ok := node.(*ast.ValueSpec) + if !ok || len(value.Names) != 1 || (value.Names[0].Name != "ynone" && value.Names[0].Name != "yret") { + return true + } + name := value.Names[0].Name + if len(value.Values) != 1 { + t.Fatalf("unrecognized %s initializer", name) + } + table, ok := value.Values[0].(*ast.CompositeLit) + if !ok { + t.Fatalf("unrecognized %s table", name) + } + classes := make(map[string]bool) + for _, element := range table.Elts { + row, ok := element.(*ast.CompositeLit) + if !ok || len(row.Elts) != 3 { + t.Fatalf("unrecognized %s operand row", name) + } + operands, ok := row.Elts[2].(*ast.CompositeLit) + if !ok || len(operands.Elts) > 1 { + t.Fatalf("unrecognized %s argument list", name) + } + class := "none" + if len(operands.Elts) == 1 { + identifier, ok := operands.Elts[0].(*ast.Ident) + if !ok { + t.Fatalf("unrecognized %s argument class", name) + } + class = identifier.Name + } + if classes[class] { + t.Fatalf("duplicate %s argument class %s", name, class) + } + classes[class] = true + } + wantCount := 1 + if name == "yret" { + wantCount = 2 + } + if len(classes) != wantCount || !classes["none"] || (name == "yret" && !classes["Yi32"]) { + t.Fatalf("actual Go %s forms changed: %v", name, classes) + } + seen[name] = true + return false + }) + if len(seen) != 2 { + t.Fatal("actual Go far-return operand tables were not both observed") + } +} + +func TestX86RawFarReturnPrefixAndModeAxes(t *testing.T) { + for _, test := range []struct { + name string + mode int + code []byte + want string + }{ + {name: "bare-386", mode: 32, code: []byte{0xcb}, want: "RETFL"}, + {name: "bare-amd64", mode: 64, code: []byte{0xcb}, want: "RETFL"}, + {name: "operand-386", mode: 32, code: []byte{0x66, 0xcb}, want: "RETFW"}, + {name: "operand-amd64", mode: 64, code: []byte{0x66, 0xcb}, want: "RETFW"}, + {name: "rex-without-w", mode: 64, code: []byte{0x47, 0xcb}, want: "RETFL"}, + {name: "rex-w", mode: 64, code: []byte{0x4f, 0xcb}, want: "RETFQ"}, + {name: "rex-w-overrides-66", mode: 64, code: []byte{0x66, 0x48, 0xcb}, want: "RETFQ"}, + {name: "repeated-operand", mode: 64, code: []byte{0x66, 0x66, 0xcf}, want: "IRETW"}, + } { + t.Run(test.name, func(t *testing.T) { + inst, err := x86asm.Decode(test.code, test.mode) + if err != nil || inst.Len != len(test.code) { + t.Fatalf("actual decoder: %+v %v", inst, err) + } + got, err := decodedX86GoSyntax(inst, test.code) + if err != nil || got != test.want { + t.Fatalf("actual prefixes %+v DataSize%d: %q %v, want %q", inst.Prefix, inst.DataSize, got, err, test.want) + } + }) + } + // 48 is an actual DEC opcode in 32-bit mode. Never treat the same + // byte string as an unsupported 64-bit far return in that mode. + decoded, err := decodeX86RawDirectiveGroup([]byte{0x48, 0xcb}, 32, 0, "32-bit DEC then far return", nil) + if err != nil || len(decoded) != 2 || decoded[0].Op != "DECL" || decoded[1].Op != "RETFL" { + t.Fatalf("32-bit 48 CB changed instruction boundaries: %+v %v", decoded, err) + } + decoded, err = decodeX86RawDirectiveGroup([]byte{0x48, 0x66, 0xcb}, 32, 0, "32-bit DEC then 16-bit far return", nil) + if err != nil || len(decoded) != 2 || decoded[0].Op != "DECL" || decoded[1].Op != "RETFW" { + t.Fatalf("32-bit 48 66 CB changed instruction boundaries: %+v %v", decoded, err) + } +} + +func TestX86RawFarReturnRejectsUnprovedForms(t *testing.T) { + for _, code := range [][]byte{ + {0xf0, 0xcb}, {0xf0, 0xcf}, {0xf2, 0xcb}, {0xf3, 0xcf}, + {0x67, 0xcb}, {0x64, 0xcf}, // Not a registered width-only prefix. + {0xca}, {0xca, 0x01}, {0x66}, {0x48}, + {0x48, 0x66, 0xcb}, // Pinned decoder rejects nonadjacent REX. + } { + for _, mode := range []int{32, 64} { + if mode == 32 && code[0] == 0x48 { + continue // Valid DEC, not a REX prefix or prefix ordering. + } + _, err := decodeX86RawDirectiveGroup(code, mode, 0, "invalid or unregistered raw far return", nil) + if err == nil { + t.Errorf("accepted unproved %x mode%d", code, mode) + } + } + } + for _, inst := range []x86asm.Inst{ + {Op: x86asm.LRET, Mode: 16, DataSize: 16}, + {Op: x86asm.LRET, Mode: 32, DataSize: 64}, + {Op: x86asm.LRET, Mode: 64, DataSize: 8}, + {Op: x86asm.IRETQ, Mode: 64, DataSize: 32}, + {Op: x86asm.IRETD, Mode: 64, DataSize: 16}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.Imm(-1)}}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.Imm(65536)}}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.EAX}}, + {Op: x86asm.LRET, Mode: 64, DataSize: 32, Args: x86asm.Args{nil, x86asm.Imm(1)}}, + {Op: x86asm.IRETD, Mode: 64, DataSize: 32, Args: x86asm.Args{x86asm.Imm(1)}}, + } { + if got, err := decodedX86GoSyntax(inst, nil); err == nil { + t.Errorf("accepted inconsistent raw form %+v as %q", inst, got) + } + } +} + +func TestX86FarReturnRejectsFloatingAndWidthlessOperands(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + instructions := []string{"LRET", "LRET $4", "RETFL $unknown"} + for op := range amd64FarReturnSpecs { + for _, argument := range []string{"$0.0", "$-0.0", "$1.5", "$-1.5"} { + instructions = append(instructions, string(op)+" "+argument) + } + } + for _, instruction := range instructions { + source := "TEXT bad(SB),516,$0-0\n" + instruction + "\n" + requireX86GoAssemblerResult(t, arch, source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + if _, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Errorf("accepted %s %q outside actual Go integer/width grammar", arch, instruction) + } + } + } +} + +func TestX86RawFarReturnRetainsDecodedWidthAndImmediate(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, width := range []struct { + bits int + prefix []byte + suffix string + }{ + {bits: 16, prefix: []byte{0x66}, suffix: "W"}, + {bits: 32, suffix: "L"}, + {bits: 64, prefix: []byte{0x48}, suffix: "Q"}, + } { + if mode == 32 && width.bits == 64 { + continue // 48 is DEC in 32-bit mode, not a width prefix. + } + for _, form := range []struct { + name string + code []byte + args string + }{ + {name: "IRET", code: []byte{0xcf}}, + {name: "RETF", code: []byte{0xcb}}, + {name: "RETF", code: []byte{0xca, 0x00, 0x00}, args: " $0"}, + {name: "RETF", code: []byte{0xca, 0x34, 0x12}, args: " $4660"}, + {name: "RETF", code: []byte{0xca, 0xff, 0xff}, args: " $65535"}, + } { + code := append(append([]byte(nil), width.prefix...), form.code...) + inst, err := x86asm.Decode(code, mode) + if err != nil || inst.Len != len(code) || inst.DataSize != width.bits { + t.Fatalf("actual decoder %x mode%d: %+v %v", code, mode, inst, err) + } + want := form.name + width.suffix + form.args + got, err := decodedX86GoSyntax(inst, code) + if err != nil || got != want { + t.Errorf("%x mode%d DataSize%d: got %q %v, want %q", code, mode, inst.DataSize, got, err, want) + } + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, "raw far-return bytes", nil) + if err != nil || len(decoded) != 1 || string(decoded[0].Op) != form.name+width.suffix { + t.Errorf("whole raw group %x mode%d: %+v %v", code, mode, decoded, err) + } + } + } + } +} + +func TestX86RawFarReturnCompleteFiveTargetObjectsMatchGo(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 is required") + } + for _, target := range []struct{ os, arch, triple string }{ + {os: "darwin", arch: "amd64", triple: "x86_64-apple-darwin"}, + {os: "linux", arch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {os: "windows", arch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {os: "linux", arch: "386", triple: "i386-unknown-linux-gnu"}, + {os: "windows", arch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var rawSource strings.Builder + sigs := make(map[string]FuncSig) + wantBytes := make(map[string][]byte) + for op, spec := range amd64FarReturnSpecs { + if target.arch == "386" && spec.bits == 64 { + continue + } + forms := []amd64FarReturnForm{{}} + if spec.allowImmediate { + for _, value := range []uint16{0, 1, 0x1234, 0xffff} { + forms = append(forms, amd64FarReturnForm{hasImmediate: true, immediate: value}) + } + } + for _, form := range forms { + name := fmt.Sprintf("far_%s_%t_%d", op, form.hasImmediate, form.immediate) + prefix := []byte(nil) + if spec.bits == 16 { + prefix = []byte{0x66} + } else if spec.bits == 64 { + prefix = []byte{0x48} + } + code := append(prefix, byte(0xcb)) + if spec.kind == amd64InterruptReturn { + code[len(code)-1] = 0xcf + } else if form.hasImmediate { + code[len(code)-1] = 0xca + code = append(code, byte(form.immediate), byte(form.immediate>>8)) + } + raw := fmt.Sprintf("TEXT %s(SB),516,$0-0\n", name) + for _, value := range code { + raw += fmt.Sprintf("BYTE $0x%02x\n", value) + } + named := fmt.Sprintf("TEXT %s(SB),516,$0-0\n%s", name, op) + if form.hasImmediate { + named += fmt.Sprintf(" $%d", form.immediate) + } + named += "\n" + // Independently authenticate the raw and canonical named + // encodings with Go, not merely the LLVM assembly spelling. + for _, source := range []string{raw, named} { + dir := t.TempDir() + path := filepath.Join(dir, "far.s") + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + command := exec.Command("go", "tool", "asm", "-S", "-o", filepath.Join(dir, "far.o"), path) + command.Env = append(os.Environ(), "GOOS="+target.os, "GOARCH="+target.arch, "GOTOOLCHAIN=local") + output, err := command.CombinedOutput() + parts := make([]string, len(code)) + for i, value := range code { + parts[i] = fmt.Sprintf("%02x", value) + } + want := regexp.MustCompile(`(?m)^\s*0x0000 ` + strings.Join(parts, " ") + `(?:\s|$)`) + if err != nil || !want.Match(output) { + t.Fatalf("original Go bytes %x: %v\n%s\n%s", code, err, source, output) + } + } + rawSource.WriteString(raw) + sigs[name] = FuncSig{Name: name, Ret: Void} + wantBytes[name] = code + } + } + file, err := Parse(ArchAMD64, rawSource.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for op, spec := range amd64FarReturnSpecs { + if target.arch == "386" && spec.bits == 64 { + continue + } + if !strings.Contains(ir, spec.encoding(amd64FarReturnForm{})) { + t.Errorf("raw %s lost typed byte lowering", op) + } + } + if strings.Count(ir, "unreachable") != len(sigs) { + t.Fatal("raw far return lost terminal control semantics") + } + requireX86RawFarReturnObjectBytes(t, llc, target.triple, ir, wantBytes) + }) + } +} + +func requireX86RawFarReturnObjectBytes(t *testing.T, llc, triple, ir string, want map[string][]byte) { + t.Helper() + objdump := findLLVM22Tool("llvm-objdump") + if objdump == "" { + t.Fatal("LLVM22 llvm-objdump is required") + } + dir := t.TempDir() + llPath := filepath.Join(dir, "far.ll") + object := filepath.Join(dir, "far.o") + if err := os.WriteFile(llPath, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + output, err := exec.Command(llc, "-O0", "-mtriple="+triple, "-filetype=obj", llPath, "-o", object).CombinedOutput() + if err != nil { + t.Fatalf("LLVM22 object: %v\n%s", err, output) + } + output, err = exec.Command(objdump, "-d", object).CombinedOutput() + if err != nil { + t.Fatalf("LLVM22 disassembly: %v\n%s", err, output) + } + // Match each terminal instruction's complete bytes inside its own symbol, + // not merely a coincidental sequence in another function or an IR string. + sections := regexp.MustCompile(`(?m)^[0-9a-f]+ <_?(far_[^>]+)>:\n`).FindAllSubmatchIndex(output, -1) + seen := make(map[string]bool) + for index, section := range sections { + name := string(output[section[2]:section[3]]) + code, ok := want[name] + if !ok { + t.Fatalf("unexpected emitted far-return symbol %s", name) + } + end := len(output) + if index+1 < len(sections) { + end = sections[index+1][0] + } + parts := make([]string, len(code)) + for i, value := range code { + parts[i] = fmt.Sprintf("%02x", value) + } + instruction := regexp.MustCompile(`(?m)^\s*[0-9a-f]+:\s+` + strings.Join(parts, " ") + `\s+(?:iret|lret)`) + if !instruction.Match(output[section[1]:end]) { + t.Fatalf("LLVM22 symbol %s lacks exact terminal bytes %x:\n%s", name, code, output[section[1]:end]) + } + seen[name] = true + } + if len(seen) != len(want) { + t.Fatalf("LLVM22 object contains %d/%d authenticated far-return symbols", len(seen), len(want)) + } +} diff --git a/amd64_raw_float_logical_literal_test.go b/amd64_raw_float_logical_literal_test.go new file mode 100644 index 000000000..6e966888d --- /dev/null +++ b/amd64_raw_float_logical_literal_test.go @@ -0,0 +1,109 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawFloatLogicalLiteral(opcode, pp byte, vex3 bool, width int) []byte { + code := encodeX86VEXPackedFloatLogical(opcode, pp, vex3, width == 32, false, 1, 2, 0) + code[len(code)-1] = 0x0d // RIP+disp32 rather than X0; destination stays X/Y1. + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*13+5)) + } + return code +} + +func TestDecodeX86RawFloatLogicalRIPDataCompleteVEXFamily(t *testing.T) { + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, vex3 := range []bool{false, true} { + for _, width := range []int{16, 32} { + name := fmt.Sprintf("%s%s/vex3%t/%d", stem, suffix, vex3, width) + t.Run(name, func(t *testing.T) { + code := x86RawFloatLogicalLiteral(opcode, pp, vex3, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != Op(stem+suffix) || + len(decoded[0].Args) != 3 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s%s source-local data", code, decoded, stem, suffix) + } + if !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("source-local data = %x, want %x", decoded[0].x86RIPLiteralData, code[len(code)-width:]) + } + }) + } + } + } + } +} + +func TestDecodeX86RawFloatLogicalRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { + code[5] = 127 + return code + }, + "overlapping instruction": func(code []byte) []byte { + code[5] = 0 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawFloatLogicalLiteral(0x57, 1, true, 32)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe floating logical RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawFloatLogicalRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + name := fmt.Sprintf("literal%s%s", stem, suffix) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawFloatLogicalLiteral(opcode, pp, true, 32) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-float-logical-literal.ll", "raw-float-logical-literal.o", ir) + }) + } +} diff --git a/amd64_raw_fma3_literal_test.go b/amd64_raw_fma3_literal_test.go new file mode 100644 index 000000000..dd8580b3a --- /dev/null +++ b/amd64_raw_fma3_literal_test.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "sort" + "strings" + "testing" +) + +func x86FMA3Opcodes() []byte { + opcodes := make([]byte, 0, len(decodedX86VEXFMA3Ops)) + for opcode := range decodedX86VEXFMA3Ops { + opcodes = append(opcodes, opcode) + } + sort.Slice(opcodes, func(i, j int) bool { return opcodes[i] < opcodes[j] }) + return opcodes +} + +func x86RawFMA3Literal(opcode byte, width64, evex bool, width int, broadcast bool) []byte { + spec := decodedX86VEXFMA3Ops[opcode] + var code []byte + if evex { + code = encodeX86EVEXFMA3(opcode, width64, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), broadcast, true, 7, 0, 0, 0) + } else { + code = encodeX86VEXFMA3(opcode, width64, width == 32, 0, 0, 0) + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if spec.scalar || broadcast { + dataWidth = 4 + if width64 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*31+13)) + } + return code +} + +func TestDecodeX86RawFMA3RIPDataCompleteFamily(t *testing.T) { + for _, opcode := range x86FMA3Opcodes() { + spec := decodedX86VEXFMA3Ops[opcode] + for _, width64 := range []bool{false, true} { + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + if spec.scalar { + widths = []int{16} + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && !spec.scalar { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", spec.op(width64), evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawFMA3Literal(opcode, width64, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := spec.op(width64) + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } + } +} + +func TestDecodeX86RawFMA3RIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawFMA3Literal(0xb8, false, true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawFMA3RIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, opcode := range x86FMA3Opcodes() { + spec := decodedX86VEXFMA3Ops[opcode] + width64 := index%2 == 0 + evex := index%3 == 0 + width := 32 + if spec.scalar { + width = 16 + } else if evex { + width = 64 + } + broadcast := evex && !spec.scalar + code := x86RawFMA3Literal(opcode, width64, evex, width, broadcast) + name := fmt.Sprintf("fma3Literal%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if spec.scalar || broadcast { + dataWidth = 4 + if width64 { + dataWidth = 8 + } + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "fma3-literal.ll", "fma3-literal.o", ir) + }) + } +} diff --git a/amd64_raw_fp16_binary_test.go b/amd64_raw_fp16_binary_test.go new file mode 100644 index 000000000..2f5ce64b7 --- /dev/null +++ b/amd64_raw_fp16_binary_test.go @@ -0,0 +1,252 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var rawFP16BinaryOpcodes = map[byte]Op{ + 0x58: "VADDPH", + 0x59: "VMULPH", + 0x5c: "VSUBPH", + 0x5d: "VMINPH", + 0x5e: "VDIVPH", + 0x5f: "VMAXPH", +} + +func encodeRawFP16Binary(opcode, width byte, source1, source2, destination, mask int, memory, broadcast, zero bool) []byte { + code := encodeX86EVEXBinaryFloat(opcode, 0, width, broadcast, zero, mask, destination, source1, source2) + code[1] = code[1]&^byte(0x0f) | 5 + if memory { + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestDecodeRawFP16BinaryGoATReproduction(t *testing.T) { + // VADDPH 64(SI)(R8*2), Z1, Z1 from go-highway's GoAT output. + code := []byte{0x62, 0xb5, 0x74, 0x48, 0x58, 0x4c, 0x46, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "GoAT VADDPH", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VADDPH" { + t.Fatalf("decoded %#v, want one VADDPH", decoded) + } +} + +func TestDecodeRawFP16BinaryCompleteForms(t *testing.T) { + for opcode, baseOp := range rawFP16BinaryOpcodes { + for width := byte(0); width < 3; width++ { + for _, memory := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && !memory { + continue + } + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zero := range []bool{false, true} { + if zero && !masked { + continue + } + code := encodeRawFP16Binary(opcode, width, 20, 21, 22, mask, memory, broadcast, zero) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("%x: decoded %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := baseOp + if broadcast { + wantOp += ".BCST" + } + if zero { + wantOp += ".Z" + } + prefix := [...]string{"X", "Y", "Z"}[width] + if got.Op != wantOp || !got.x86Encoded || + got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s22", prefix) || + got.Args[1].String() != fmt.Sprintf("%s20", prefix) { + t.Fatalf("%x: decoded %+v, want %s on %s", code, got, wantOp, prefix) + } + if !memory && got.Args[0].String() != fmt.Sprintf("%s21", prefix) { + t.Fatalf("%x: wrong register source %+v", code, got) + } + if masked && (len(got.Args) != 4 || got.Args[2].String() != "K3") { + t.Fatalf("%x: missing mask %+v", code, got) + } + } + } + } + } + } + } +} + +func TestTranslateRawFP16BinaryLLVM22Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16Binary(SB),4,$0-0\n") + for opcode := range rawFP16BinaryOpcodes { + for width := byte(0); width < 3; width++ { + for _, memory := range []bool{false, true} { + code := encodeRawFP16Binary(opcode, width, 1, 2, 3, 1, memory, memory, true) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawFP16Binary": {Name: "rawFP16Binary", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "fadd <8 x half>") || + !strings.Contains(ir, "fsub <16 x half>") || + !strings.Contains(ir, "fdiv <32 x half>") || + !strings.Contains(ir, "load i16, ptr") { + t.Fatalf("missing FP16 lane operation or broadcast:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "raw-fp16-binary.ll", "raw-fp16-binary.o", ir) + }) + } +} + +func TestDecodeRawFP16BinaryEmbeddedControlAndRejections(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16Control(SB),4,$0-0\n") + for opcode, baseOp := range rawFP16BinaryOpcodes { + for control := byte(0); control < 4; control++ { + code := encodeRawFP16Binary(opcode, control, 1, 2, 3, 1, false, true, true) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("embedded control %x: %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + want := baseOp + ".SAE.Z" + if opcode != 0x5d && opcode != 0x5f { + want = baseOp + "." + [...]Op{"RN_SAE.Z", "RD_SAE.Z", "RU_SAE.Z", "RZ_SAE.Z"}[control] + } + if got.Op != want || got.Args[0].String() != "Z2" || + got.Args[1].String() != "Z1" || got.Args[2].String() != "K1" || + got.Args[3].String() != "Z3" { + t.Fatalf("embedded control %x: %+v, want %s Z2,Z1,K1,Z3", code, got, want) + } + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawFP16Control": {Name: "rawFP16Control", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "llvm.experimental.constrained.fadd.v32f16") || + !strings.Contains(ir, "llvm.experimental.constrained.fdiv.v32f16") { + t.Fatal("embedded FP16 rounding did not reach constrained LLVM operations") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-fp16-control.ll", "raw-fp16-control.o", ir) + + valid := encodeRawFP16Binary(0x58, 0, 1, 2, 3, 1, false, false, false) + for _, mutation := range []struct { + name string + index int + mask byte + }{ + {"EVEX.W", 2, 0x80}, + {"zero without mask", 3, 0x81}, + {"reserved width", 3, 0x60}, + } { + bad := append([]byte(nil), valid...) + bad[mutation.index] ^= mutation.mask + if _, _, matched, err := decodedX86VEXBinaryFloatInstruction(bad, 64); !matched || err == nil { + t.Errorf("%s %x: matched=%v, err=%v", mutation.name, bad, matched, err) + } + } + if _, _, matched, err := decodedX86VEXBinaryFloatInstruction(valid[:5], 64); matched && err == nil { + t.Fatal("accepted truncated FP16 binary encoding") + } +} + +func TestTranslateRawFP16Binary386Mask(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16386(SB),4,$0-0\n") + for _, code := range [][]byte{ + encodeRawFP16Binary(0x58, 0, 1, 2, 3, 1, false, false, true), + encodeRawFP16Binary(0x5c, 1, 1, 0, 3, 1, true, true, true), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "386", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawFP16386": {Name: "rawFP16386", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-fp16-386.ll", "raw-fp16-386.o", ir) + }) + } +} + +func TestRawFP16BinaryRejectsNamedGoForm(t *testing.T) { + const source = "TEXT invalidNamedHalf(SB),4,$0-0\n\tVADDPH X0, X1, X2\n\tRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"invalidNamedHalf": {Name: "invalidNamedHalf", Ret: Void}}, + }); err == nil { + t.Fatal("accepted FP16 mnemonic absent from Go's named assembler table") + } +} diff --git a/amd64_raw_fp16_convert_runtime_test.go b/amd64_raw_fp16_convert_runtime_test.go new file mode 100644 index 000000000..2a2349550 --- /dev/null +++ b/amd64_raw_fp16_convert_runtime_test.go @@ -0,0 +1,250 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestRawFP16ConversionPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + for _, narrow := range []bool{false, true} { + for length := 0; length < 3; length++ { + for _, broadcast := range []bool{false, true} { + for _, zero := range []bool{false, true} { + name := fmt.Sprintf("halfConversion_%t_%d_%t_%t", narrow, length, broadcast, zero) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ input+0(FP), AX\nMOVQ out+8(FP), SI\nMOVQ old+16(FP), DI\nMOVQ mask+24(FP), CX\nKMOVQ CX, K1\nVMOVDQU64 (DI), Z0\n") + code := rawFP16Conversion(narrow, length, 1, 0, 0, true, broadcast, zero) + code[len(code)-1] = 0 + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void *, void *, const void *, uint64_t);\n", name) + input, output, bytes, total, inactive := "halves", "singles", 2, 16, "UINT32_C(0xa5a5a5a5)" + if narrow { + input, output, bytes, total, inactive = "singles", "halves", 4, 32, "0xa5a5" + } + if zero { + inactive = "0" + } + index := "lane" + if broadcast { + index = "0" + } + fmt.Fprintf(&calls, ` + { + const void *pointer = %s; + if ((mask & ((UINT64_C(1) << %d) - 1)) == 0) pointer = NULL; + if (mask == 1) { + memcpy(guard + page - %d, %s, %d); + pointer = guard + page - %d; + } + %s(pointer, &output, old, mask); + for (int lane = 0; lane < %d; lane++) { + uint32_t expected = lane < %d ? ((mask & (UINT64_C(1) << lane)) ? %s[%s] : %s) : 0; + uint32_t actual = output.%s[lane]; + if (actual != expected) { + fprintf(stderr, "%s mask %%llx lane %%d: %%08x != %%08x\n", + (unsigned long long)mask, lane, actual, expected); + return 2; + } + } + } +`, input, 4<> 10; + double value = bits & 1023; + int power = -24; + if (exponent) { + value += 1024; + power = (int)exponent - 25; + } + while (power < 0) { value *= 0.5; power++; } + while (power > 0) { value *= 2; power--; } + return value; +} + +// Independent numeric oracle: bracket the float between adjacent positive +// half values by binary search, then choose a bound according to sign/mode. +static uint16_t reference(uint32_t bits, int mode) { + unsigned sign = bits >> 31; + bits &= 0x7fffffff; + if (bits >= 0x7f800000) { + unsigned payload = bits == 0x7f800000 ? 0 : ((bits >> 13) & 1023) | 512; + return (uint16_t)((sign << 15) | 0x7c00 | payload); + } + float source; + memcpy(&source, &bits, 4); + double value = source; + unsigned low = 0, high = 0x7bff; + while (low < high) { + unsigned mid = (low + high + 1) / 2; + if (half_value(mid) <= value) low = mid; + else high = mid - 1; + } + unsigned result = low; + double bottom = half_value(low); + if (value != bottom) { + if (mode == 0) { + double top = low == 0x7bff ? 65536.0 : half_value(low + 1); + double midpoint = (bottom + top) * 0.5; + if (value > midpoint || (value == midpoint && (low & 1))) result++; + } else if ((mode == 1 && sign) || (mode == 2 && !sign)) { + result++; + } + } + return (uint16_t)((sign << 15) | result); +} + +int main(void) { + // Exact halfway values, signed overflow, and half-subnormal ties. + const uint32_t values[8] = {0x3f801000, 0x3f803000, 0xbf801000, 0xbf803000, + 0x47800000, 0xc7800000, 0x33000000, 0xb3000000}; + const uint16_t expected[4][8] = { + {0x3c00, 0x3c02, 0xbc00, 0xbc02, 0x7c00, 0xfc00, 0, 0x8000}, + {0x3c00, 0x3c01, 0xbc01, 0xbc02, 0x7bff, 0xfc00, 0, 0x8001}, + {0x3c01, 0x3c02, 0xbc00, 0xbc01, 0x7c00, 0xfbff, 1, 0x8000}, + {0x3c00, 0x3c01, 0xbc00, 0xbc01, 0x7bff, 0xfbff, 0, 0x8000} + }; + uint32_t input[16]; + uint16_t output[32]; + for (int lane = 0; lane < 16; lane++) input[lane] = values[lane % 8]; +` + calls.String() + ` + void (*functions[])(const void *, void *) = {halfRound0, halfRound1, halfRound2, halfRound3}; + uint32_t random = 0x13579bdf; + for (int batch = 0; batch < 4096; batch++) { + for (int lane = 0; lane < 16; lane++) { + random ^= random << 13; + random ^= random >> 17; + random ^= random << 5; + input[lane] = random; + } + for (int mode = 0; mode < 4; mode++) { + functions[mode](input, output); + for (int lane = 0; lane < 16; lane++) { + uint16_t want = reference(input[lane], mode); + if (output[lane] != want) { + fprintf(stderr, "mode %d input %08x: %04x != %04x\n", mode, input[lane], output[lane], want); + return 2; + } + } + } + } + return 0; +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_conversion_rounding", ir, mainC) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + const triple = "x86_64-apple-macosx" + x86IR, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "fp16_conversion_rounding_x86", triple, x86IR, mainC, []string{"/usr/bin/arch", "-x86_64"}) + } +} diff --git a/amd64_raw_fp16_convert_test.go b/amd64_raw_fp16_convert_test.go new file mode 100644 index 000000000..f55651ae3 --- /dev/null +++ b/amd64_raw_fp16_convert_test.go @@ -0,0 +1,194 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func rawFP16Conversion(narrow bool, length, mask, destination, source int, memory, control, zero bool) []byte { + form := x86RawHalfConversionForm{mapNumber: 6, opcode: 0x13} + if narrow { + form.mapNumber, form.opcode = 5, 0x1d + } + code := encodeX86RawEVEXHalfConversion(form, length, mask, zero, control, destination, source, memory, 0) + if memory { + code[len(code)-1] = 1 + } + return code +} + +func TestRawFP16ConversionCompleteFormats(t *testing.T) { + for _, narrow := range []bool{false, true} { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if length == 3 && (memory || !control) { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := rawFP16Conversion(narrow, length, mask, 20, 21, memory, control, zero) + got, size, ok, err := decodedX86PackedHalfConversionInstruction(code, 64) + if err != nil || !ok || size != len(code) || !got.x86Encoded { + t.Fatalf("%x: %+v, size=%d, ok=%v, err=%v", code, got, size, ok, err) + } + vl := length + if control && !memory { + vl = 2 + } + wide, short := [...]string{"X", "Y", "Z"}[vl], [...]string{"X", "X", "Y"}[vl] + src, dst, width := short+"21", wide+"20", 8< %s", code, got, want, src, dst) + } + } + } + } + } + } + } +} + +func TestRawFP16ConversionInvalidFormats(t *testing.T) { + for _, narrow := range []bool{false, true} { + for _, change := range []struct { + index int + bits byte + }{ + {2, 0x80}, {2, 0x04}, {2, 0x08}, {3, 0x08}, {3, 0x80}, {3, 0x60}, + } { + code := rawFP16Conversion(narrow, 0, 0, 0, 1, false, false, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid FP16 conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved encoding %x", code) + } + } + for _, code := range [][]byte{ + rawFP16Conversion(narrow, 0, 0, 8, 1, false, false, false), + rawFP16Conversion(narrow, 0, 0, 1, 8, false, false, false), + } { + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 high FP16 conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted extended register %x", code) + } + } + } +} + +func TestRawFP16ConversionLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT rawFP16Conversion(SB),4,$0-0\n") + for _, narrow := range []bool{false, true} { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if length == 3 && (memory || !control) { + continue + } + for _, mask := range []int{0, 1} { + for _, b := range rawFP16Conversion(narrow, length, mask, 1, 2, memory, control, mask != 0) { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + } + } + } + } + source.WriteString("RET\n") + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawFP16Conversion": {Name: "rawFP16Conversion", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"fpext <4 x half>", "fptrunc <4 x float>", "fcmp olt <16 x float>", "fcmp ogt <16 x float>", + "@llvm.masked.load.v4i16", "@llvm.masked.load.v4i32", "phi i16", "phi i32", "+avx512fp16", "+avx512vl"} { + if !strings.Contains(ir, want) { + t.Fatalf("missing %s", want) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-fp16-conversion.ll", "raw-fp16-conversion.o", ir) + }) + } +} + +func TestRawFP16ConversionRIPAndNamedForms(t *testing.T) { + for _, narrow := range []bool{false, true} { + for length := 0; length < 3; length++ { + for _, broadcast := range []bool{false, true} { + code := rawFP16Conversion(narrow, length, 1, 0, 0, false, broadcast, true) + code[5] = 5 + code = append(code, 1, 0, 0, 0, 0xc3) + width := 8 << length + if broadcast { + width = 2 + } + if narrow { + width *= 2 + } + code = append(code, make([]byte, width)...) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "FP16 conversion literal", map[string]bool{}) + if err != nil || len(got) != 2 || !got[0].x86RIPLiteral || got[0].Args[0].Kind != OpSym { + t.Fatalf("%x: %+v, err=%v", code, got, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated FP16 pool", map[string]bool{}); err == nil { + t.Fatal("accepted truncated constant pool") + } + } + } + } + for _, op := range []string{"VCVTPH2PSX", "VCVTPS2PHX"} { + source := "TEXT namedFP16Conversion(SB),4,$0-0\n" + op + " X0, X1\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedFP16Conversion": {Name: "namedFP16Conversion", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted named %s", op) + } + } +} diff --git a/amd64_raw_fp16_fma_runtime_test.go b/amd64_raw_fp16_fma_runtime_test.go new file mode 100644 index 000000000..ed13791a4 --- /dev/null +++ b/amd64_raw_fp16_fma_runtime_test.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Execute the portable IR on the host, without requiring an AVX512-FP16 +// CPU. Separate object tests check the actual x86 features and every target. +func TestRawFP16FMAPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, cases strings.Builder + sigs := make(map[string]FuncSig) + ops := rawFP16FMAOps() + for _, opcode := range x86FMA3Opcodes() { + op := ops[opcode] + scalar := strings.HasSuffix(string(op), "SH") + lengths := []byte{1} + if scalar { + lengths = []byte{0} + } else if opcode == 0x98 { + lengths = []byte{0, 1, 2} + } + for _, length := range lengths { + for _, zero := range []bool{false, true} { + for _, broadcast := range []bool{false, true} { + if broadcast && (scalar || opcode != 0x98) { + continue + } + name := fmt.Sprintf("halfFMA_%x_%d_%t_%t", opcode, length, zero, broadcast) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ input+0(FP), DI\nMOVQ memory+8(FP), AX\nMOVQ out+16(FP), SI\nMOVQ mask+24(FP), CX\n") + source.WriteString("KMOVQ CX, K1\nVMOVDQU64 (DI), Z0\nVMOVDQU64 64(DI), Z1\n") + code := encodeRawFP16FMA(opcode, length, 1, 0, 0, 1, true, broadcast, zero) + code[len(code)-1] = 0 + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, + Attrs: "nounwind", // Suppress x86-only inference for this portable oracle. + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + order := []int{132, 213, 231}[(opcode>>4)-9] + mode := int(opcode&15) - 8 + if mode < 0 { + mode += 6 // 6/7 are alternating subtract-even/subtract-odd. + } else { + mode /= 2 + } + lanes := 8 << length + if scalar { + lanes = 1 + } + flag := func(value bool) int { + if value { + return 1 + } + return 0 + } + fmt.Fprintf(&declarations, "extern void %s(const void *, const void *, void *, uint64_t);\n", name) + fmt.Fprintf(&cases, " {%s, %d, %d, %d, %d, %d, %d},\n", name, order, mode, lanes, flag(scalar), flag(zero), flag(broadcast)) + } + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: testTargetTriple(runtime.GOOS, runtime.GOARCH), Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := "#include \n#include \n#include \n#include \n" + declarations.String() + x86TestGuardPagesC + ` +typedef void (*operation)(const void *, const void *, void *, uint64_t); +struct test_case { operation run; int order, mode, lanes, scalar, zero, broadcast; }; +static struct test_case cases[] = { +` + cases.String() + `}; + +// These operands and their exact products/sums fit in binary64. Round only +// once to FP16, independently of the translator's half-precision FMA. +static double from_half(uint16_t bits) { + _Float16 value; + memcpy(&value, &bits, 2); + return value; +} + +static uint16_t to_half(double value) { + _Float16 result = (_Float16)value; + uint16_t bits; + memcpy(&bits, &result, 2); + return bits; +} + +int main(void) { + if (setup_guard()) return 1; + uint16_t input[64], memory[32], output[32]; + for (int lane = 0; lane < 32; lane++) { + input[lane] = 0x3c01; // 1 + 2^-10 + input[32 + lane] = 0xbc00; // -1 + memory[lane] = 0x3bfe; // 1 - 2^-10 + } + const uint64_t masks[] = {0, 1, 0xaaaaaaaa, 0xffffffff, UINT64_C(1) << 60}; + for (size_t index = 0; index < sizeof(cases) / sizeof(cases[0]); index++) { + struct test_case test = cases[index]; + for (size_t mi = 0; mi < sizeof(masks) / sizeof(masks[0]); mi++) { + uint64_t mask = masks[mi]; + const void *pointer = memory; + if ((mask & ((UINT64_C(1) << test.lanes) - 1)) == 0) pointer = NULL; + if (mask == 1) { + memcpy(guard + page - 3, memory, 2); + pointer = guard + page - 3; + } + memset(output, 0xcc, sizeof(output)); + test.run(input, pointer, output, mask); + for (int lane = 0; lane < 32; lane++) { + uint16_t expected = 0; + if (lane < test.lanes && (mask & (UINT64_C(1) << lane))) { + double d = from_half(input[lane]); + double s = from_half(input[32 + lane]); + double m = from_half(memory[test.broadcast ? 0 : lane]); + double a = d, b = m, c = s; + if (test.order == 213) { a = s; b = d; c = m; } + if (test.order == 231) { a = s; b = m; c = d; } + if (test.mode == 2 || test.mode == 3) a = -a; + if (test.mode == 1 || test.mode == 3 || + (test.mode == 4 && lane % 2 == 0) || + (test.mode == 5 && lane % 2 != 0)) c = -c; + expected = to_half(a * b + c); + } else if (lane < test.lanes && !test.zero) { + expected = input[lane]; + } else if (test.scalar && lane < 8 && lane != 0) { + expected = input[lane]; + } + if (output[lane] != expected) { + fprintf(stderr, "case %zu order %d mode %d mask %llx lane %d: %04x != %04x\n", + index, test.order, test.mode, (unsigned long long)mask, lane, output[lane], expected); + return 2; + } + } + } + } + return free_guard(); +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_fma_portable", ir, mainC) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + const triple = "x86_64-apple-macosx" + x86IR, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "fp16_fma_portable_x86", triple, x86IR, mainC, []string{"/usr/bin/arch", "-x86_64"}) + } +} diff --git a/amd64_raw_fp16_fma_test.go b/amd64_raw_fp16_fma_test.go new file mode 100644 index 000000000..12057f0e2 --- /dev/null +++ b/amd64_raw_fp16_fma_test.go @@ -0,0 +1,266 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Intel AVX512-FP16 specification, sections 5.44-5.49: MAP6.66.W0, +// packed X/Y/Z and scalar LLIG, with the same opcode/order axes as FMA3. +// Go 1.27 has no named PH/SH FMA forms, so these are raw encodings only. +func rawFP16FMAOps() map[byte]Op { + ops := make(map[byte]Op) + for orderIndex, order := range []int{132, 213, 231} { + for familyIndex, family := range []string{"VFMADD", "VFMSUB", "VFNMADD", "VFNMSUB"} { + for scalarIndex, suffix := range []string{"PH", "SH"} { + opcode := byte(0x98 + orderIndex*16 + familyIndex*2 + scalarIndex) + ops[opcode] = Op(fmt.Sprintf("%s%d%s", family, order, suffix)) + } + } + ops[byte(0x96+orderIndex*16)] = Op(fmt.Sprintf("VFMADDSUB%dPH", order)) + ops[byte(0x97+orderIndex*16)] = Op(fmt.Sprintf("VFMSUBADD%dPH", order)) + } + return ops +} + +func encodeRawFP16FMA(opcode, length byte, source1, source2, destination, mask int, memory, control, zero bool) []byte { + code := encodeX86EVEXFMA3(opcode, false, length, control, zero, mask, destination, source1, source2) + code[1] = code[1]&^byte(15) | 6 + if memory { + // Keep register extension tests independent of memory base extensions. + code[1] |= 0x60 + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestDecodeRawFP16FMAGoATReproduction(t *testing.T) { + for _, tc := range []struct { + code []byte + op Op + }{ + {[]byte{0x62, 0xb6, 0x7d, 0x48, 0xa8, 0x24, 0x4a}, "VFMADD213PH"}, + {[]byte{0x62, 0xf6, 0x7d, 0x48, 0x98, 0x0e}, "VFMADD132PH"}, + {[]byte{0x62, 0xb6, 0x7d, 0x08, 0x99, 0x0c, 0x76}, "VFMADD132SH"}, + } { + got, err := decodeX86RawDirectiveGroup(tc.code, 64, 0, "GoAT FP16 FMA", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Op != tc.op || !got[0].x86Encoded { + t.Fatalf("%x decoded as %+v, want raw %s", tc.code, got, tc.op) + } + } +} + +func TestDecodeRawFP16FMACompleteFormats(t *testing.T) { + for opcode, base := range rawFP16FMAOps() { + scalar := strings.HasSuffix(string(base), "SH") + for length := byte(0); length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if scalar && memory && control || !scalar && length == 3 && (!control || memory) { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := encodeRawFP16FMA(opcode, length, 20, 21, 22, mask, memory, control, zero) + got, size, ok, err := decodedX86VEXFMA3Instruction(code, 64) + if err != nil || !ok || size != len(code) { + t.Fatalf("%x: got %+v, size=%d, ok=%v, err=%v", code, got, size, ok, err) + } + want := base + prefix, width := "X", 16 + if !scalar { + vl := length + if control && !memory { + vl = 2 + } + prefix, width = [...]string{"X", "Y", "Z"}[vl], 16<>6 != 3 { + return fail("scalar conversion does not support memory broadcast") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register or address in 32-bit mode") + } + spec := amd64HalfConversionSpecs[string(form.op)] + source, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", spec.inputBits/8) + if err != nil { + return Instr{}, 0, true, err + } + op := form.op + if p.broadcast { + if spec.inputBits == 16 { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } + if p.zero { + op += ".Z" + } + args := []Operand{source, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + destination := int(code[p.modRM]>>3&7) + p.r*8 + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destination))}) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), x86Encoded: true}, p.modRM + consumed, true, nil +} + +func decodeX86RawScalarHalfConversionRIPData(code []byte, offset, mode int) (Instr, int, x86RawLiteralRange, bool, error) { + if mode != 64 || offset >= len(code) { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + p, recognized := decodeX86RawVectorEncoding(code[offset:]) + form, matched := x86RawScalarHalfConversionFormFor(p) + if !recognized || !matched || p.segment != "" || p.addressOverride { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + modRM := offset + p.modRM + if len(code) <= modRM || code[modRM]&0xc7 != 5 { + return Instr{}, 0, x86RawLiteralRange{}, false, nil + } + width := amd64HalfConversionSpecs[string(form.op)].inputBits / 8 + return x86RawRIPDataThroughDecoder(code, offset, mode, modRM, width, decodedX86ScalarHalfConversionInstruction, "scalar half conversion") +} diff --git a/amd64_raw_fp16_scalar_convert_runtime_test.go b/amd64_raw_fp16_scalar_convert_runtime_test.go new file mode 100644 index 000000000..1ed391047 --- /dev/null +++ b/amd64_raw_fp16_scalar_convert_runtime_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +func TestRawFP16ScalarConversionPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + for index, tc := range fp16ScalarConversionCases { + for _, zero := range []bool{false, true} { + name := fmt.Sprintf("scalarHalf_%d_%t", index, zero) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ input+0(FP), AX\nMOVQ vectors+8(FP), DI\nMOVQ out+16(FP), SI\nMOVQ mask+24(FP), CX\nKMOVQ CX, K1\nVMOVDQU64 (DI), Z0\nVMOVDQU64 64(DI), Z1\n") + code := rawFP16ScalarConversion(index, 3, 1, 0, 0, 1, true, false, zero) + code[len(code)-1] = 0 + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + bits := map[int]uint64{16: 0x4000, 32: 0x40000000, 64: 0x4000000000000000} + fmt.Fprintf(&declarations, "extern void %s(const void *, const void *, void *, uint64_t);\n", name) + inactive := "memcpy(expected, vectors, output_bytes);" + if zero { + inactive = "memset(expected, 0, output_bytes);" + } + fmt.Fprintf(&calls, ` + { + const int input_bytes = %d, output_bytes = %d; + uint64_t input = UINT64_C(%d), result = UINT64_C(%d); + const void *pointer = NULL; + if (mask & 1) { + memcpy(guard + page - input_bytes - 1, &input, input_bytes); + pointer = guard + page - input_bytes - 1; + } + memset(expected, 0, sizeof(expected)); + if (mask & 1) memcpy(expected, &result, output_bytes); + else { %s } + memcpy(expected + output_bytes, vectors + 64 + output_bytes, 16 - output_bytes); + %s(pointer, vectors, output, mask); + if (memcmp(expected, output, 64)) { + fprintf(stderr, "%s mask %%llx: incorrect low, preserved or upper lanes\n", (unsigned long long)mask); + return 2; + } + } +`, tc.inputBits/8, tc.outBits/8, bits[tc.inputBits], bits[tc.outBits], inactive, name, name) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: testTargetTriple(runtime.GOOS, runtime.GOARCH), Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := "#include \n#include \n#include \n#include \n" + declarations.String() + x86TestGuardPagesC + ` +int main(void) { + if (setup_guard()) return 1; + unsigned char vectors[128], output[64], expected[64]; + for (int i = 0; i < 128; i++) vectors[i] = (unsigned char)(0x20 + i); + const uint64_t masks[] = {0, 1, 2, UINT64_C(1) << 60, UINT64_MAX}; + for (int mi = 0; mi < 5; mi++) { + uint64_t mask = masks[mi]; +` + calls.String() + ` + } + return free_guard(); +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_scalar_conversion", ir, mainC) +} + +func TestRawFP16ScalarConversionRoundingRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + for _, index := range []int{1, 3} { + for rounding := 0; rounding < 4; rounding++ { + name := fmt.Sprintf("scalarRound_%d_%d", index, rounding) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-16\nMOVQ input+0(FP), AX\nMOVQ out+8(FP), SI\nVMOVDQU64 (AX), Z2\n", name) + for _, b := range rawFP16ScalarConversion(index, rounding, 1, 2, 0, 0, false, true, false) { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + } + fmt.Fprintf(&declarations, "extern void %s(const void *, void *);\n", name) + input := "single" + if index == 3 { + input = "wide" + } + fmt.Fprintf(&calls, ` + %s(%s, output); + if (output[0] != expected[%d][sample]) { + fprintf(stderr, "%s sample %%d: %%04x != %%04x\n", sample, output[0], expected[%d][sample]); + return 1; + } +`, name, input, rounding, name, rounding) + if index == 3 { + fmt.Fprintf(&calls, ` + // These exactly representable doubles straddle a half tie but round + // to that same tie in float; a double->float->half chain is wrong. + if (sample == 0) { + wide[0] = 1.0 + 0x1p-11 + 0x1p-40; + %s(wide, output); + const uint16_t above[] = {0x3c01, 0x3c00, 0x3c01, 0x3c00}; + if (output[0] != above[%d]) return 2; + wide[0] = 1.0 + 0x1p-11 - 0x1p-40; + %s(wide, output); + const uint16_t below[] = {0x3c00, 0x3c00, 0x3c01, 0x3c00}; + if (output[0] != below[%d]) return 3; + wide[0] = values[sample]; + } +`, name, rounding, name, rounding) + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: testTargetTriple(runtime.GOOS, runtime.GOARCH), Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := "#include \n#include \n" + declarations.String() + ` +int main(void) { + const double values[] = {1 + 0x1p-11, 1 + 3 * 0x1p-11, -1 - 0x1p-11, -1 - 3 * 0x1p-11, + 65536, -65536, 0x1p-25, -0x1p-25}; + const uint16_t expected[4][8] = { + {0x3c00, 0x3c02, 0xbc00, 0xbc02, 0x7c00, 0xfc00, 0, 0x8000}, + {0x3c00, 0x3c01, 0xbc01, 0xbc02, 0x7bff, 0xfc00, 0, 0x8001}, + {0x3c01, 0x3c02, 0xbc00, 0xbc01, 0x7c00, 0xfbff, 1, 0x8000}, + {0x3c00, 0x3c01, 0xbc00, 0xbc01, 0x7bff, 0xfbff, 0, 0x8000} + }; + uint16_t output[32]; + for (int sample = 0; sample < 8; sample++) { + double wide[8] = {values[sample]}; + float single[16] = {(float)values[sample]}; +` + calls.String() + ` + } + return 0; +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_scalar_rounding", ir, mainC) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + const triple = "x86_64-apple-macosx" + x86IR, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "fp16_scalar_rounding_x86", triple, x86IR, mainC, []string{"/usr/bin/arch", "-x86_64"}) + } +} diff --git a/amd64_raw_fp16_scalar_convert_test.go b/amd64_raw_fp16_scalar_convert_test.go new file mode 100644 index 000000000..ce615a5c2 --- /dev/null +++ b/amd64_raw_fp16_scalar_convert_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// Independent Intel AVX512-FP16 sections 5.18, 5.19, 5.21 and 5.24 rows. +var fp16ScalarConversionCases = []struct { + op Op + mapNumber, pp, code byte + inputBits, outBits int + w bool +}{ + {"VCVTSH2SS", 6, 0, 0x13, 16, 32, false}, + {"VCVTSS2SH", 5, 0, 0x1d, 32, 16, false}, + {"VCVTSH2SD", 5, 2, 0x5a, 16, 64, false}, + {"VCVTSD2SH", 5, 3, 0x5a, 64, 16, true}, +} + +func rawFP16ScalarConversion(index, length, source1, source2, destination, mask int, memory, control, zero bool) []byte { + tc := fp16ScalarConversionCases[index] + code := encodeX86EVEXFMA3(tc.code, tc.w, byte(length), control, zero, mask, destination, source1, source2) + code[1] = code[1]&0xf0 | tc.mapNumber + code[2] = code[2]&^3 | tc.pp + if memory { + code[1] |= 0x60 + code[5] = 0x40 | byte(destination&7)<<3 + code = append(code, 1) + } + return code +} + +func TestRawFP16ScalarConversionFormats(t *testing.T) { + for index, tc := range fp16ScalarConversionCases { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if control && memory { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := rawFP16ScalarConversion(index, length, 20, 21, 22, mask, memory, control, zero) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "FP16 scalar conversion", map[string]bool{}) + if err != nil || len(got) != 1 || !got[0].x86Encoded { + t.Fatalf("%x: %+v, err=%v", code, got, err) + } + want := tc.op + if control { + if tc.inputBits == 16 { + want += ".SAE" + } else { + want += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[length] + } + } + if zero { + want += ".Z" + } + source := "X21" + if memory { + source = fmt.Sprintf("%d(AX)", tc.inputBits/8) + } + ins := got[0] + if ins.Op != want || ins.Args[0].String() != source || ins.Args[1].String() != "X20" || ins.Args[len(ins.Args)-1].String() != "X22" { + t.Fatalf("%x: %+v, want %s %s, X20, [mask], X22", code, ins, want, source) + } + } + } + } + } + } + } +} + +func TestRawFP16ScalarConversionInvalidAndNamedForms(t *testing.T) { + for index, tc := range fp16ScalarConversionCases { + for _, change := range []struct { + index int + bits byte + }{{2, 4}, {2, 0x80}, {3, 0x80}} { + code := rawFP16ScalarConversion(index, 0, 1, 2, 3, 0, false, false, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid scalar conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved %x", code) + } + } + code := rawFP16ScalarConversion(index, 0, 1, 2, 3, 0, true, true, false) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid scalar broadcast", map[string]bool{}); err == nil { + t.Fatalf("accepted scalar broadcast %x", code) + } + for operand := 0; operand < 3; operand++ { + registers := [3]int{1, 2, 3} + registers[operand] = 8 + code := rawFP16ScalarConversion(index, 0, registers[0], registers[1], registers[2], 1, false, false, false) + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 high scalar conversion", map[string]bool{}); err == nil { + t.Fatalf("accepted 386 extended register %x", code) + } + } + source := "TEXT namedScalarHalf(SB),4,$0-0\n" + string(tc.op) + " X0, X1, X2\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedScalarHalf": {Name: "namedScalarHalf", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted named %s", tc.op) + } + } +} + +func TestRawFP16ScalarConversionLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT rawScalarHalf(SB),4,$0-0\n") + for index := range fp16ScalarConversionCases { + for length := 0; length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, mask := range []int{0, 1} { + for _, b := range rawFP16ScalarConversion(index, length, 1, 2, 3, mask, memory, !memory, mask != 0) { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + } + } + } + source.WriteString("RET\n") + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawScalarHalf": {Name: "rawScalarHalf", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-scalar-half.ll", "raw-scalar-half.o", ir) + }) + } +} + +func TestRawFP16ScalarConversionRIPLiteral(t *testing.T) { + for index, tc := range fp16ScalarConversionCases { + code := rawFP16ScalarConversion(index, 3, 1, 0, 0, 1, false, false, true) + code[5] = 5 + code = append(code, 1, 0, 0, 0, 0xc3) + code = append(code, make([]byte, tc.inputBits/8)...) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "scalar FP16 literal", map[string]bool{}) + if err != nil || len(got) != 2 || !got[0].x86RIPLiteral || got[0].Args[0].Kind != OpSym { + t.Fatalf("%x: %+v, err=%v", code, got, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated scalar FP16 literal", map[string]bool{}); err == nil { + t.Fatal("accepted truncated constant pool") + } + } +} diff --git a/amd64_raw_fp16_scalar_move_test.go b/amd64_raw_fp16_scalar_move_test.go new file mode 100644 index 000000000..cf5e2bcb9 --- /dev/null +++ b/amd64_raw_fp16_scalar_move_test.go @@ -0,0 +1,211 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeRawFP16ScalarMove(opcode, low, upper, dst, mask int, zero bool, length byte) []byte { + code := encodeRawScalarMove(true, 2, opcode, low, upper, dst, mask, zero) + code[1] = code[1]&^byte(15) | 5 + code[3] = code[3]&^byte(0x60) | length<<5 + return code +} + +func TestDecodeRawFP16ScalarMoveGoATReproduction(t *testing.T) { + for _, code := range [][]byte{ + // VMOVSH (DI)(CX*2), X0 and VMOVSH X0, (DX)(CX*2) + // from go-highway's GoAT output. + {0x62, 0xf5, 0x7e, 0x08, 0x10, 0x04, 0x4f}, + {0x62, 0xf5, 0x7e, 0x08, 0x11, 0x04, 0x4a}, + // A complete add/store group, not just a standalone move. + {0x62, 0xf5, 0x7e, 0x08, 0x58, 0x04, 0x4e, + 0x62, 0xf5, 0x7e, 0x08, 0x11, 0x04, 0x4a}, + } { + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "GoAT VMOVSH", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + move := decoded[len(decoded)-1] + if move.Op != "VMOVSH" || !move.x86Encoded { + t.Fatalf("decoded %#v, want final raw VMOVSH", decoded) + } + } +} + +func TestDecodeRawFP16ScalarMoveCompleteForms(t *testing.T) { + for _, opcode := range []int{0x10, 0x11} { + for length := byte(0); length < 4; length++ { + for _, registers := range [][3]int{{0, 1, 2}, {7, 8, 15}, {16, 23, 31}} { + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if mask == 0 && zero { + continue + } + low, upper, dst := registers[0], registers[1], registers[2] + code := encodeRawFP16ScalarMove(opcode, low, upper, dst, mask, zero, length) + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + wantOp := Op("VMOVSH") + if zero { + wantOp += ".Z" + } + want := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", low))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", upper))}, + } + if mask != 0 { + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}) + if err != nil || !ok || size != len(code) || got.Op != wantOp || + !got.x86Encoded || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("%x: got %+v, size=%d, ok=%v, err=%v; want %s %+v", code, got, size, ok, err, wantOp, want) + } + } + } + } + } + } + for _, opcode := range []byte{0x10, 0x11} { + for length := byte(0); length < 4; length++ { + for _, mask := range []byte{0, 1, 7} { + for _, zero := range []bool{false, true} { + if opcode == 0x11 && zero || mask == 0 && zero { + continue + } + code := []byte{0x62, 0xf5, 0x7e, 0x08 | length<<5 | mask, opcode, 0x04, 0x4f} + if zero { + code[3] |= 0x80 + } + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + if err != nil || !ok || size != len(code) || got.Op != "VMOVSH" && got.Op != "VMOVSH.Z" { + t.Fatalf("%x: got %+v, size=%d, ok=%v, err=%v", code, got, size, ok, err) + } + if opcode == 0x10 && (got.Args[0].Kind != OpMem || got.Args[len(got.Args)-1].Reg != "X0") { + t.Fatalf("%x: wrong memory load %+v", code, got) + } + if opcode == 0x11 && (got.Args[0].Reg != "X0" || got.Args[len(got.Args)-1].Kind != OpMem) { + t.Fatalf("%x: wrong memory store %+v", code, got) + } + } + } + } + } +} + +func TestDecodeRawFP16ScalarMoveInvalidForms(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"missing-modrm", []byte{0x62, 0xf5, 0x7e, 0x08, 0x10}, 64}, + {"address-override", []byte{0x67, 0x62, 0xf5, 0x7e, 0x08, 0x10, 0xc0}, 64}, + {"wrong-width", []byte{0x62, 0xf5, 0xfe, 0x08, 0x10, 0xc0}, 64}, + {"missing-fixed", []byte{0x62, 0xf5, 0x7a, 0x08, 0x10, 0xc0}, 64}, + {"broadcast", []byte{0x62, 0xf5, 0x7e, 0x18, 0x10, 0xc0}, 64}, + {"zero-k0", []byte{0x62, 0xf5, 0x7e, 0x88, 0x10, 0xc0}, 64}, + {"zero-store", []byte{0x62, 0xf5, 0x7e, 0x89, 0x11, 0x00}, 64}, + {"memory-vvvv", []byte{0x62, 0xf5, 0x76, 0x08, 0x10, 0x00}, 64}, + {"memory-vprime", []byte{0x62, 0xf5, 0x7e, 0x00, 0x10, 0x00}, 64}, + {"386-high-register", []byte{0x62, 0x75, 0x7e, 0x08, 0x10, 0xc0}, 32}, + {"wrong-mode", []byte{0x62, 0xf5, 0x7e, 0x08, 0x10, 0xc0}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, ok, err := decodedX86ScalarMoveInstruction(tc.code, tc.mode); !ok || err == nil { + t.Fatalf("invalid %x: matched=%v, err=%v", tc.code, ok, err) + } + }) + } +} + +func TestDecodeRawFP16ScalarMoveRIPLiteral(t *testing.T) { + code := []byte{0x62, 0xf5, 0x7e, 0x08, 0x10, 0x05, 1, 0, 0, 0, 0xc3, 0x00, 0x3c} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VMOVSH RIP literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VMOVSH" || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want source-local FP16 literal", code, decoded) + } +} + +func TestTranslateRawFP16ScalarMoveLLVM22Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16ScalarMove(SB),4,$0-0\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7e, 0x08, 0x10, 0x04, 0x4f}, + {0x62, 0xf5, 0x7e, 0x08, 0x11, 0x04, 0x4f}, + encodeRawFP16ScalarMove(0x10, 1, 2, 3, 7, true, 3), + encodeRawFP16ScalarMove(0x11, 4, 5, 6, 1, false, 1), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawFP16ScalarMove": {Name: "rawFP16ScalarMove", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "load i16") || !strings.Contains(ir, "store i16") { + t.Fatalf("missing scalar FP16 memory operation:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "raw-fp16-scalar-move.ll", "raw-fp16-scalar-move.o", ir) + }) + } +} + +func TestTranslateRawFP16ScalarMove386LLVM22Targets(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawFP16ScalarMove386(SB),4,$0-0\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7e, 0x08, 0x10, 0x04, 0x4f}, + {0x62, 0xf5, 0x7e, 0x09, 0x11, 0x04, 0x4f}, + encodeRawFP16ScalarMove(0x10, 1, 2, 3, 1, true, 2), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "386", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawFP16ScalarMove386": {Name: "rawFP16ScalarMove386", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-fp16-scalar-move386.ll", "raw-fp16-scalar-move386.o", ir) + }) + } +} diff --git a/amd64_raw_fp16_sqrt_test.go b/amd64_raw_fp16_sqrt_test.go new file mode 100644 index 000000000..716552b89 --- /dev/null +++ b/amd64_raw_fp16_sqrt_test.go @@ -0,0 +1,246 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +// Intel AVX512-FP16 sections 5.74/5.75. Packed SQRT reserves vvvv; +// scalar SQRT instead uses it as the source of preserved upper X lanes. +func rawFP16PackedSqrt(length byte, source, destination, mask int, memory, control, zero bool) []byte { + code := encodeRawFP16Binary(0x51, length, 0, source, destination, mask, false, control, zero) + if memory { + code[1] |= 0x60 + code[5] = byte(0x40 | (destination&7)<<3) + code = append(code, 1) + } + return code +} + +func TestRawFP16PackedSqrtRejectsNamedForms(t *testing.T) { + const source = "TEXT namedHalfSqrt(SB),4,$0-0\nVSQRTPH X0, X1\nRET\n" + requireX86GoAssemblerResult(t, "amd64", source, false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedHalfSqrt": {Name: "namedHalfSqrt", Ret: Void}}, + }); err == nil { + t.Fatal("accepted VSQRTPH absent from Go's named assembler table") + } +} + +func TestRawFP16SqrtPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + for _, scalar := range []bool{false, true} { + for _, zero := range []bool{false, true} { + name := fmt.Sprintf("halfSqrt_%t_%t", scalar, zero) + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\n", name) + source.WriteString("MOVQ input+0(FP), DI\nMOVQ memory+8(FP), AX\nMOVQ out+16(FP), SI\nMOVQ mask+24(FP), CX\n") + source.WriteString("KMOVQ CX, K1\nVMOVDQU64 (DI), Z0\nVMOVDQU64 64(DI), Z1\n") + code := rawFP16PackedSqrt(0, 0, 0, 1, true, false, zero) + if scalar { + code = encodeRawFP16ScalarBinary(0x51, 0, 1, 0, 0, 1, true, false, zero) + } + code[len(code)-1] = 0 + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + source.WriteString("VMOVDQU64 Z0, (SI)\nRET\n") + sigs[name] = FuncSig{ + Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}, + } + lanes, fallback := 8, "input[lane]" + if scalar { + lanes = 1 + } + if zero { + fallback = "0" + } + upper := "0" + if scalar { + upper = "lane < 8 ? input[32 + lane] : 0" + } + fmt.Fprintf(&declarations, "extern void %s(const void *, const void *, void *, uint64_t);\n", name) + fmt.Fprintf(&calls, ` + %s(input, pointer, output, mask); + for (int lane = 0; lane < 32; lane++) { + uint16_t expected = lane < %d ? ((mask & (UINT64_C(1) << lane)) ? roots[lane] : %s) : (%s); + if (output[lane] != expected) { + fprintf(stderr, "%s mask %%llx lane %%d: %%04x != %%04x\n", + (unsigned long long)mask, lane, output[lane], expected); + return 2; + } + } +`, name, lanes, fallback, upper, name) + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: testTargetTriple(runtime.GOOS, runtime.GOARCH), Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := "#include \n#include \n#include \n#include \n" + declarations.String() + x86TestGuardPagesC + ` +int main(void) { + if (setup_guard()) return 1; + // Independent bit-pattern oracle: sqrt(0, -0, 1, 4, 9, 16, 2^-24, +inf). + const uint16_t memory[] = {0, 0x8000, 0x3c00, 0x4400, 0x4880, 0x4c00, 1, 0x7c00}; + const uint16_t roots[] = {0, 0x8000, 0x3c00, 0x4000, 0x4200, 0x4400, 0x0c00, 0x7c00}; + const uint64_t masks[] = {0, 1, 0xff, UINT64_C(1) << 60}; + uint16_t input[64], output[32]; + for (int lane = 0; lane < 64; lane++) input[lane] = (uint16_t)(0x1000 + lane); + for (int mi = 0; mi < 4; mi++) { + uint64_t mask = masks[mi]; + const void *pointer = memory; + if ((mask & 0xff) == 0) pointer = NULL; + if (mask == 1) { + memcpy(guard + page - 3, memory, 2); + pointer = guard + page - 3; + } +` + calls.String() + ` + } + return free_guard(); +} +` + compileAndRunRuntimeTest(t, llc, clang, "fp16_sqrt_portable", ir, mainC) +} + +func TestDecodeRawFP16PackedSqrtFormats(t *testing.T) { + for length := byte(0); length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if length == 3 && (!control || memory) { + continue + } + for _, mask := range []int{0, 1, 7} { + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := rawFP16PackedSqrt(length, 20, 21, mask, memory, control, zero) + got, size, ok, err := decodedX86SameWidthConversionInstruction(code, 64) + if err != nil || !ok || size != len(code) { + t.Fatalf("%x: got %+v, size=%d, ok=%v, err=%v", code, got, size, ok, err) + } + want := Op("VSQRTPH") + vl := length + if control { + if memory { + want += ".BCST" + } else { + want += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[length] + vl = 2 + } + } + if zero { + want += ".Z" + } + prefix := [...]string{"X", "Y", "Z"}[vl] + if got.Op != want || !got.x86Encoded || got.Args[len(got.Args)-1].String() != prefix+"21" { + t.Fatalf("%x: got %+v, want raw %s with %s21", code, got, want, prefix) + } + wantSource := prefix + "20" + if memory { + width := 16 << vl + if control { + width = 2 + } + wantSource = fmt.Sprintf("%d(AX)", width) + } + if got.Args[0].String() != wantSource { + t.Fatalf("%x: source %s, want %s", code, got.Args[0], wantSource) + } + } + } + } + } + } +} + +func TestDecodeRawFP16PackedSqrtInvalidForms(t *testing.T) { + for _, tc := range []struct { + index int + bits byte + }{ + {2, 0x80}, // W1. + {2, 0x04}, // Fixed bit cleared. + {2, 0x08}, // Reserved vvvv. + {3, 0x08}, // Reserved V'. + {3, 0x80}, // Zeroing with K0. + {3, 0x60}, // Reserved packed length. + } { + code := rawFP16PackedSqrt(0, 1, 2, 0, false, false, false) + code[tc.index] ^= tc.bits + if _, _, ok, err := decodedX86SameWidthConversionInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted invalid %x: matched=%v, err=%v", code, ok, err) + } + } +} + +func TestTranslateRawFP16PackedSqrtLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT rawHalfSqrt(SB),4,$0-0\n") + for length := byte(0); length < 4; length++ { + for _, memory := range []bool{false, true} { + for _, control := range []bool{false, true} { + if length == 3 && (!control || memory) { + continue + } + code := rawFP16PackedSqrt(length, 1, 2, 1, memory, control, true) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + } + } + source.WriteString("RET\n") + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawHalfSqrt": {Name: "rawHalfSqrt", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"@llvm.sqrt.v8f16", "@llvm.sqrt.v16f16", "@llvm.sqrt.v32f16", + "@llvm.experimental.constrained.sqrt.v32f16", "@llvm.masked.load.v8i16", "phi i16"} { + if !strings.Contains(ir, want) { + t.Fatalf("missing %s", want) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-half-sqrt.ll", "raw-half-sqrt.o", ir) + }) + } +} diff --git a/amd64_raw_funnel_shift_test.go b/amd64_raw_funnel_shift_test.go new file mode 100644 index 000000000..b938de215 --- /dev/null +++ b/amd64_raw_funnel_shift_test.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawDirectiveGroupFunnelShiftCompleteFamily(t *testing.T) { + // Bytes are Go 1.27 assembler oracle cases spanning immediate/variable, + // left/right, W/D/Q, and high EVEX registers. + for _, test := range []struct { + name string + code []byte + want string + }{ + {name: "left immediate w", code: []byte{0x62, 0x13, 0xc5, 0x0b, 0x70, 0xd8, 0x41}, want: "VPSHLDW $65, X24, X7, K3, X11"}, + {name: "left immediate d", code: []byte{0x62, 0x43, 0x05, 0x04, 0x71, 0xd0, 0x2f}, want: "VPSHLDD $47, X8, X31, K4, X26"}, + {name: "left immediate q", code: []byte{0x62, 0x33, 0xd5, 0x04, 0x71, 0xfe, 0x5e}, want: "VPSHLDQ $94, X22, X21, K4, X15"}, + {name: "right immediate w", code: []byte{0x62, 0xc3, 0xfd, 0x09, 0x72, 0xf7, 0x1b}, want: "VPSHRDW $27, X15, X0, K1, X22"}, + {name: "right immediate q", code: []byte{0x62, 0x53, 0xfd, 0x03, 0x73, 0xee, 0x2a}, want: "VPSHRDQ $42, X14, X16, K3, X13"}, + {name: "left variable w", code: []byte{0x62, 0x32, 0xb5, 0x0f, 0x70, 0xea}, want: "VPSHLDVW X18, X9, K7, X13"}, + {name: "left variable d", code: []byte{0x62, 0xd2, 0x75, 0x0f, 0x71, 0xff}, want: "VPSHLDVD X15, X1, K7, X7"}, + {name: "left variable q", code: []byte{0x62, 0x72, 0xf5, 0x07, 0x71, 0xe3}, want: "VPSHLDVQ X3, X17, K7, X12"}, + {name: "right variable w", code: []byte{0x62, 0xf2, 0xed, 0x0f, 0x72, 0xc2}, want: "VPSHRDVW X2, X2, K7, X0"}, + {name: "right variable d", code: []byte{0x62, 0x32, 0x1d, 0x0a, 0x73, 0xc7}, want: "VPSHRDVD X23, X12, K2, X8"}, + {name: "right variable q", code: []byte{0x62, 0x22, 0xa5, 0x09, 0x73, 0xc4}, want: "VPSHRDVQ X20, X11, K1, X24"}, + {name: "left variable d broadcast zero", code: []byte{0x62, 0xf2, 0x75, 0xdb, 0x71, 0x50, 0x7f}, want: "VPSHLDVD.BCST.Z 508(AX), Z1, K3, Z2"}, + {name: "right immediate q broadcast zero", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x73, 0x50, 0x7f, 0x09}, want: "VPSHRDQ.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } { + t.Run(test.name, func(t *testing.T) { + decoded, err := decodeX86RawDirectiveGroup(test.code, 64, 0, test.name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, test.want+" ") { + t.Fatalf("decoded %x as %#v, want %q", test.code, decoded, test.want) + } + }) + } +} + +func TestDecodedX86RawFunnelShiftAllRegisterFields(t *testing.T) { + for _, operation := range []struct { + op Op + opcode byte + mapValue byte + w byte + variable bool + }{ + {op: "VPSHLDW", opcode: 0x70, mapValue: 3, w: 1}, + {op: "VPSHLDD", opcode: 0x71, mapValue: 3}, + {op: "VPSHLDQ", opcode: 0x71, mapValue: 3, w: 1}, + {op: "VPSHRDW", opcode: 0x72, mapValue: 3, w: 1}, + {op: "VPSHRDD", opcode: 0x73, mapValue: 3}, + {op: "VPSHRDQ", opcode: 0x73, mapValue: 3, w: 1}, + {op: "VPSHLDVW", opcode: 0x70, mapValue: 2, w: 1, variable: true}, + {op: "VPSHLDVD", opcode: 0x71, mapValue: 2, variable: true}, + {op: "VPSHLDVQ", opcode: 0x71, mapValue: 2, w: 1, variable: true}, + {op: "VPSHRDVW", opcode: 0x72, mapValue: 2, w: 1, variable: true}, + {op: "VPSHRDVD", opcode: 0x73, mapValue: 2, variable: true}, + {op: "VPSHRDVQ", opcode: 0x73, mapValue: 2, w: 1, variable: true}, + } { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | operation.mapValue + p1 := operation.w<<7 | byte(^second&15)<<3 | 5 + p2 := byte(vectorBits)<<5 | byte((^second>>4)&1)<<3 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, operation.opcode, modRM} + if !operation.variable { + encoding = append(encoding, 0xa5) + } + instruction, length, ok, err := decodedX86FunnelShiftInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("op=%s width=%s first=%d second=%d destination=%d: ok=%v length=%d err=%v", operation.op, vectorName, first, second, destination, ok, length, err) + } + immediate := "" + if !operation.variable { + immediate = "$165, " + } + want := fmt.Sprintf("%s %s%s%d, %s%d, %s%d", operation.op, immediate, vectorName, first, vectorName, second, vectorName, destination) + if instruction.Raw != want { + t.Fatalf("encoding %x decoded as %q, want %q", encoding, instruction.Raw, want) + } + } + } + } + } + } +} + +func TestDecodedX86RawFunnelShiftRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 immediate operand limit", mode: 32, encoding: []byte{0x62, 0xf3, 0xf5, 0x08, 0x73, 0xc2, 1}, wantOK: true}, + {name: "386 variable masked operand limit", mode: 32, encoding: []byte{0x62, 0xf2, 0xf5, 0x09, 0x73, 0xc2}, wantOK: true}, + {name: "386 variable extended register", mode: 32, encoding: []byte{0x62, 0x72, 0xf5, 0x08, 0x73, 0xc2}, wantOK: true}, + {name: "reserved vector length", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x68, 0x73, 0xc2}, wantOK: true}, + {name: "word broadcast", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x18, 0x72, 0x10}, wantOK: true}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x18, 0x73, 0xc2}, wantOK: true}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x88, 0x73, 0xc2}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0x62, 0xf2, 0xf5, 0x08, 0x73, 0xc2}, wantOK: true}, + {name: "wrong map", mode: 64, encoding: []byte{0x62, 0xf1, 0xf5, 0x08, 0x73, 0xc2}, wantOK: false}, + {name: "wrong prefix", mode: 64, encoding: []byte{0x62, 0xf2, 0xf4, 0x08, 0x73, 0xc2}, wantOK: false}, + {name: "word wrong w", mode: 64, encoding: []byte{0x62, 0xf2, 0x75, 0x08, 0x72, 0xc2}, wantOK: true}, + {name: "truncated immediate", mode: 64, encoding: []byte{0x62, 0xf3, 0xf5, 0x08, 0x73, 0xc2}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86FunnelShiftInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x: ok=%v, want %v (err=%v)", test.encoding, ok, test.wantOK, err) + } + if test.wantOK && err == nil { + t.Fatalf("encoding %x was recognized without rejection", test.encoding) + } + }) + } +} diff --git a/amd64_raw_gather.go b/amd64_raw_gather.go new file mode 100644 index 000000000..f1edd36cc --- /dev/null +++ b/amd64_raw_gather.go @@ -0,0 +1,179 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// The eight data-gather spellings share the same opcode axes in Go's +// _yvgatherdps, _yvgatherdpd and _yvgatherqps tables. The ModRM/SIB operand +// is VSIB memory: its index is a vector register, never a GP register. +var x86GatherOps = [4][2]Op{ + {"VPGATHERDD", "VPGATHERDQ"}, + {"VPGATHERQD", "VPGATHERQQ"}, + {"VGATHERDPS", "VGATHERDPD"}, + {"VGATHERQPS", "VGATHERQPD"}, +} + +// The four packed and four floating-point scatter spellings mirror the EVEX +// gather VSIB axes, but encode the data vector in ModRM.reg and store to VSIB. +var x86ScatterOps = [4][2]Op{ + {"VPSCATTERDD", "VPSCATTERDQ"}, + {"VPSCATTERQD", "VPSCATTERQQ"}, + {"VSCATTERDPS", "VSCATTERDPD"}, + {"VSCATTERQPS", "VSCATTERQPD"}, +} + +func decodedX86RawScatterInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || !p.evex || p.mapNumber != 2 || p.pp != 1 || + p.opcode < 0xa0 || p.opcode > 0xa3 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw scatter: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.vectorLength > 2 || p.mask == 0 || p.zero || + p.broadcast || p.upper&15 != 0 { + return fail("invalid EVEX fixed, length, mask, zeroing, broadcast or vvvv bits") + } + if len(code) < p.modRM+2 { + return fail("missing ModRM or VSIB byte") + } + modRM := code[p.modRM] + if modRM>>6 == 3 || modRM&7 != 4 { + return fail("scatter requires VSIB memory") + } + w := 0 + if p.w { + w = 1 + } + op := x86ScatterOps[p.opcode-0xa0][w] + spec := amd64ScatterSpecs[string(op)] + widths := [...]string{"X", "Y", "Z"} + dataWidth := widths[p.vectorLength] + indexWidth := dataWidth + switch spec.table { + case amd64GatherDPD: + if p.vectorLength > 0 { + indexWidth = widths[p.vectorLength-1] + } + case amd64GatherQPS: + if p.vectorLength > 0 { + dataWidth = widths[p.vectorLength-1] + } + } + dataNumber := int(modRM>>3&7) + p.r*8 + indexNumber := int(code[p.modRM+1]>>3&7) + p.x*8 + p.upper&16 + if mode == 32 && (indexNumber >= 8 || p.b != 0 || + dataWidth == "Z" && dataNumber >= 8) { + return fail("extended register unavailable in 32-bit mode") + } + memory, consumed, err := decodedX86RMSource( + code[p.modRM:], mode, p.b, p.x, p.segment, spec.elemBits/8, + ) + if err != nil { + return Instr{}, 0, true, err + } + memory.Mem.Index = Reg(fmt.Sprintf("%s%d", indexWidth, indexNumber)) + memory.Mem.Scale = int64(1 << (code[p.modRM+1] >> 6)) + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", dataWidth, dataNumber))} + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))} + args := []Operand{data, mask, memory} + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s, %s", op, data.String(), mask.String(), memory.String()), + }, p.modRM + consumed, true, nil +} + +func decodedX86RawGatherInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || p.mapNumber != 2 || p.pp != 1 || p.opcode < 0x90 || p.opcode > 0x93 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw gather: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.evex && (!p.fixed || p.vectorLength > 2 || p.mask == 0 || p.zero || p.broadcast || p.upper&15 != 0) { + return fail("invalid EVEX fixed, length, mask, zeroing, broadcast or vvvv bits") + } + if len(code) < p.modRM+2 { + return fail("missing ModRM or VSIB byte") + } + modRM := code[p.modRM] + if modRM>>6 == 3 || modRM&7 != 4 { + return fail("gather requires VSIB memory") + } + + w := 0 + if p.w { + w = 1 + } + op := x86GatherOps[p.opcode-0x90][w] + spec := amd64GatherSpecs[op] + widths := [...]string{"X", "Y", "Z"} + destinationWidth := widths[p.vectorLength] + indexWidth := destinationWidth + switch spec.table { + case amd64GatherDPD: + if p.vectorLength > 0 { + indexWidth = widths[p.vectorLength-1] + } + case amd64GatherQPS: + if p.vectorLength > 0 { + destinationWidth = widths[p.vectorLength-1] + } + } + destinationNumber := int(modRM>>3&7) + p.r*8 + indexNumber := int(code[p.modRM+1]>>3&7) + p.x*8 + if p.evex { + indexNumber += p.upper & 16 + } + if mode == 32 && (indexNumber >= 8 || p.b != 0 || + !p.evex && (destinationNumber >= 8 || p.upper >= 8) || + p.evex && destinationWidth == "Z" && destinationNumber >= 8) { + return fail("extended register unavailable in 32-bit mode") + } + + disp8Scale := 1 + if p.evex { + disp8Scale = spec.elemBits / 8 + } + memory, consumed, err := decodedX86RMSource(code[p.modRM:], mode, p.b, p.x, p.segment, disp8Scale) + if err != nil { + return Instr{}, 0, true, err + } + memory.Mem.Index = Reg(fmt.Sprintf("%s%d", indexWidth, indexNumber)) + memory.Mem.Scale = int64(1 << (code[p.modRM+1] >> 6)) + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationWidth, destinationNumber))} + var args []Operand + if p.evex { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))} + args = []Operand{memory, mask, destination} + } else { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationWidth, p.upper))} + args = []Operand{mask, memory, destination} + } + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_gather_test.go b/amd64_raw_gather_test.go new file mode 100644 index 000000000..2be129880 --- /dev/null +++ b/amd64_raw_gather_test.go @@ -0,0 +1,220 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawVEXGatherCorpusEncoding(t *testing.T) { + const source = "TEXT rawgather(SB),4,$0-0\n\tVPGATHERDD Y2, (R8)(Y0*2), Y1\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } +} + +func TestX86RawVEXGatherGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT gatherforms(SB),4,$0-0\n") + for _, variants := range x86GatherOps { + for _, op := range variants { + for width := 0; width < 2; width++ { + destinationWidth, indexWidth := "X", "X" + if width == 1 { + switch amd64GatherSpecs[op].table { + case amd64GatherDPS: + destinationWidth, indexWidth = "Y", "Y" + case amd64GatherDPD: + destinationWidth = "Y" + case amd64GatherQPS: + indexWidth = "Y" + } + } + fmt.Fprintf(&source, "%s %s1, 4(BX)(%s2*2), %s3\n", op, destinationWidth, indexWidth, destinationWidth) + fmt.Fprintf(&source, "%s %s1, 256(%s4*8), %s3\n", op, destinationWidth, indexWidth, destinationWidth) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s14, -32(R8)(%s13*4), %s15\n", op, destinationWidth, indexWidth, destinationWidth) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + }) + } +} + +func TestX86RawEVEXGatherGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT evexgatherforms(SB),4,$0-0\n") + for _, variants := range x86GatherOps { + for _, op := range variants { + for width := 0; width < 3; width++ { + widths := [...]string{"X", "Y", "Z"} + destinationWidth := widths[width] + indexWidth := destinationWidth + switch amd64GatherSpecs[op].table { + case amd64GatherDPD: + if width > 0 { + indexWidth = widths[width-1] + } + case amd64GatherQPS: + if width > 0 { + destinationWidth = widths[width-1] + } + } + fmt.Fprintf(&source, "%s 8(BX)(%s2*4), K1, %s3\n", op, indexWidth, destinationWidth) + if arch == "amd64" { + fmt.Fprintf(&source, "%s -32(R8)(%s18*2), K7, %s21\n", op, indexWidth, destinationWidth) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + }) + } +} + +func TestX86RawVEXGatherRejectsMalformedForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {"missing SIB", []byte{0xc4, 0xc2, 0x6d, 0x90, 0x0c}, 64}, + {"register source", []byte{0xc4, 0xc2, 0x6d, 0x90, 0xc0, 0x40}, 64}, + {"GP index", []byte{0xc4, 0xc2, 0x6d, 0x90, 0x08, 0x40}, 64}, + {"missing displacement", []byte{0xc4, 0xc2, 0x6d, 0x90, 0x4c, 0x40}, 64}, + {"address override", []byte{0x67, 0xc4, 0xc2, 0x6d, 0x90, 0x0c, 0x40}, 64}, + {"high VSIB index on 386", []byte{0xc4, 0x82, 0x6d, 0x90, 0x0c, 0x40}, 32}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86RawGatherInstruction(test.code, test.mode); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", test.code, matched, err) + } + }) + } +} + +func TestX86RawEVEXGatherRejectsReservedForms(t *testing.T) { + const source = "TEXT evexgather(SB),4,$0-0\n\tVPGATHERDD 8(BX)(X2*4), K1, X3\n\tRET\n" + assembled := assembleX87ControlBytes(t, "amd64", source) + if len(assembled) < 8 || assembled[0] != 0x62 { + t.Fatalf("Go did not emit an EVEX gather: %x", assembled) + } + valid := assembled[:len(assembled)-1] + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing mask", func(code []byte) []byte { code[3] &^= 7; return code }}, + {"zeroing", func(code []byte) []byte { code[3] |= 0x80; return code }}, + {"broadcast", func(code []byte) []byte { code[3] |= 0x10; return code }}, + {"reserved length", func(code []byte) []byte { code[3] |= 0x60; return code }}, + {"reserved vvvv", func(code []byte) []byte { code[2] &^= 0x08; return code }}, + {"missing fixed bit", func(code []byte) []byte { code[2] &^= 0x04; return code }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), valid...)) + if _, _, matched, err := decodedX86RawGatherInstruction(code, 64); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", code, matched, err) + } + }) + } +} + +func TestTranslateX86RawGatherLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, form := range []struct { + name string + source string + }{ + {"VEX", "TEXT rawgather(SB),4,$0-0\n\tVPGATHERDD X2, (BX)(X0*2), X1\n\tRET\n"}, + {"EVEX", "TEXT rawgather(SB),4,$0-0\n\tVPGATHERDD (BX)(X0*2), K1, X2\n\tRET\n"}, + } { + t.Run(form.name, func(t *testing.T) { + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + code := assembleX87ControlBytes(t, target.goarch, form.source) + file := x86RawUnprovedReturnFormProbeFile(t, target.goarch, "rawgather", code) + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawgather": {Name: "rawgather", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-gather.ll", "raw-gather.o", ir) + }) + } + }) + } +} diff --git a/amd64_raw_gfni.go b/amd64_raw_gfni.go new file mode 100644 index 000000000..aac52af6b --- /dev/null +++ b/amd64_raw_gfni.go @@ -0,0 +1,58 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// The Go 1.27 table has one GFNI family: multiply (0F38 CF, W0) and the +// affine/inverse pair (0F3A CE/CF, W1). Its VEX rows are X/Y; EVEX adds Z, +// masks, zeroing and, only for the affine matrix source, qword broadcast. +func decodedX86GFNIInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok { + return Instr{}, 0, false, nil + } + + var op Op + var immediate bool + switch { + case p.mapNumber == 2 && p.opcode == 0xcf: + op = "VGF2P8MULB" + case p.mapNumber == 3 && p.opcode == 0xce: + op, immediate = "VGF2P8AFFINEQB", true + case p.mapNumber == 3 && p.opcode == 0xcf: + op, immediate = "VGF2P8AFFINEINVQB", true + default: + return Instr{}, 0, false, nil + } + if p.w != immediate { + return Instr{}, 0, true, fmt.Errorf("%s has invalid W bit", op) + } + if p.broadcast && !immediate { + return Instr{}, 0, true, fmt.Errorf("%s has no broadcast form", op) + } + + laneBits := 8 + if immediate { + laneBits = 64 + } + instruction, length, _, err := decodedX86BinaryVectorOperands(code, p, mode, op, laneBits) + if err != nil { + return Instr{}, 0, true, err + } + if !immediate { + return instruction, length, true, nil + } + if len(code) <= length { + return Instr{}, 0, true, fmt.Errorf("%s is missing its imm8", op) + } + instruction.Args = append([]Operand{{Kind: OpImm, Imm: int64(code[length])}}, instruction.Args...) + length++ + parts := make([]string, len(instruction.Args)) + for index, arg := range instruction.Args { + parts[index] = arg.String() + } + instruction.Raw = fmt.Sprintf("%s %s", instruction.Op, strings.Join(parts, ", ")) + return instruction, length, true, nil +} diff --git a/amd64_raw_gfni_test.go b/amd64_raw_gfni_test.go new file mode 100644 index 000000000..045651907 --- /dev/null +++ b/amd64_raw_gfni_test.go @@ -0,0 +1,173 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawGFNIGoFECRegression(t *testing.T) { + // GoFEC v1.4.2 emits this EVEX-512 affine instruction as BYTE directives. + code := []byte{0x62, 0xf3, 0x85, 0x48, 0xce, 0xc8, 0x00} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "GoFEC GFNI", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, "VGF2P8AFFINEQB $0, Z0, Z15, Z1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawGFNICompleteGoFamily(t *testing.T) { + for _, family := range []struct { + name string + mapID byte + opcode byte + w byte + imm bool + }{ + {name: "VGF2P8MULB", mapID: 2, opcode: 0xcf}, + {name: "VGF2P8AFFINEQB", mapID: 3, opcode: 0xce, w: 1, imm: true}, + {name: "VGF2P8AFFINEINVQB", mapID: 3, opcode: 0xcf, w: 1, imm: true}, + } { + for _, encoding := range []struct { + name string + width int + evex bool + masked bool + zeroing bool + }{ + {name: "vex-x", width: 0}, + {name: "vex-y", width: 1}, + {name: "evex-x", width: 0, evex: true}, + {name: "evex-y-mask", width: 1, evex: true, masked: true}, + {name: "evex-z-zero", width: 2, evex: true, masked: true, zeroing: true}, + } { + t.Run(family.name+"/"+encoding.name, func(t *testing.T) { + width := []string{"X", "Y", "Z"}[encoding.width] + var code []byte + if encoding.evex { + last := byte(encoding.width<<5) | 0x08 + if encoding.masked { + last |= 1 + } + if encoding.zeroing { + last |= 0x80 + } + code = []byte{0x62, 0xf0 | family.mapID, family.w<<7 | 0x75, last, family.opcode, 0xd0} + } else { + code = []byte{0xc4, 0xe0 | family.mapID, family.w<<7 | 0x71 | byte(encoding.width<<2), family.opcode, 0xd0} + } + if family.imm { + code = append(code, 0x63) + } + instruction, length, ok, err := decodedX86GFNIInstruction(code, 64) + if !ok || err != nil || length != len(code) { + t.Fatalf("decode %x: ok=%v length=%d err=%v", code, ok, length, err) + } + args := fmt.Sprintf("%s0, %s1", width, width) + if family.imm { + args = "$99, " + args + } + if encoding.masked { + args += ", K1" + } + wantOp := family.name + if encoding.zeroing { + wantOp += ".Z" + } + want := fmt.Sprintf("%s %s, %s2", wantOp, args, width) + if instruction.Raw != want { + t.Fatalf("decode %x as %q, want %q", code, instruction.Raw, want) + } + }) + } + } +} + +func TestDecodeX86RawGFNIRejectsReservedForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + }{ + {name: "affine wrong W", code: []byte{0x62, 0xf3, 0x75, 0x48, 0xce, 0xd0, 0x63}}, + {name: "multiply wrong W", code: []byte{0x62, 0xf2, 0xf5, 0x48, 0xcf, 0xd0}}, + {name: "multiply broadcast", code: []byte{0x62, 0xf2, 0x75, 0x58, 0xcf, 0x50, 0x01}}, + {name: "affine broadcast register", code: []byte{0x62, 0xf3, 0xf5, 0x58, 0xce, 0xd0, 0x63}}, + {name: "zero without mask", code: []byte{0x62, 0xf3, 0xf5, 0xc8, 0xce, 0xd0, 0x63}}, + {name: "truncated immediate", code: []byte{0x62, 0xf3, 0xf5, 0x48, 0xce, 0xd0}}, + {name: "reserved vector length", code: []byte{0x62, 0xf3, 0xf5, 0x68, 0xce, 0xd0, 0x63}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86GFNIInstruction(test.code, 64) + if !ok || err == nil { + t.Fatalf("decode %x: ok=%v err=%v, want recognized rejection", test.code, ok, err) + } + }) + } +} + +func TestDecodeX86RawGFNIMemoryAndBroadcast(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + {name: "vex multiply memory", code: []byte{0xc4, 0xe2, 0x71, 0xcf, 0x50, 0x08}, want: "VGF2P8MULB 8(AX), X1, X2"}, + {name: "evex affine broadcast", code: []byte{0x62, 0xf3, 0xf5, 0x59, 0xce, 0x50, 0x01, 0x63}, want: "VGF2P8AFFINEQB.BCST $99, 8(AX), Z1, K1, Z2"}, + {name: "evex inverse broadcast zero", code: []byte{0x62, 0xf3, 0xf5, 0xd9, 0xcf, 0x50, 0x01, 0xa5}, want: "VGF2P8AFFINEINVQB.BCST.Z $165, 8(AX), Z1, K1, Z2"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86GFNIInstruction(test.code, 64) + if !ok || err != nil || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode %x = %q, length=%d, ok=%v, err=%v; want %q", test.code, instruction.Raw, length, ok, err, test.want) + } + }) + } +} + +func TestTranslateX86RawGFNIAllObjectTargets(t *testing.T) { + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawgfni(SB),$0-0\n") + for _, instruction := range [][]byte{ + {0xc4, 0xe2, 0x71, 0xcf, 0xd0}, + {0xc4, 0xe3, 0xf1, 0xce, 0xd0, 0x63}, + {0x62, 0xf3, 0xf5, 0xc9, 0xcf, 0xd0, 0xa5}, + } { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawgfni": {Name: "rawgfni", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "raw-gfni.ll", "raw-gfni.o", ir) + }) + } +} diff --git a/amd64_raw_horizontal_float_literal_test.go b/amd64_raw_horizontal_float_literal_test.go new file mode 100644 index 000000000..f2f608ca2 --- /dev/null +++ b/amd64_raw_horizontal_float_literal_test.go @@ -0,0 +1,97 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawHorizontalFloatLiteral(opcode, pp byte, vex3 bool, width int) []byte { + code := encodeX86VEXHorizontalFloat(opcode, pp, vex3, width == 32, false, 1, 2, 0) + code[len(code)-1] = 0x0d // RIP+disp32, destination X/Y1. + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*11+7)) + } + return code +} + +func TestDecodeX86RawHorizontalFloatRIPDataCompleteFamily(t *testing.T) { + for opcode, stem := range map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"} { + for pp, suffix := range map[byte]string{1: "PD", 3: "PS"} { + for _, vex3 := range []bool{false, true} { + for _, width := range []int{16, 32} { + name := fmt.Sprintf("%s%s/vex3%t/%d", stem, suffix, vex3, width) + t.Run(name, func(t *testing.T) { + code := x86RawHorizontalFloatLiteral(opcode, pp, vex3, width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != Op(stem+suffix) || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-width:]) { + t.Fatalf("decoded %x as %#v", code, decoded) + } + }) + } + } + } + } +} + +func TestDecodeX86RawHorizontalFloatRIPDataRejectsUnsafeSources(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[5] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[5] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawHorizontalFloatLiteral(0xd0, 1, true, 32)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe horizontal/alternating RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawHorizontalFloatRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for opcode, stem := range map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"} { + for pp, suffix := range map[byte]string{1: "PD", 3: "PS"} { + name := "literal" + stem + suffix + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawHorizontalFloatLiteral(opcode, pp, true, 32) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-horizontal-float-literal.ll", "raw-horizontal-float-literal.o", ir) + }) + } +} diff --git a/amd64_raw_horizontal_integer.go b/amd64_raw_horizontal_integer.go new file mode 100644 index 000000000..b619228f4 --- /dev/null +++ b/amd64_raw_horizontal_integer.go @@ -0,0 +1,52 @@ +package plan9asm + +import "fmt" + +// Go's VPHADD/PHSUB family has two _yvaddsubpd rows, X and Y, without +// EVEX/masks. W is ignored (Go emits W0); Intel XED leaves W unconstrained: +// https://github.com/intelxed/xed/blob/main/datafiles/avx/avx-isa.txt +func decodedX86VEXHorizontalIntegerInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 { + return Instr{}, 0, false, nil + } + var op Op + for name, spec := range amd64HorizontalIntegerSpecs { + if spec.vector && int(spec.opcode) == p.opcode { + op = name + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("horizontal integer: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.pp != 1 || p.vectorLength > 1 { + return fail("requires VEX.66.0F38 X/Y encoding") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.x != 0 || p.b != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + width := []string{"X", "Y"}[p.vectorLength] + source2, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + source1 := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8))} + return Instr{ + Op: op, Args: []Operand{source2, source1, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", op, source2.String(), source1.String(), destination.String()), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_horizontal_integer_test.go b/amd64_raw_horizontal_integer_test.go new file mode 100644 index 000000000..a7d89fb95 --- /dev/null +++ b/amd64_raw_horizontal_integer_test.go @@ -0,0 +1,296 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86RawHorizontalIntegerEncodingAxes(t *testing.T) { + count := 0 + for _, form := range []struct { + op Op + opcode byte + }{ + {"VPHADDW", 0x01}, {"VPHADDD", 0x02}, {"VPHADDSW", 0x03}, + {"VPHSUBW", 0x05}, {"VPHSUBD", 0x06}, {"VPHSUBSW", 0x07}, + } { + for width, prefix := range []string{"X", "Y"} { + for _, w := range []bool{false, true} { + for dst := 0; dst < 16; dst++ { + for src1 := 0; src1 < 16; src1++ { + for src2 := 0; src2 < 16; src2++ { + code := encodeX86VEXHorizontalFloat(form.opcode, 1, true, width == 1, w, dst, src1, src2) + code[1] = code[1]&0xe0 | 2 + got, size, matched, err := decodedX86VEXHorizontalIntegerInstruction(code, 64) + want := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src2))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src1))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, dst))}} + if err != nil || !matched || size != len(code) || got.Op != form.op || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v size=%d matched=%v err=%v; want %s %+v", code, got, size, matched, err, form.op, want) + } + _, _, matched, err = decodedX86VEXHorizontalIntegerInstruction(code, 32) + if !matched || (err != nil) != (dst >= 8 || src1 >= 8 || src2 >= 8) { + t.Fatalf("386 decode %x matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 98304 { + t.Fatalf("covered %d encodings, want 98304", count) + } +} + +func TestX86RawHorizontalIntegerInvalidAndSegmentForms(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x79, 0x02}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0x02, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0x02, 0x45}, // Missing disp8. + {0xc4, 0xe2, 0x78, 0x02, 0xc0}, // pp=none. + {0xc4, 0xe2, 0x7a, 0x02, 0xc0}, // pp=F3. + {0xc4, 0xe2, 0x7b, 0x02, 0xc0}, // pp=F2. + {0x62, 0xf2, 0x7d, 0x08, 0x02, 0xc0}, // No EVEX variant. + {0x67, 0xc4, 0xe2, 0x79, 0x02, 0x00}, + } { + if _, _, matched, err := decodedX86VEXHorizontalIntegerInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x matched=%v err=%v", code, matched, err) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{segment.prefix, 0xc4, 0x02, 0xe5, 0x06, 0x64, 0x88, 0x20} + got, size, matched, err := decodedX86VEXHorizontalIntegerInstruction(code, 64) + want := []Operand{{Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: 32}}, {Kind: OpReg, Reg: "Y3"}, {Kind: OpReg, Reg: "Y12"}} + if err != nil || !matched || size != len(code) || got.Op != "VPHSUBD" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v", code, got, err) + } + } +} + +func TestX86HorizontalIntegerGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"VPHADDW", "VPHADDD", "VPHADDSW", "VPHSUBW", "VPHSUBD", "VPHSUBSW"} { + lines := []string{op + " X0,Y1,Y2", op + " Z0,Z1,Z2", op + " X0,X1,K1,X2", op + " X0,X1,(AX)", op + ".Z X0,X1,X2", op + ".SAE X0,X1,X2", op + " X16,X0,X1"} + for _, line := range lines { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(path, []byte("TEXT invalid(SB),4,$0-0\n"+line+"\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + if output, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("Go %s accepted %s; repair oracle: %s", arch, line, output) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86HorizontalIntegerRejected(t, arch, triple, line) + } + } + } +} + +func TestTranslateX86HorizontalIntegerGo386RegisterCompatibility(t *testing.T) { + // Go's VEX Yxr/Yyr classes accept X/Y8-15 even for its 386 frontend. + // This named compatibility grammar is distinct from architectural raw + // 32-bit decoding, which still rejects extended register fields. + var source strings.Builder + source.WriteString("TEXT compat(SB),4,$0-0\n") + for _, op := range []string{"VPHADDW", "VPHADDD", "VPHADDSW", "VPHSUBW", "VPHSUBD", "VPHSUBSW"} { + for _, width := range []string{"X", "Y"} { + for _, regs := range [][3]int{{8, 0, 1}, {0, 8, 1}, {0, 1, 8}, {15, 14, 13}} { + fmt.Fprintf(&source, "%s %s%d,%s%d,%s%d\n", op, width, regs[0], width, regs[1], width, regs[2]) + } + } + } + source.WriteString("RET\n") + assembleX87ControlBytes(t, "386", source.String()) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(triple, func(t *testing.T) { + options := Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"compat": {Name: "compat", Ret: Void}}} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("Go frontend acceptance invented high physical mode32 registers: %v", err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("direct module invented mode32 high registers", err) + } + translated, err := translateGoModuleInContext(ctx, x86SADGoPackage(t, "386", "()"), []byte(strings.Replace(source.String(), "compat(SB)", "·sad(SB)", 1)), GoModuleOptions{GOARCH: "386", TargetTriple: triple}) + if translated != nil && !translated.Module.IsNil() { + defer translated.Module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("typed module invented mode32 high registers", err) + } + }) + } +} + +// All six VEX instructions share Go's two _yvaddsubpd rows (X/Y). +// The legacy siblings use yxm_q4, except PHADDD's MMX-capable ymmxmm0f38. +func TestX86RawHorizontalIntegerGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 15 + } + var source strings.Builder + source.WriteString("TEXT horizontal(SB),4,$0-0\nPHADDD M0,M7\nPHADDD -7(BX)(CX*4),M0\n") + for _, op := range []string{"PHADDD", "PHADDSW", "PHADDW", "PHSUBD", "PHSUBSW", "PHSUBW"} { + for _, reg := range []int{0, last} { + for _, input := range []string{fmt.Sprintf("X%d", last-reg), "(AX)", "-7(BX)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "%s %s,X%d\n", op, input, reg) + } + for _, width := range []string{"X", "Y"} { + for _, input := range []string{fmt.Sprintf("%s%d", width, last-reg), "(AX)", "-7(BX)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "V%s %s,%s%d,%s%d\n", op, input, width, last-reg, width, reg) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(want) != 147 || len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions; expected %d, including RET", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "horizontal", code) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"horizontal": {Name: "horizontal", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-horizontal.ll", "raw-horizontal.o", ir) + } + }) + } +} + +func TestX86HorizontalIntegerViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; separate five-target object tests are required") + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, spec := range []struct { + op string + bits, subtract, saturate int + }{{"VPHADDW", 16, 0, 0}, {"VPHADDD", 32, 0, 0}, {"VPHADDSW", 16, 0, 1}, {"VPHSUBW", 16, 1, 0}, {"VPHSUBD", 32, 1, 0}, {"VPHSUBSW", 16, 1, 1}} { + for width, prefix := range []string{"X", "Y"} { + for _, memory := range []bool{false, true} { + for dst := 0; dst < 3; dst++ { + name := fmt.Sprintf("horizontal%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\nMOVQ old+24(FP),SI\nVMOVUPS (AX),Z0\nVMOVUPS (BX),Z1\nVMOVUPS (SI),Z2\n", name) + input := prefix + "1" + if memory { + input = "(BX)" + } + fmt.Fprintf(&source, "%s %s,%s0,%s%d\nVMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\nRET\n", spec.op, input, prefix, prefix, dst, dst, dst, dst) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: Ptr, Index: 3, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,const void *);\n", name) + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d,%d)) return %d;\n", name, 16< +#include +#include +` + declarations.String() + ` +static int64_t lane(const uint8_t *p,int bytes) { + if(bytes==2) {int16_t v;memcpy(&v,p,2);return v;} + int32_t v;memcpy(&v,p,4);return v; +} +static int check(void (*fn)(const void *,const void *,void *,const void *),int bytes,int laneBytes,int sub,int sat) { + for(int seed=0;seed<128;seed++) { + uint8_t a[64],b[64],old[64],out[112],want[64]={0}; + for(int i=0;i<64;i++) {a[i]=(uint8_t)(seed*17+i*31);b[i]=(uint8_t)(seed*43+i*7);old[i]=(uint8_t)(193-i);} + for(int i=0;i32767)result=32767;} + uint64_t bits=(uint64_t)result; + for(int j=0;j>(j*8)); + } + fn(a,b,out,old); + for(int view=0;view<3;view++) { + int offset=view==0?0:view==1?64:96; + for(int i=0;i<(64>>view);i++) if(out[offset+i]!=want[i]) { + fprintf(stderr,"horizontal mismatch bytes=%d lane=%d sub=%d sat=%d seed=%d view=%d byte=%d\n",bytes,laneBytes,sub,sat,seed,view,i);return 1; + } + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "horizontal_integer", triple, ir, mainC, commandPrefix) +} diff --git a/amd64_raw_hypercall_test.go b/amd64_raw_hypercall_test.go new file mode 100644 index 000000000..5dc9fdfa8 --- /dev/null +++ b/amd64_raw_hypercall_test.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +// KVM defines both vendor encodings of the same zero-operand hypercall ABI: +// https://docs.kernel.org/virt/kvm/x86/hypercalls.html +func TestX86RawKVMHypercallFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, op := range []struct { + name string + last byte + }{ + {name: "VMCALL", last: 0xc1}, + {name: "VMMCALL", last: 0xd9}, + } { + for _, target := range []struct { + arch, triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(op.name+"/"+target.triple, func(t *testing.T) { + source := fmt.Sprintf(`TEXT hypercall(SB),$0-0 + MOVL $9, AX + MOVL $17, BX + MOVL $23, CX + MOVL $29, DX + MOVL $31, SI + BYTE $0x0f; BYTE $0x01; BYTE $0x%02x + RET +`, op.last) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + if decoded.Instrs[len(decoded.Instrs)-2].Op != Op(op.name) { + t.Fatalf("wrong hypercall decoding: %+v", decoded.Instrs) + } + word := I64 + if target.arch == "386" { + word = I32 + } + ir, err := Translate(file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"hypercall": {Name: "hypercall", Ret: word}}, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `asm sideeffect "` + strings.ToLower(op.name) + `"`, + `={ax},0,{bx},{cx},{dx},{si},`, `~{memory}`, + } { + if !strings.Contains(ir, want) { + t.Fatalf("missing hypercall ABI %q", want) + } + } + compileLLVMToObject(t, llc, target.triple, "hypercall.ll", "hypercall.o", ir) + }) + } + } +} + +func TestX86KVMHypercallRejectsExplicitOperandsAndSuffixes(t *testing.T) { + for _, instruction := range []string{"VMCALL AX", "VMMCALL AX", "VMCALL.Z", "VMMCALL.Z"} { + file, err := Parse(ArchAMD64, "TEXT invalid(SB),$0-0\n"+instruction+"\nRET\n") + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"invalid": {Name: "invalid", Ret: Void}}, + }) + if err == nil { + t.Fatalf("accepted invalid hypercall %q", instruction) + } + } +} diff --git a/amd64_raw_ifma_test.go b/amd64_raw_ifma_test.go new file mode 100644 index 000000000..9fbe4a353 --- /dev/null +++ b/amd64_raw_ifma_test.go @@ -0,0 +1,98 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawDirectiveGroupIFMACompleteFamily(t *testing.T) { + // Bytes come from Go 1.27's avx512_ifma assembler test table and cover + // both opcodes, all widths, high registers, memory, and masking. + for _, test := range []struct { + name string + code []byte + want string + }{ + {name: "high x high registers", code: []byte{0x62, 0xe2, 0xa5, 0x09, 0xb5, 0xd7}, want: "VPMADD52HUQ X7, X11, K1, X18"}, + {name: "high y high registers", code: []byte{0x62, 0x82, 0x85, 0x27, 0xb5, 0xcc}, want: "VPMADD52HUQ Y28, Y31, K7, Y17"}, + {name: "low y memory", code: []byte{0x62, 0xf2, 0xb5, 0x29, 0xb4, 0x94, 0x2c, 0x11, 0, 0, 0}, want: "VPMADD52LUQ 17(SP)(BP*1), Y9, K1, Y2"}, + {name: "low z high registers", code: []byte{0x62, 0x22, 0xd5, 0x47, 0xb4, 0xe0}, want: "VPMADD52LUQ Z16, Z21, K7, Z28"}, + {name: "low z broadcast zeroing", code: []byte{0x62, 0xf2, 0xf5, 0xdb, 0xb4, 0x50, 0x7f}, want: "VPMADD52LUQ.BCST.Z 1016(AX), Z1, K3, Z2"}, + } { + t.Run(test.name, func(t *testing.T) { + decoded, err := decodeX86RawDirectiveGroup(test.code, 64, 0, test.name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, test.want+" ") { + t.Fatalf("decoded %x as %#v, want %q", test.code, decoded, test.want) + } + }) + } +} + +func TestDecodedX86RawIFMAAllRegisterFields(t *testing.T) { + for _, operation := range []struct { + opcode byte + op Op + }{ + {opcode: 0xb4, op: "VPMADD52LUQ"}, + {opcode: 0xb5, op: "VPMADD52HUQ"}, + } { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 2 + p1 := byte(1)<<7 | byte(^second&15)<<3 | 5 + p2 := byte(vectorBits)<<5 | byte((^second>>4)&1)<<3 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, operation.opcode, modRM} + instruction, length, ok, err := decodedX86IFMAInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("op=%s width=%s first=%d second=%d destination=%d: ok=%v length=%d err=%v", operation.op, vectorName, first, second, destination, ok, length, err) + } + want := fmt.Sprintf("%s %s%d, %s%d, %s%d", operation.op, vectorName, first, vectorName, second, vectorName, destination) + if instruction.Raw != want { + t.Fatalf("encoding %x decoded as %q, want %q", encoding, instruction.Raw, want) + } + } + } + } + } + } +} + +func TestDecodedX86RawIFMARejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 masked operand limit", mode: 32, encoding: []byte{0x62, 0xf2, 0xf5, 0x09, 0xb4, 0xc2}, wantOK: true}, + {name: "386 extended register", mode: 32, encoding: []byte{0x62, 0x72, 0xf5, 0x08, 0xb4, 0xc2}, wantOK: true}, + {name: "reserved vector length", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x68, 0xb4, 0xc2}, wantOK: true}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x18, 0xb4, 0xc2}, wantOK: true}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x88, 0xb5, 0xc2}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0x62, 0xf2, 0xf5, 0x08, 0xb4, 0xc2}, wantOK: true}, + {name: "wrong map", mode: 64, encoding: []byte{0x62, 0xf3, 0xf5, 0x08, 0xb4, 0xc2}, wantOK: false}, + {name: "wrong prefix", mode: 64, encoding: []byte{0x62, 0xf2, 0xf4, 0x08, 0xb4, 0xc2}, wantOK: false}, + {name: "wrong w", mode: 64, encoding: []byte{0x62, 0xf2, 0x75, 0x08, 0xb4, 0xc2}, wantOK: true}, + {name: "truncated", mode: 64, encoding: []byte{0x62, 0xf2, 0xf5, 0x08, 0xb4}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86IFMAInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x: ok=%v, want %v (err=%v)", test.encoding, ok, test.wantOK, err) + } + if test.wantOK && err == nil { + t.Fatalf("encoding %x was recognized without rejection", test.encoding) + } + }) + } +} diff --git a/amd64_raw_indexed_permute_test.go b/amd64_raw_indexed_permute_test.go new file mode 100644 index 000000000..d8c815486 --- /dev/null +++ b/amd64_raw_indexed_permute_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86RawIndexedPermuteForms = []struct { + opcode int + widthBit int + op Op + laneBytes int +}{ + {0x75, 0, "VPERMI2B", 1}, {0x75, 1, "VPERMI2W", 2}, + {0x76, 0, "VPERMI2D", 4}, {0x76, 1, "VPERMI2Q", 8}, + {0x77, 0, "VPERMI2PS", 4}, {0x77, 1, "VPERMI2PD", 8}, + {0x7d, 0, "VPERMT2B", 1}, {0x7d, 1, "VPERMT2W", 2}, + {0x7e, 0, "VPERMT2D", 4}, {0x7e, 1, "VPERMT2Q", 8}, + {0x7f, 0, "VPERMT2PS", 4}, {0x7f, 1, "VPERMT2PD", 8}, +} + +func TestDecodeX86RawIndexedPermuteSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPERMT2W Y18, Y19, Y21. + code := []byte{0x62, 0xa2, 0xe5, 0x20, 0x7d, 0xea, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "validUTF8 AVX512 VPERMT2W", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPERMT2W" || + decoded[0].Args[0].Reg != "Y18" || decoded[0].Args[1].Reg != "Y19" || + decoded[0].Args[2].Reg != "Y21" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func encodeX86RawIndexedPermute(opcode, widthBit, vectorBits, mask int, broadcast, memory bool) []byte { + p1 := byte(0x75) // EVEX.vvvv = vector register 1, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // first source register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX), destination register 2. + } + code := []byte{0x62, 0xf2, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawIndexedPermuteCompleteGoForms(t *testing.T) { + if len(x86RawIndexedPermuteForms) != 12 || len(amd64IndexedPermuteSpecs) != 14 { + t.Fatalf("raw/indexed-permute grammar has %d/%d rows, want 12/14", len(x86RawIndexedPermuteForms), len(amd64IndexedPermuteSpecs)) + } + for _, form := range x86RawIndexedPermuteForms { + for vectorBits, vector := range []string{"X", "Y", "Z"} { + for _, variant := range []struct { + name string + mask int + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "memory", memory: true}, + {name: "masked memory", mask: 7, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + } { + if variant.broadcast && form.laneBytes < 4 { + continue + } + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawIndexedPermute(form.opcode, form.widthBit, vectorBits, variant.mask, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86IndexedPermuteInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.broadcast { + wantOp += ".BCST" + } + if got.Op != wantOp || got.Args[1].String() != vector+"1" || + got.Args[len(got.Args)-1].String() != vector+"2" || + len(got.Args) != 3+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawIndexedPermuteRIPLiteral(t *testing.T) { + for _, form := range x86RawIndexedPermuteForms { + for _, broadcast := range []bool{false, true} { + if broadcast && form.laneBytes < 4 { + continue + } + code := encodeX86RawIndexedPermute(form.opcode, form.widthBit, 2, 1, broadcast, false) + code[5] = 0x15 // RIP-relative first source. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 64 + if broadcast { + literalWidth = form.laneBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/broadcast%t", form.op, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != wantOp { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestTranslateX86RawIndexedPermuteLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawIndexedPermute(SB),$0-0\n") + for _, opcode := range []int{0x75, 0x76, 0x77, 0x7d, 0x7e, 0x7f} { + for _, widthBit := range []int{0, 1} { + code := encodeX86RawIndexedPermute(opcode, widthBit, 0, 0, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + if opcode == 0x75 || opcode == 0x7d { + continue + } + mask := 3 + if target.goarch == "386" { + mask = 0 + } + code = encodeX86RawIndexedPermute(opcode, widthBit, 0, mask, true, true) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawIndexedPermute": {Name: "rawIndexedPermute", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-indexed-permute.ll", "raw-indexed-permute.o", ir) + }) + } +} diff --git a/amd64_raw_insertps_mode32_test.go b/amd64_raw_insertps_mode32_test.go new file mode 100644 index 000000000..c6057a89c --- /dev/null +++ b/amd64_raw_insertps_mode32_test.go @@ -0,0 +1,148 @@ +package plan9asm + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +func x86RawINSERTPSSource(name string, code []byte) string { + var source strings.Builder + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + return source.String() +} + +func TestX86RawINSERTPSMode32CompleteRegisterGrammar(t *testing.T) { + // Go's 386 frontend rejects four operands. BYTE acceptance is a distinct + // source role, not permission to accept the equivalent named instruction. + for _, row := range []string{ + "VINSERTPS $28,X7,X7,X7", + "VINSERTPS $28,1(BX),X7,X7", + } { + goAssemblerOracleOutput(t, "386", "TEXT f(SB),4,$0-0\n"+row+"\nRET\n", "", false) + file, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if err != nil { + t.Fatal(err) + } + ins := file.Funcs[0].Instrs[1] + ins.Raw += " /* decoded from BYTE $0xc4 */" + if _, known, err := x86InsertExtractProperties(ins, "386"); !known || err == nil { + t.Fatalf("a textual annotation granted physical mode32 identity: %+v", ins) + } + } + goAssemblerOracleOutput(t, "386", x86RawINSERTPSSource("f", encodeX86RawVEXINSERTPS(7, 7, 7, 28)), "", true) + + count := 0 + for source := 0; source < 8; source++ { + for base := 0; base < 8; base++ { + for destination := 0; destination < 8; destination++ { + immediate := byte(source*32 + base*4 + destination) + code := encodeX86RawVEXINSERTPS(source, base, destination, immediate) + ins, length, known, err := decodedX86InsertPSInstruction(code, 32) + if err != nil || !known || length != len(code) || !ins.x86Encoded { + t.Fatalf("physical mode32 decode %x: %+v, length=%d, known=%t, err=%v", code, ins, length, known, err) + } + form, known, err := x86InsertExtractProperties(ins, "386") + if err != nil || !known || form.immediate != immediate || form.width != 16 || form.output != 16 || form.memory != -1 { + t.Fatalf("physical mode32 grammar %x: %+v, known=%t, err=%v", code, form, known, err) + } + count++ + } + } + } + if count != 512 { + t.Fatalf("mode32 register axis has %d cases, want 512", count) + } +} + +func TestX86RawINSERTPSVEXIgnoredWLLVMOracle(t *testing.T) { + mc := findLLVM22Tool("llvm-mc") + if mc == "" { + t.Fatal("LLVM 22 llvm-mc not found") + } + // Intel XED's VINSERTPS patterns do not constrain REXW, unlike adjacent + // VINSERTF128 patterns: datafiles/avx/avx-isa.txt. Both VEX.W values decode + // to the same instruction; Go's named encoder always chooses W=0. + for _, mode := range []int{32, 64} { + var input strings.Builder + for _, w := range []byte{0, 0x80} { + code := encodeX86RawVEXINSERTPS(7, 7, 7, 28) + code[2] |= w + for _, value := range code { + fmt.Fprintf(&input, "%#02x ", value) + } + input.WriteByte('\n') + ins, length, known, err := decodedX86InsertPSInstruction(code, mode) + if err != nil || !known || length != len(code) || ins.Raw != "VINSERTPS $28, X7, X7, X7" { + t.Fatalf("VEX.WIG mode%d decode %x: %+v, length=%d, known=%t, err=%v", mode, code, ins, length, known, err) + } + } + triple := "i386-unknown-linux-gnu" + if mode == 64 { + triple = "x86_64-unknown-linux-gnu" + } + ctx, cancel := runtimeOracleStageContext(t) + command := exec.CommandContext(ctx, mc, "--disassemble", "--triple="+triple, "--mattr=+avx") + command.Stdin = strings.NewReader(input.String()) + output, err := runtimeOracleCombinedOutput(ctx, command) + cancel() + if err != nil || strings.Count(string(output), "vinsertps\t$28, %xmm7, %xmm7, %xmm7") != 2 { + t.Fatalf("LLVM 22 independent VEX.WIG mode%d: %v\n%s", mode, err, output) + } + } +} + +func TestX86RawINSERTPSMode32MemoryAndFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Independent ModRM/SIB axes: direct, signed disp8/disp32 and every scale. + memory := [][]byte{ + {0x3b}, {0x7b, 0x80}, {0x7b, 0x7f}, + {0xbb, 0, 0, 0, 0x80}, {0xbb, 0xff, 0xff, 0xff, 0x7f}, + {0x7c, 0x33, 1}, {0x7c, 0x73, 1}, {0x7c, 0xb3, 1}, {0x7c, 0xf3, 1}, + } + var source strings.Builder + sigs := map[string]FuncSig{} + for index, modRM := range memory { + for _, w := range []byte{0, 0x80} { + code := append([]byte{0xc4, 0xe3, 0x41 | w, 0x21}, modRM...) + code = append(code, 0xff) + ins, length, known, err := decodedX86InsertPSInstruction(code, 32) + if err != nil || !known || length != len(code) { + t.Fatalf("mode32 memory decode %x: %+v, length=%d, known=%t, err=%v", code, ins, length, known, err) + } + form, known, err := x86InsertExtractProperties(ins, "386") + if err != nil || !known || form.memory != 1 || form.spec.chunk != 4 || form.immediate != 255 { + t.Fatalf("mode32 memory grammar %x: %+v, known=%t, err=%v", code, form, known, err) + } + name := fmt.Sprintf("memory%d_w%d", index, w>>7) + source.WriteString(x86RawINSERTPSSource(name, code)) + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + for _, arch := range []string{"386", "amd64"} { + goAssemblerOracleOutput(t, arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triples := []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + if arch == "amd64" { + triples = []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-insertps-memory.ll", "raw-insertps-memory.o", ir) + } + } +} diff --git a/amd64_raw_insertps_test.go b/amd64_raw_insertps_test.go new file mode 100644 index 000000000..c09308c6b --- /dev/null +++ b/amd64_raw_insertps_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawLegacyINSERTPS(source, destination int, immediate byte) []byte { + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + code := []byte{0x66} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x3a, 0x21, byte(0xc0|destination&7<<3|source&7), immediate) +} + +func encodeX86RawVEXINSERTPS(source, base, destination int, immediate byte) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^source>>3)&1)<<5 | 3 + p1 := byte(^base&15)<<3 | 1 + return []byte{0xc4, p0, p1, 0x21, byte(0xc0 | destination&7<<3 | source&7), immediate} +} + +func encodeX86RawEVEXINSERTPS(source, base, destination int, immediate byte) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^source>>4)&1)<<6 | + byte((^source>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := byte(^base&15)<<3 | 5 + p2 := byte((^base>>4)&1) << 3 + return []byte{0x62, p0, p1, p2, 0x21, byte(0xc0 | destination&7<<3 | source&7), immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVINSERTPS(t *testing.T) { + // vinsertps $0x1c, %xmm10, %xmm8, %xmm1; + // vshufps $0x24, %xmm0, %xmm1, %xmm0 + code := []byte{0xc4, 0xc3, 0x39, 0x21, 0xca, 0x1c, 0xc5, 0xf0, 0xc6, 0xc0, 0x24} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VINSERTPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VINSERTPS $28, X10, X8, X1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawINSERTPSCompleteRegisterFamily(t *testing.T) { + count := 0 + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawLegacyINSERTPS(source, destination, immediate) + got, length, ok, err := decodedX86InsertPSInstruction(code, 64) + want := fmt.Sprintf("INSERTPS $%d, X%d, X%d", immediate, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for source := 0; source < 16; source++ { + for base := 0; base < 16; base++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawVEXINSERTPS(source, base, destination, immediate) + got, length, ok, err := decodedX86InsertPSInstruction(code, 64) + want := fmt.Sprintf("VINSERTPS $%d, X%d, X%d, X%d", immediate, source, base, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + for source := 0; source < 32; source++ { + for base := 0; base < 32; base++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(source*8 + destination) + code := encodeX86RawEVEXINSERTPS(source, base, destination, immediate) + got, length, ok, err := decodedX86InsertPSInstruction(code, 64) + want := fmt.Sprintf("VINSERTPS $%d, X%d, X%d, X%d", immediate, source, base, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + if count != 37120 { + t.Fatalf("covered %d INSERTPS/VINSERTPS register encodings, want 37120", count) + } +} + +func TestDecodedX86RawINSERTPSMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0x66, 0x47, 0x0f, 0x3a, 0x21, 0x7c, 0x8b, 0x01, 0xff} + got, length, ok, err := decodedX86InsertPSInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "INSERTPS" || got.Args[2].String() != "X15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x03, 0x01, 0x21, 0x7c, 0x8b, 0x02, 0x80} + got, length, ok, err = decodedX86InsertPSInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VINSERTPS" || got.Args[2].String() != "X15" || got.Args[3].String() != "X15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0x03, 0x05, 0x00, 0x21, 0x7c, 0x8b, 0x02, 0x40} + got, length, ok, err = decodedX86InsertPSInstruction(evexMemory, 64) + wantMemory.Off = 8 + if err != nil || !ok || length != len(evexMemory) || got.Op != "VINSERTPS" || got.Args[2].String() != "X31" || got.Args[3].String() != "X31" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXINSERTPS(7, 7, 7, 1) + validEVEX := encodeX86RawEVEXINSERTPS(31, 31, 31, 1) + for name, invalid := range map[string][]byte{ + "vex vector length": {validVEX[0], validVEX[1], validVEX[2] | 0x04, validVEX[3], validVEX[4], validVEX[5]}, + "evex W": {validEVEX[0], validEVEX[1], validEVEX[2] | 0x80, validEVEX[3], validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 1, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex zero": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x80, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex broadcast": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x10, validEVEX[4], validEVEX[5], validEVEX[6]}, + "truncated vex immediate": validVEX[:len(validVEX)-1], + "truncated evex immediate": validEVEX[:len(validEVEX)-1], + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86InsertPSInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + if instruction, _, matched, decodeErr := decodedX86InsertPSInstruction(validEVEX, 32); !matched || decodeErr == nil { + t.Fatalf("386 EVEX encoding %x decoded as %+v, ok=%v, err=%v", validEVEX, instruction, matched, decodeErr) + } + extended386 := encodeX86RawVEXINSERTPS(8, 0, 0, 1) + if instruction, _, matched, decodeErr := decodedX86InsertPSInstruction(extended386, 32); !matched || decodeErr == nil { + t.Fatalf("386 extended encoding %x decoded as %+v, ok=%v, err=%v", extended386, instruction, matched, decodeErr) + } +} + +func TestTranslateX86RawINSERTPSTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + code := [][]byte{ + encodeX86RawLegacyINSERTPS(target.registerMax, target.registerMax, 0xff), + encodeX86RawVEXINSERTPS(target.registerMax, target.registerMax, target.registerMax, 0x1c), + } + if target.goarch == "amd64" { + code = append(code, encodeX86RawEVEXINSERTPS(31, 30, 29, 0x80)) + } + var source strings.Builder + source.WriteString("TEXT raw_insertps(SB),NOSPLIT,$0-0\n") + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_insertps": {Name: "raw_insertps", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-insertps-"+target.name+".ll", "raw-insertps-"+target.name+".o", ir) + }) + } +} diff --git a/amd64_raw_invalid_prefix_observation_test.go b/amd64_raw_invalid_prefix_observation_test.go new file mode 100644 index 000000000..230963ab2 --- /dev/null +++ b/amd64_raw_invalid_prefix_observation_test.go @@ -0,0 +1,201 @@ +package plan9asm + +import ( + "bytes" + "errors" + "fmt" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// BYTE acceptance establishes original bytes only. These instructions are not +// executed: Intel specifies #UD for their LOCK prefix. The observed decoder +// marks that prefix invalid; a textual conversion must not erase that fact. +func TestX86RawInvalidLockPrefixesMustNotDisappear(t *testing.T) { + rows := []struct { + op string + machine []byte + }{ + {"RDTSC", []byte{0x0f, 0x31}}, + {"RDTSCP", []byte{0x0f, 0x01, 0xf9}}, + {"LFENCE", []byte{0x0f, 0xae, 0xe8}}, + {"MFENCE", []byte{0x0f, 0xae, 0xf0}}, + {"SFENCE", []byte{0x0f, 0xae, 0xf8}}, + {"PAUSE", []byte{0xf3, 0x90}}, + } + for _, mode := range []int{32, 64} { + arch := "386" + if mode == 64 { + arch = "amd64" + } + for _, row := range rows { + for _, prefix := range [][]byte{{0xf0}, {0xf0, 0x66}} { + t.Run(fmt.Sprintf("%d/%s/%x", mode, row.op, prefix), func(t *testing.T) { + code := append(append([]byte(nil), prefix...), row.machine...) + original := "TEXT invalid(SB),4,$0-0\n" + for _, value := range code { + original += fmt.Sprintf("BYTE $%d\n", value) + } + goAssemblerOracleOutput(t, arch, original+"RET\n", "", true) + physical, err := x86asm.Decode(code, mode) + if err != nil { + t.Fatal("actual pinned decoder observation", err) + } + invalid := false + for _, value := range physical.Prefix { + invalid = invalid || value&x86asm.PrefixInvalid != 0 + } + if !invalid { + t.Fatalf("decoder did not identify specified LOCK invalidity: %#v", physical) + } + decoded, conversion := decodeX86RawDirectiveGroup(code, mode, 0, original, nil) + t.Logf("actual bytes=%x prefix=%#v GoSyntax=%q whole=%#v error=%v", + code, physical.Prefix, x86asm.GoSyntax(physical, 0, nil), decoded, conversion) + if conversion == nil { + t.Fatal("whole raw decoder erased a machine-invalid prefix") + } + }) + } + } + } +} + +func x86PrefixOriginalBytes(t *testing.T, mode int, code []byte) { + t.Helper() + arch := "386" + if mode == 64 { + arch = "amd64" + } + source := "TEXT prefix(SB),4,$0-0\n" + for _, value := range code { + source += fmt.Sprintf("BYTE $%d\n", value) + } + actual := x86OrderingHintGoBytes(t, arch, source+"RET\n") + want := append(append([]byte(nil), code...), 0xc3) + if !bytes.Equal(actual, want) { + t.Fatalf("original BYTE object identity %x, want %x", actual, want) + } +} + +// The rule is decoder metadata, not an opcode whitelist. Include width, +// arithmetic, control, x87 and mnemonic-alias paths through GoSyntax. +func TestX86RawInvalidPrefixEntireDecodedClass(t *testing.T) { + rows := []struct { + name string + code []byte + }{ + {"MOV", []byte{0x8b, 0xc1}}, + {"ADD register", []byte{0x01, 0xc8}}, + {"CMP memory", []byte{0x39, 0x08}}, + {"TEST memory", []byte{0x85, 0x08}}, + {"MUL", []byte{0xf7, 0xe1}}, + {"DIV", []byte{0xf7, 0xf1}}, + {"LEA", []byte{0x8d, 0x00}}, + {"NOP", []byte{0x90}}, + {"INT3", []byte{0xcc}}, + {"UD2 fault", []byte{0x0f, 0x0b}}, + {"RET", []byte{0xc3}}, + {"PACK alias", []byte{0x66, 0x0f, 0x6b, 0xca}}, + {"FINIT alias", []byte{0xdb, 0xe3}}, + } + for _, mode := range []int{32, 64} { + for _, row := range rows { + for _, prefix := range [][]byte{{0xf0}, {0xf0, 0x66}, {0x66, 0xf0}, {0xf0, 0xf0}} { + t.Run(fmt.Sprintf("%d/%s/%x", mode, row.name, prefix), func(t *testing.T) { + code := append(append([]byte(nil), prefix...), row.code...) + x86PrefixOriginalBytes(t, mode, code) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("complete original decode %+v: %v", physical, err) + } + invalid := false + for _, value := range physical.Prefix { + invalid = invalid || value&x86asm.PrefixInvalid != 0 + } + if !invalid { + t.Fatal("actual original bytes lack the expected invalid LOCK metadata") + } + if syntax, err := decodedX86GoSyntax(physical, code); err == nil { + t.Fatalf("invalid metadata became legal syntax %q", syntax) + } + if got, err := decodeX86RawDirectiveGroup(code, mode, 0, row.name, nil); err == nil { + t.Fatalf("invalid metadata escaped whole decoding: %+v", got) + } + }) + } + } + } +} + +func TestX86RawIgnoredPrefixesRemainDistinctFromInvalid(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + name string + code []byte + want Op + }{ + {"duplicate operand override", []byte{0x66, 0x66, 0x89, 0xc8}, "MOVW"}, + {"duplicate address override", []byte{0x67, 0x67, 0x0f, 0x31}, "RDTSC"}, + {"ignored operand override", []byte{0x66, 0x66, 0x0f, 0x31}, "RDTSC"}, + } { + t.Run(fmt.Sprintf("%d/%s", mode, row.name), func(t *testing.T) { + x86PrefixOriginalBytes(t, mode, row.code) + physical, err := x86asm.Decode(row.code, mode) + if err != nil || physical.Len != len(row.code) { + t.Fatalf("complete original decode %+v: %v", physical, err) + } + ignored := false + for _, prefix := range physical.Prefix { + if prefix&x86asm.PrefixInvalid != 0 { + t.Fatal("legal control has invalid metadata") + } + ignored = ignored || prefix&x86asm.PrefixIgnored != 0 + } + if !ignored { + t.Fatal("control does not observe PrefixIgnored") + } + got, err := decodeX86RawDirectiveGroup(row.code, mode, 0, row.name, nil) + if err != nil || len(got) != 1 || got[0].Op != row.want { + t.Fatalf("legal ignored prefix changed: %+v, %v", got, err) + } + }) + } + } +} + +func TestX86RawValidLockRequiresItsWholeAtomicEffect(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + name string + code []byte + }{ + {"ADD", []byte{0x01, 0x08}}, + {"AND", []byte{0x21, 0x08}}, + {"XADD", []byte{0x0f, 0xc1, 0x08}}, + {"XCHG", []byte{0x87, 0x08}}, + } { + for _, prefix := range [][]byte{{0xf0}, {0xf0, 0xf0}} { + t.Run(fmt.Sprintf("%d/%s/%x", mode, row.name, prefix), func(t *testing.T) { + code := append(append([]byte(nil), prefix...), row.code...) + x86PrefixOriginalBytes(t, mode, code) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("complete original decode %+v: %v", physical, err) + } + for _, value := range physical.Prefix { + if value&x86asm.PrefixInvalid != 0 { + t.Fatal("actual legitimate LOCK classified as machine invalid") + } + } + got, err := decodeX86RawDirectiveGroup(code, mode, 0, row.name, nil) + t.Logf("bytes=%x syntax=%q parsed=%+v error=%v", code, + x86asm.GoSyntax(physical, 0, nil), got, err) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("raw atomic-prefix effect is not bound; may not become a standalone LOCK no-op: %+v/%v", got, err) + } + }) + } + } + } +} diff --git a/amd64_raw_klogical.go b/amd64_raw_klogical.go new file mode 100644 index 000000000..c22cef5ce --- /dev/null +++ b/amd64_raw_klogical.go @@ -0,0 +1,110 @@ +package plan9asm + +import "fmt" + +// decodedX86RawKLogicalInstruction models Go 1.27's _ykaddb and _yknotb +// mask-register tables as opcode, width prefix, and K-operand arity. It +// includes KADD, KAND/ANDN, KOR, KXNOR/XOR, KUNPCK, KNOT, KTEST and KORTEST. +func decodedX86RawKLogicalInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.evex || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + stem, arity := x86RawKLogicalOpcode(p.opcode) + if stem == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("K-register operation: %s", message) + } + if p.pp > 1 { + return fail("reserved VEX.pp") + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride || p.segment != "" || p.r != 0 || p.b != 0 || p.x != 0 { + return fail("invalid address or register extension prefix") + } + length := 0 + if arity == 3 { + length = 1 + } + if p.vectorLength != length || p.upper >= 8 || arity == 2 && p.upper != 0 { + return fail("invalid VEX length or vvvv K register") + } + op, ok := x86RawKLogicalOperation(stem, p.pp, p.w) + if !ok { + return fail("reserved width or opcode combination") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + if modRM>>6 != 3 { + return fail("K-register operation requires register operands") + } + source1 := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM&7))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM>>3&7))} + args := []Operand{source1} + if arity == 3 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.upper))}) + } + args = append(args, destination) + raw := fmt.Sprintf("%s %s, %s", op, source1.String(), destination.String()) + if arity == 3 { + raw = fmt.Sprintf("%s %s, %s, %s", op, source1.String(), args[1].String(), destination.String()) + } + return Instr{Op: op, Args: args, Raw: raw}, p.modRM + 1, true, nil +} + +func x86RawKLogicalOpcode(opcode int) (stem string, arity int) { + switch opcode { + case 0x4a: + return "KADD", 3 + case 0x41: + return "KAND", 3 + case 0x42: + return "KANDN", 3 + case 0x45: + return "KOR", 3 + case 0x46: + return "KXNOR", 3 + case 0x47: + return "KXOR", 3 + case 0x4b: + return "KUNPCK", 3 + case 0x44: + return "KNOT", 2 + case 0x98: + return "KORTEST", 2 + case 0x99: + return "KTEST", 2 + } + return "", 0 +} + +func x86RawKLogicalOperation(stem string, pp int, w bool) (Op, bool) { + if stem == "KUNPCK" { + switch { + case pp == 1 && !w: + return "KUNPCKBW", true + case pp == 0 && !w: + return "KUNPCKWD", true + case pp == 0 && w: + return "KUNPCKDQ", true + } + return "", false + } + suffix := "W" + if pp == 1 { + suffix = "B" + } + if w { + suffix = "Q" + if pp == 1 { + suffix = "D" + } + } + return Op(stem + suffix), true +} diff --git a/amd64_raw_klogical_test.go b/amd64_raw_klogical_test.go new file mode 100644 index 000000000..2bd6983ac --- /dev/null +++ b/amd64_raw_klogical_test.go @@ -0,0 +1,192 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type x86RawKLogicalRow struct { + opcode byte + pp byte + w bool + name Op + ary int +} + +func x86RawKLogicalRows() []x86RawKLogicalRow { + var rows []x86RawKLogicalRow + for opcode, stem := range map[byte]string{ + 0x4a: "KADD", 0x41: "KAND", 0x42: "KANDN", 0x45: "KOR", + 0x46: "KXNOR", 0x47: "KXOR", 0x44: "KNOT", + 0x98: "KORTEST", 0x99: "KTEST", + } { + arity := 3 + if opcode == 0x44 || opcode == 0x98 || opcode == 0x99 { + arity = 2 + } + for _, width := range []struct { + pp byte + w bool + suffix string + }{ + {pp: 1, suffix: "B"}, {pp: 0, suffix: "W"}, + {pp: 1, w: true, suffix: "D"}, {pp: 0, w: true, suffix: "Q"}, + } { + rows = append(rows, x86RawKLogicalRow{opcode, width.pp, width.w, Op(stem + width.suffix), arity}) + } + } + rows = append(rows, + x86RawKLogicalRow{0x4b, 1, false, "KUNPCKBW", 3}, + x86RawKLogicalRow{0x4b, 0, false, "KUNPCKWD", 3}, + x86RawKLogicalRow{0x4b, 0, true, "KUNPCKDQ", 3}, + ) + return rows +} + +func encodeX86RawKLogical(row x86RawKLogicalRow, source1, source2, destination int) []byte { + length := byte(0) + if row.ary == 3 { + length = 4 + } + vvvv := 0 + if row.ary == 3 { + vvvv = source2 + } + p1 := byte((^vvvv&15)<<3) | length | row.pp + modRM := byte(0xc0 | destination<<3 | source1) + if row.w { + return []byte{0xc4, 0xe1, p1 | 0x80, row.opcode, modRM} + } + return []byte{0xc5, p1 | 0x80, row.opcode, modRM} +} + +func TestDecodedX86RawKLogicalCompleteGo127Family(t *testing.T) { + rows := x86RawKLogicalRows() + if len(rows) != 39 { + t.Fatalf("modeled %d Go 1.27 K logical/arithmetic/unpack forms, want 39", len(rows)) + } + for _, row := range rows { + if _, ok := amd64MaskLogicalSpecs[row.name]; !ok { + if _, ok := amd64MaskArithmeticSpecs[row.name]; !ok { + if _, ok := amd64MaskUnpackBits[row.name]; !ok { + t.Fatalf("%s has no lowering grammar", row.name) + } + } + } + for source1 := 0; source1 < 8; source1++ { + for source2 := 0; source2 < 8; source2++ { + if row.ary == 2 && source2 != 0 { + continue + } + for destination := 0; destination < 8; destination++ { + code := encodeX86RawKLogical(row, source1, source2, destination) + got, length, ok, err := decodedX86RawKLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != row.name || len(got.Args) != row.ary { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s", code, got, length, ok, err, row.name) + } + want := fmt.Sprintf("%s K%d, K%d", row.name, source1, destination) + if row.ary == 3 { + want = fmt.Sprintf("%s K%d, K%d, K%d", row.name, source1, source2, destination) + } + if got.Raw != want { + t.Fatalf("decode %x = %q, want %q", code, got.Raw, want) + } + if _, _, ok, err := decodedX86RawKLogicalInstruction(code, 32); !ok || err != nil { + t.Fatalf("386 decode %x: ok=%v err=%v", code, ok, err) + } + } + } + } + } +} + +func TestTranslateRawKLogicalArgReduceAVX512Regression(t *testing.T) { + const source = "TEXT rawKORW(SB),$0-0\n\tLONG $0xc945fcc5\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{ + "rawKORW": {Name: "rawKORW", Ret: Void}, + }}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "or i64") { + t.Fatalf("KORW did not lower to mask OR:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "raw-korw.ll", "raw-korw.o", ir) + }) + } +} + +func TestTranslateRawKLogicalCompleteGo127FamilyObjects(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawKLogicalFamily(SB),$0-0\n") + for _, row := range x86RawKLogicalRows() { + for _, value := range encodeX86RawKLogical(row, 1, 2, 3) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawKLogicalFamily": {Name: "rawKLogicalFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-klogical-family.ll", "raw-klogical-family.o", ir) + }) + } +} + +func TestDecodedX86RawKLogicalRejectsReservedForms(t *testing.T) { + row := x86RawKLogicalRow{0x45, 0, false, "KORW", 3} + base := encodeX86RawKLogical(row, 1, 2, 3) + for name, code := range map[string][]byte{ + "length": {base[0], base[1] &^ 4, base[2], base[3]}, + "prefix": {base[0], base[1] | 2, base[2], base[3]}, + "high vvvv": {base[0], base[1] &^ 0x40, base[2], base[3]}, + "memory source": {base[0], base[1], base[2], base[3] &^ 0xc0}, + "address override": append([]byte{0x67}, base...), + "missing ModRM": base[:3], + } { + t.Run(name, func(t *testing.T) { + if _, _, ok, err := decodedX86RawKLogicalInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved K encoding %x: ok=%v err=%v", code, ok, err) + } + }) + } +} diff --git a/amd64_raw_kshift.go b/amd64_raw_kshift.go new file mode 100644 index 000000000..ffa78ae57 --- /dev/null +++ b/amd64_raw_kshift.go @@ -0,0 +1,59 @@ +package plan9asm + +import "fmt" + +// Go 1.27 _ykshiftlb uses opcode 30/31 for right and 32/33 for left; +// VEX.W distinguishes B/W or D/Q at each opcode. +var x86RawMaskShiftOps = [4][2]Op{ + {"KSHIFTRB", "KSHIFTRW"}, + {"KSHIFTRD", "KSHIFTRQ"}, + {"KSHIFTLB", "KSHIFTLW"}, + {"KSHIFTLD", "KSHIFTLQ"}, +} + +func decodedX86RawMaskShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.evex || p.mapNumber != 3 || p.pp != 1 || p.opcode < 0x30 || p.opcode > 0x33 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("mask shift: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength != 0 || p.upper != 0 || p.r != 0 || p.b != 0 || p.x != 0 { + return fail("invalid reserved VEX bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + if modRM>>6 != 3 { + return fail("KSHIFT requires a K-register source") + } + if len(code) <= p.modRM+1 { + return fail("missing imm8 shift count") + } + w := 0 + if p.w { + w = 1 + } + op := x86RawMaskShiftOps[p.opcode-0x30][w] + count := code[p.modRM+1] + source := modRM & 7 + destination := modRM >> 3 & 7 + args := []Operand{ + {Kind: OpImm, Imm: int64(count)}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", source))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", destination))}, + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s $%d, K%d, K%d", op, count, source, destination), + }, p.modRM + 2, true, nil +} diff --git a/amd64_raw_kshift_test.go b/amd64_raw_kshift_test.go new file mode 100644 index 000000000..4c843ea68 --- /dev/null +++ b/amd64_raw_kshift_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawMaskShift(opcode byte, w bool, source, destination int, count byte) []byte { + p1 := byte(0x79) + if w { + p1 |= 0x80 + } + return []byte{0xc4, 0xe3, p1, opcode, byte(0xc0 | destination<<3 | source), count} +} + +func TestTranslateRawMaskShiftGorseRegression(t *testing.T) { + const source = `TEXT rawMaskShift(SB), $0-0 + LONG $0x3179e3c4 + WORD $0x10d1 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawMaskShift": {Name: "rawMaskShift", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "lshr i64") { + t.Fatalf("raw KSHIFTRD omitted mask shift:\n%s", ir) + } +} + +func TestDecodedX86RawMaskShiftCompleteGoRows(t *testing.T) { + if len(x86RawMaskShiftOps) != 4 { + t.Fatal("Go 1.27 KSHIFT opcode family should have four direction/width rows") + } + seen := make(map[Op]bool) + for row, widths := range x86RawMaskShiftOps { + for width, op := range widths { + spec, ok := amd64MaskArithmeticSpecs[op] + if !ok || seen[op] || spec.shift == "" { + t.Fatalf("KSHIFT row %d/%d = %s, spec %+v", row, width, op, spec) + } + seen[op] = true + for source := 0; source < 8; source++ { + for destination := 0; destination < 8; destination++ { + for _, count := range []byte{0, 7, 8, 16, 32, 64, 255} { + code := encodeX86RawMaskShift(byte(0x30+row), width != 0, source, destination, count) + got, length, recognized, err := decodedX86RawMaskShiftInstruction(code, 64) + if err != nil || !recognized || length != len(code) || got.Op != op { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + if len(got.Args) != 3 || got.Args[0].Imm != int64(count) || + got.Args[1].String() != fmt.Sprintf("K%d", source) || + got.Args[2].String() != fmt.Sprintf("K%d", destination) { + t.Fatalf("decode %x = %+v, want $%d, K%d, K%d", code, got, count, source, destination) + } + if _, _, recognized, err := decodedX86RawMaskShiftInstruction(code, 32); !recognized || err != nil { + t.Fatalf("386 decode %x: recognized=%v err=%v", code, recognized, err) + } + } + } + } + } + } + if len(seen) != 8 { + t.Fatalf("covered %d KSHIFT opcodes, want eight", len(seen)) + } +} + +func TestDecodedX86RawMaskShiftRejectsInvalidForms(t *testing.T) { + base := encodeX86RawMaskShift(0x31, false, 1, 2, 16) + for _, test := range []struct { + name string + code []byte + }{ + {"VEX length", []byte{base[0], base[1], base[2] | 4, base[3], base[4], base[5]}}, + {"vvvv", []byte{base[0], base[1], base[2] &^ 8, base[3], base[4], base[5]}}, + {"extended register", []byte{base[0], base[1] &^ 0x20, base[2], base[3], base[4], base[5]}}, + {"memory source", []byte{base[0], base[1], base[2], base[3], base[4] &^ 0xc0, base[5]}}, + {"missing immediate", base[:5]}, + {"missing ModRM", base[:4]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawMaskShiftInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } +} + +func TestTranslateRawMaskShiftLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawMaskShiftFamily(SB), $0-0\n") + for row, widths := range x86RawMaskShiftOps { + for width := range widths { + for _, count := range []byte{0, 16, 255} { + code := encodeX86RawMaskShift(byte(0x30+row), width != 0, 1, 2, count) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawMaskShiftFamily": {Name: "rawMaskShiftFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-mask-shift.ll", "raw-mask-shift.o", ir) + }) + } +} diff --git a/amd64_raw_lea_address_test.go b/amd64_raw_lea_address_test.go new file mode 100644 index 000000000..1938f3006 --- /dev/null +++ b/amd64_raw_lea_address_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawDisplacementLEAAddressSizeIdentity(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, addressBits := range []int{mode, mode / 2} { + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + t.Run(fmt.Sprintf("mode%d/address%d/width%d", mode, addressBits, width), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, width, addressBits, 0, 0, -64) + instruction := x86RawDisplacementDecodedLEA(t, mode, encoding) + want := 0 + if addressBits != mode { + want = addressBits + } + if instruction.x86AddressBits != want { + t.Fatalf("raw LEA lost physical address size: got %d, want %d", instruction.x86AddressBits, want) + } + }) + } + } + } +} + +func TestX86RawDisplacementLEANarrowAddressFiveObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + t.Run(fmt.Sprintf("%s/width%d", target.triple, width), func(t *testing.T) { + encoding := x86RawLEADisplacementEncoding(mode, width, mode/2, 0, 0, -64) + body := "@MOV x+0(FP),AX\n@MOV AX,BX\n@MOV $3,SI\n@MOV $0,CX\n" + + x86RawDisplacementDirectives(encoding) + "@MOV CX,ret+@WORD(FP)\nRET" + source, sig, _ := x86SourceStackNumericalFixture(target.arch, body) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + t.Fatal(err) + } + want := "zext i32" + if mode == 32 { + want = "zext i16" + } + if !strings.Contains(ir, want) { + t.Fatalf("narrow effective offset was not zero-extended before destination: %s", want) + } + compileLLVMToObject(t, llc, target.triple, "raw-lea-address.ll", "raw-lea-address.o", ir) + }) + } + } +} + +func TestX86RawDisplacementLEARejectsUnboundAddressMarkers(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, bits := range []int{1, 8, 16, 32, 64, 128} { + if target.arch == "386" && bits == 16 || target.arch == "amd64" && bits == 32 { + continue + } + t.Run(fmt.Sprintf("%s/%d", target.triple, bits), func(t *testing.T) { + instruction := Instr{Op: "LEAL", x86AddressBits: bits, Args: []Operand{ + {Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: CX}, + }} + if _, err := x86LEAOperandWidth(instruction, target.arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("forged physical address size gained a grammar: %v", err) + } + ctx, output := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{instruction}}, FuncSig{Ret: Void}, nil) + ctx.goarch = target.arch + before := output.String() + if ok, terminated, err := ctx.lowerArith("LEAL", instruction); !ok || terminated || !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatalf("invalid raw address marker emitted IR: (%v, %v, %v)", ok, terminated, err) + } + }) + } + } +} + +func TestX86RawDisplacementLEARelativeAndTruncatedRemainClosed(t *testing.T) { + for _, encoding := range [][]byte{ + {0x48, 0x8d, 0x0d, 0xc0, 0xff, 0xff, 0xff}, + {0x67, 0x48, 0x8d, 0x0d, 0xc0, 0xff, 0xff, 0xff}, + {0x48, 0x8d}, {0x48, 0x8d, 0x0c}, + {0x48, 0x8d, 0x0c, 0x25, 0xc0, 0xff, 0xff}, + } { + t.Run(fmt.Sprintf("%x", encoding), func(t *testing.T) { + file, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+x86RawDisplacementDirectives(encoding)+"RET\n") + if err != nil { + t.Fatal(err) + } + if _, err := decodeX86RawDirectives(file.Funcs[0], "amd64"); err == nil { + t.Fatal("unbound relative target or truncated LEA became a numerical address") + } + }) + } +} + +func TestX86RawDisplacementLEAIgnoresSegmentBase(t *testing.T) { + for _, prefix := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65} { + t.Run(fmt.Sprintf("prefix%x", prefix), func(t *testing.T) { + encoding := append([]byte{prefix}, x86RawLEADisplacementEncoding(64, 64, 64, 0, 0, -64)...) + instruction := x86RawDisplacementDecodedLEA(t, 64, encoding) + memory := instruction.Args[0].Mem + if memory.Segment != "" || memory.Base != AX || memory.Off != -64 { + t.Fatalf("LEA used a segment base instead of its effective offset: %+v", memory) + } + }) + } +} + +func TestX86RawDisplacementEIPCannotBorrowRIPRelocation(t *testing.T) { + for _, opcode := range []byte{0x8d, 0x8b, 0x03} { + t.Run(fmt.Sprintf("opcode%x", opcode), func(t *testing.T) { + encoding := []byte{0x67, 0x48, opcode, 0x0d, 0xc0, 0xff, 0xff, 0xff} + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Len != len(encoding) || inst.AddrSize != 32 || inst.PCRel != 0 { + t.Fatalf("EIP decoder witness changed: %+v, %v", inst, err) + } + if syntax, err := decodedX86GoSyntax(inst, encoding); !errors.Is(err, ErrProbeNeedsContext) || syntax != "" { + t.Fatalf("unbound EIP-relative address acquired a source relocation: %q, %v", syntax, err) + } + }) + } +} + +func TestX86RawDisplacementNonLEASegmentOverridesStayPhysical(t *testing.T) { + for _, prefix := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65} { + t.Run(fmt.Sprintf("prefix%x", prefix), func(t *testing.T) { + encoding := []byte{prefix, 0x48, 0x8b, 0x80, 0xc0, 0xff, 0xff, 0xff} + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Len != len(encoding) || inst.Op != x86asm.MOV { + t.Fatalf("physical segmented MOV witness: %+v, %v", inst, err) + } + if got := normalizeDecodedX86LEASegments(inst); !reflect.DeepEqual(got, inst) { + t.Fatal("LEA normalization stripped an actual load's segment") + } + original, ok := inst.Args[1].(x86asm.Mem) + // In 64-bit mode x/arch already ignores ES/CS/SS/DS. FS and + // GS still carry a physical segment base and must not be erased. + if !ok || (prefix == 0x64 || prefix == 0x65) && original.Segment == 0 { + t.Fatalf("physical MOV lost its segment: %+v", inst) + } + normalized := normalizeDecodedX86MemoryDisplacements(inst) + memory := normalized.Args[1].(x86asm.Mem) + if memory.Segment != original.Segment || memory.Base != original.Base || memory.Disp != -64 { + t.Fatalf("signed displacement changed actual memory identity: %+v", memory) + } + }) + } +} + +func TestX86RawDisplacementNarrowLEADoesNotGrantCallerPointerOrStack(t *testing.T) { + for _, modRM := range []byte{0x87, 0x85} { // DI caller pointer, BP private stack. + t.Run(fmt.Sprintf("base%x", modRM), func(t *testing.T) { + encoding := []byte{0x67, 0x48, 0x8d, modRM, 0xc0, 0xff, 0xff, 0xff} + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", + x86RawDisplacementDirectives(encoding)+"MOVL AX,(DX)", 1) + requireLEANativeGoObject(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("narrow physical address borrowed native caller/stack identity: %v", err) + } + }) + } +} diff --git a/amd64_raw_lea_runtime_test.go b/amd64_raw_lea_runtime_test.go new file mode 100644 index 000000000..c94989d94 --- /dev/null +++ b/amd64_raw_lea_runtime_test.go @@ -0,0 +1,244 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +type x86RawLEAAddressRuntimeForm struct { + name string + addressBits, width, shape int + scale int + alias bool +} + +func x86RawLEAAddressRuntimeForms(arch string) []x86RawLEAAddressRuntimeForm { + mode := 64 + if arch == "386" { + mode = 32 + } + var forms []x86RawLEAAddressRuntimeForm + for _, addressBits := range []int{mode, mode / 2} { + for _, width := range []int{16, 32, 64} { + if width > mode { + continue + } + for _, shape := range []int{0, 1, 2, 3} { + if addressBits == 16 && shape != 0 { + continue // The physical fixture here is BX+SI, not a SIB. + } + for scale := 0; scale < 4; scale++ { + if (shape == 0 || shape == 3) && scale != 0 { + continue + } + for _, alias := range []bool{false, true} { + name := fmt.Sprintf("leaA%dW%dS%dC%d", addressBits, width, shape, scale) + if alias { + name += "Alias" + } + forms = append(forms, x86RawLEAAddressRuntimeForm{ + name: name, addressBits: addressBits, width: width, + shape: shape, scale: scale, alias: alias, + }) + } + } + } + } + } + return forms +} + +func x86RawLEAAddressRuntimeSources(arch string) (string, string, string, string, map[string]FuncSig) { + mode, word, move, typ, ctype, cmax := 64, 8, "MOVQ", "uint64", "uint64_t", "UINT64_MAX" + seed := uint64(0x1122334455667788) + if arch == "386" { + mode, word, move, typ, ctype, cmax = 32, 4, "MOVL", "uint32", "uint32_t", "UINT32_MAX" + seed = 0x55667788 + } + var declarations, assembly, goTest, c strings.Builder + declarations.WriteString("package rawlea\n") + fmt.Fprintf(&goTest, "package rawlea\nimport \"testing\"\nfunc TestPhysicalAddressMathematics(t *testing.T) {\n"+ + " values := []%s{0,1,63,64,65,65535,65536,1<<31-1,1<<31,1<<32-1,^%s(0)}\n", typ, typ) + if mode == 64 { + goTest.WriteString(" values = append(values,1<<32,1<<63,0x1122334455667788)\n") + } + c.WriteString("#include \n#if defined(__APPLE__)\n#define LABEL(name) \"_\" name\n#else\n#define LABEL(name) name\n#endif\n") + sigs := make(map[string]FuncSig) + for _, form := range x86RawLEAAddressRuntimeForms(arch) { + fmt.Fprintf(&declarations, "func %s(x %s) %s\n", form.name, typ, typ) + encoding := x86RawLEADisplacementEncoding(mode, form.width, form.addressBits, form.shape, form.scale, -64) + destination, initial := "CX", fmt.Sprintf("0x%x", seed) + if form.alias { + destination, initial = "AX", "x" + register := byte(0) + if form.addressBits == 16 { + destination, register = "BX", 3 + } + opcode := bytes.IndexByte(encoding, 0x8d) + encoding[opcode+1] = encoding[opcode+1]&^0x38 | register<<3 + } + fmt.Fprintf(&assembly, "TEXT ·%s(SB),4,$0-%d\n%s x+0(FP),AX\n%s $3,BX\n%s $0x%x,CX\n", + form.name, 2*word, move, move, move, seed) + if form.addressBits == 16 { + fmt.Fprintf(&assembly, "%s AX,BX\n%s $3,SI\n", move, move) + } + assembly.WriteString(x86RawDisplacementDirectives(encoding)) + fmt.Fprintf(&assembly, "%s %s,ret+%d(FP)\nRET\n", move, destination, word) + sig := crossUnarySig(arch, "rawlea."+form.name) + sigs[sig.Name] = sig + + // These two independent scalar oracles model effective-address width + // first and destination width second, never decoded IR or GoSyntax. + effective := "x - 64" + if form.addressBits == 16 { + effective = "x + 3 - 64" + } else if form.shape == 1 { + effective = fmt.Sprintf("x + 3*%d - 64", 1<= 8 { + code = append(code, 0x44) + } + code = append(code, 0x0f, opcode, 0x05, 1, 0, 0, 0, 0xc3) + for i := 0; i < width; i++ { + code = append(code, byte(i*31+7)) + } + return code +} + +func TestDecodeX86RawLegacyScalarMoveConstantCompleteFamily(t *testing.T) { + for _, form := range []struct { + op Op + prefix byte + width int + }{ + {"MOVSS", 0xf3, 4}, + {"MOVSD", 0xf2, 8}, + } { + for _, register := range []int{0, 8} { + t.Run(fmt.Sprintf("%s/X%d", form.op, register), func(t *testing.T) { + code := x86RawLegacySIMDMoveConstant(form.prefix, 0x10, register, form.width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, string(form.op), map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.op || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[0].Args[1].String() != fmt.Sprintf("X%d", register) || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, form.op) + } + }) + } + } +} + +func TestTranslateX86RawLegacyScalarMoveConstantObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, form := range []struct { + prefix byte + width int + }{ + {0xf3, 4}, {0xf2, 8}, + } { + code := x86RawLegacySIMDMoveConstant(form.prefix, 0x10, 8, form.width) + name := fmt.Sprintf("legacyScalarMoveConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 2 || len(normalized.Data[0].Payload) != 4 || len(normalized.Data[1].Payload) != 8 { + t.Fatalf("materialized wrong scalar constants: %#v", normalized.Data) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "legacy-scalar-move-literal.ll", "legacy-scalar-move-literal.o", ir) + }) + } +} + +func TestDecodeX86RawLegacyPackedMoveConstantCompleteFamily(t *testing.T) { + for _, form := range x86RawLegacyPackedMoveForms { + for _, register := range []int{0, 8} { + t.Run(fmt.Sprintf("%s/X%d", form.name, register), func(t *testing.T) { + code := x86RawLegacyPackedMoveConstantRegister(form.prefix, form.opcode, register) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, string(form.name), map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.name || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[0].Args[1].String() != fmt.Sprintf("X%d", register) || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, form.name) + } + }) + } + } +} + +func TestDecodeX86RawLegacyPackedMoveConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawLegacyPackedMoveConstant(0, 0x10) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[3] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[3] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawLegacyPackedMoveConstantObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, form := range x86RawLegacyPackedMoveForms { + code := x86RawLegacyPackedMoveConstantRegister(form.prefix, form.opcode, 8) + name := fmt.Sprintf("legacyPackedMoveConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + pools = append(pools, code[len(code)-16:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "legacy-packed-move-literal.ll", "legacy-packed-move-literal.o", ir) + }) + } +} diff --git a/amd64_raw_legacy_scalar_rip.go b/amd64_raw_legacy_scalar_rip.go new file mode 100644 index 000000000..b6ae2b74d --- /dev/null +++ b/amd64_raw_legacy_scalar_rip.go @@ -0,0 +1,70 @@ +package plan9asm + +import ( + "fmt" + "strings" + + "golang.org/x/arch/x86/x86asm" +) + +// These Go legacy SSE scalar families read a fixed-width RIP-relative source +// and update only a vector register or flags. A source-local constant can be +// materialized without losing an in-place memory write or relocation. +var x86RawLegacyScalarReadOps = map[Op]bool{ + "MOVL": true, "MOVQ": true, "MOVLQSX": true, + "UCOMISD": true, "UCOMISS": true, "COMISD": true, "COMISS": true, + "CMPSS": true, "CMPSD": true, + "ADDSS": true, "ADDSD": true, "SUBSS": true, "SUBSD": true, + "MULSS": true, "MULSD": true, "DIVSS": true, "DIVSD": true, + "MINSS": true, "MINSD": true, "MAXSS": true, "MAXSD": true, + "SQRTSS": true, "SQRTSD": true, +} + +func decodeX86RawLegacyScalarRIPData( + code []byte, offset int, inst x86asm.Inst, syntax string, +) (Instr, x86RawLiteralRange, bool, error) { + if inst.Mode != 64 || inst.AddrSize != 64 || inst.PCRel != 4 || + inst.MemBytes != 4 && inst.MemBytes != 8 { + return Instr{}, x86RawLiteralRange{}, false, nil + } + op := Op(strings.Fields(syntax)[0]) + if !x86RawLegacyScalarReadOps[op] { + return Instr{}, x86RawLiteralRange{}, false, nil + } + if _, ok := inst.Args[0].(x86asm.Reg); !ok { + return Instr{}, x86RawLiteralRange{}, false, nil + } + mem, ok := inst.Args[1].(x86asm.Mem) + if !ok || mem.Base != x86asm.RIP { + return Instr{}, x86RawLiteralRange{}, false, nil + } + if len(code) <= offset || code[offset] == 0x64 || code[offset] == 0x65 || code[offset] == 0x67 { + return Instr{}, x86RawLiteralRange{}, false, nil + } + modRMIndex := offset + inst.PCRelOff - 1 + if modRMIndex < offset || modRMIndex >= len(code) || code[modRMIndex]&0xc7 != 0x05 { + return Instr{}, x86RawLiteralRange{}, false, nil + } + trailing := inst.Len - inst.PCRelOff - inst.PCRel + if trailing != 0 && (op != "CMPSS" && op != "CMPSD" || trailing != 1) { + return Instr{}, x86RawLiteralRange{}, false, nil + } + patched, data, literal, err := x86RawRIPBytesWithSuffix(code, offset, modRMIndex, inst.MemBytes, trailing) + if err != nil { + return Instr{}, x86RawLiteralRange{}, true, err + } + physical, err := x86asm.Decode(patched, 64) + if err != nil || physical.Len != len(patched) { + return Instr{}, x86RawLiteralRange{}, true, fmt.Errorf("patched legacy scalar encoding is invalid: %v", err) + } + patchedSyntax, err := decodedX86GoSyntax(physical, patched) + if err != nil { + return Instr{}, x86RawLiteralRange{}, true, err + } + instrs, err := parseDecodedX86Instruction(patchedSyntax) + if err != nil || len(instrs) != 1 || len(instrs[0].Args) < 2 || instrs[0].Args[0].Kind != OpMem { + return Instr{}, x86RawLiteralRange{}, true, fmt.Errorf("patched legacy scalar instruction %q has no memory source: %v", patchedSyntax, err) + } + setX86RawRIPDataOperand(&instrs[0], 0, data) + return instrs[0], literal, true, nil +} diff --git a/amd64_raw_legacy_scalar_rip_test.go b/amd64_raw_legacy_scalar_rip_test.go new file mode 100644 index 000000000..a24daa552 --- /dev/null +++ b/amd64_raw_legacy_scalar_rip_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawLegacyScalarSourceLocalRIPData(t *testing.T) { + for _, op := range []string{ + "MOVD", "UCOMISD", "UCOMISS", "COMISD", "COMISS", + "CMPSS", "CMPSD", + "ADDSS", "ADDSD", "SUBSS", "SUBSD", "MULSS", "MULSD", + "DIVSS", "DIVSD", "MINSS", "MINSD", "MAXSS", "MAXSD", + "SQRTSS", "SQRTSD", + } { + for _, destination := range []string{"X0", "X10"} { + t.Run(op+"/"+destination, func(t *testing.T) { + instruction := op + " pool(SB), " + destination + if op == "CMPSS" || op == "CMPSD" { + instruction += ", $1" + } + code := assembleX87ControlBytes(t, "amd64", + "TEXT scalarRIP(SB),4,$0-0\n\t"+instruction+"\n\tRET\n") + inst, err := x86asm.Decode(code, 64) + if err != nil || inst.PCRel != 4 || inst.MemBytes <= 0 || inst.Len >= len(code) { + t.Fatalf("Go assembler %s %s: %x, decode %+v, %v", op, destination, code, inst, err) + } + binary.LittleEndian.PutUint32(code[inst.PCRelOff:], 1) + for value := 0; value < inst.MemBytes; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "scalar RIP", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || len(decoded[0].x86RIPLiteralData) != inst.MemBytes { + t.Fatalf("%s %s decoded as %#v", op, destination, decoded) + } + }) + } + } +} + +func TestX86RawMOVD32PreservesPhysicalWidth(t *testing.T) { + const source = "TEXT movd32(SB),4,$0-0\n\tMOVL pool(SB), X0\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + physical, err := x86asm.Decode(code, 64) + if err != nil || physical.Op != x86asm.MOVD || physical.MemBytes != 4 { + t.Fatalf("Go MOVL to X0 encoding: %x, %+v, %v", code, physical, err) + } + binary.LittleEndian.PutUint32(code[physical.PCRelOff:], 1) + code = append(code, 1, 2, 3, 4) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "MOVD32 literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "MOVL" || + len(decoded[0].x86RIPLiteralData) != 4 { + t.Fatalf("MOVD32 decoded as %#v", decoded) + } + var raw strings.Builder + raw.WriteString("TEXT ·rawMovd32(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&raw, "\tBYTE $0x%02x\n", value) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, raw.String(), "func rawMovd32()\n", "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "load volatile i32, ptr") || + strings.Contains(ir, "load volatile i64, ptr") { + t.Fatal("physical MOVD32 must retain exactly one fault-observable 32-bit load") + } + compileLLVMToObject(t, llc, triple, "raw-movd32-"+api+".ll", "raw-movd32-"+api+".o", ir) + }) + } + }) + } +} diff --git a/amd64_raw_local_address_test.go b/amd64_raw_local_address_test.go new file mode 100644 index 000000000..054b61ff8 --- /dev/null +++ b/amd64_raw_local_address_test.go @@ -0,0 +1,150 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawLocalAddressPreservesPointerOffsets(t *testing.T) { + // Two RIP-relative LEAs address different bytes of one unreachable table. + code := []byte{ + 0x48, 0x8d, 0x05, 0x08, 0, 0, 0, // LEAQ table+0, AX + 0x48, 0x8d, 0x1d, 0x04, 0, 0, 0, // LEAQ table+3, BX + 0xc3, + 1, 2, 3, 4, 5, 6, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "local address table", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 || decoded[0].Op != "LEAQ" || decoded[1].Op != "LEAQ" || + decoded[0].Args[1].Reg != AX || decoded[1].Args[1].Reg != BX || + decoded[0].x86RIPAddressOff != 0 || decoded[1].x86RIPAddressOff != 3 { + t.Fatalf("decoded %x as %#v", code, decoded) + } + var source strings.Builder + source.WriteString("TEXT localAddress(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 1 || len(normalized.Funcs[0].Instrs) != 4 { + t.Fatalf("shared local address layout: %d globals, %d instructions", len(normalized.Data), len(normalized.Funcs[0].Instrs)) + } + firstAddress := normalized.Funcs[0].Instrs[1].Args[0].Sym + secondAddress := normalized.Funcs[0].Instrs[2].Args[0].Sym + if firstAddress != normalized.Data[0].Sym+"(SB)" || + secondAddress != normalized.Data[0].Sym+"+3(SB)" { + t.Fatalf("shared address offsets = %q, %q", firstAddress, secondAddress) + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"localAddress": {Name: "localAddress", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "raw_address") { + t.Fatal("shared local-address data global missing") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "raw-local-address.ll", "raw-local-address.o", ir) + }) + } +} + +func TestDecodeX86RawLocalAddressSharesLiteralPool(t *testing.T) { + code := []byte{ + 0x48, 0x8d, 0x05, 0x09, 0, 0, 0, // LEAQ table, AX + 0x66, 0x0f, 0x6f, 0x05, 0x01, 0, 0, 0, // MOVDQU table, X0 + 0xc3, + } + for value := byte(0); value < 16; value++ { + code = append(code, value) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "shared literal pool", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 || len(decoded[0].x86RIPAddressData) != 16 || + len(decoded[1].x86RIPAddressData) != 16 || + decoded[0].x86RIPAddressOff != 0 || decoded[1].x86RIPAddressOff != 0 { + t.Fatalf("shared pool decoded as %#v", decoded) + } + var source strings.Builder + source.WriteString("TEXT localAddressAndLiteral(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 1 || + len(normalized.Funcs[0].Instrs) != 4 || + normalized.Funcs[0].Instrs[1].Args[0].Sym != normalized.Data[0].Sym+"(SB)" || + normalized.Funcs[0].Instrs[2].Args[0].Sym != normalized.Data[0].Sym+"(SB)" { + t.Fatalf("local address and literal did not share data: %d globals, %d instructions", len(normalized.Data), len(normalized.Funcs[0].Instrs)) + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"localAddressAndLiteral": {Name: "localAddressAndLiteral", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "raw-shared-pool.ll", "raw-shared-pool.o", ir) + }) + } +} + +func TestDecodeX86RawLocalAddressRejectsReachableCode(t *testing.T) { + code := []byte{ + 0x48, 0x8d, 0x05, 0x00, 0, 0, 0, // LEAQ next instruction, AX + 0xc3, + } + _, err := decodeX86RawDirectiveGroup(code, 64, 0, "code address", map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "no preceding RET") { + t.Fatalf("address of reachable code: %v", err) + } +} + +func TestDecodeX86RawLocalAddressSignExtendsNegativeDisplacement(t *testing.T) { + // LEAQ -7(RIP), AX points to its own first byte, not byte 2^32. + code := []byte{0x48, 0x8d, 0x05, 0xf9, 0xff, 0xff, 0xff, 0xc3} + _, err := decodeX86RawDirectiveGroup(code, 64, 0, "self address", map[string]bool{}) + if err == nil || !strings.Contains(err.Error(), "no preceding RET") { + t.Fatalf("self-addressed native TEXT must fail with layout reason, got %v", err) + } +} diff --git a/amd64_raw_logical_literal_test.go b/amd64_raw_logical_literal_test.go new file mode 100644 index 000000000..ffa778c5e --- /dev/null +++ b/amd64_raw_logical_literal_test.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawPackedLogicalLiteral(op Op, evex bool, width, mask int, zeroing, broadcast bool) []byte { + var code []byte + if evex { + length := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) + code = encodeX86RawEVEXPackedLogical(op, length, mask, zeroing, broadcast, 0, 0, 0) + } else { + opcode := map[Op]byte{"VPAND": 0xdb, "VPANDN": 0xdf, "VPOR": 0xeb, "VPXOR": 0xef}[op] + code = encodeX86VEXPackedIntegerLogical(opcode, true, width == 32, false, 0, 0, 0) + } + code[len(code)-1] = 0x05 // RIP+disp32 rather than X0. + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = 4 + if strings.HasSuffix(string(op), "Q") { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*9+7)) + } + return code +} + +func TestTranslateX86RawPackedLogicalRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + add := func(name string, code []byte, width int) { + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + pools = append(pools, code[len(code)-width:]) + } + for index, op := range []Op{"VPAND", "VPANDN", "VPOR", "VPXOR"} { + add(fmt.Sprintf("vexLogical%d", index), x86RawPackedLogicalLiteral(op, false, 32, 0, false, false), 32) + } + for index, op := range []Op{ + "VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", + "VPORD", "VPORQ", "VPXORD", "VPXORQ", + } { + width := 4 + if strings.HasSuffix(string(op), "Q") { + width = 8 + } + add(fmt.Sprintf("evexLogical%d", index), x86RawPackedLogicalLiteral(op, true, 64, 7, true, true), width) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "logical-literal.ll", "logical-literal.o", ir) + }) + } +} + +func TestDecodeX86RawPackedLogicalRIPDataRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "outside_pool", edit: func(code []byte) []byte { + code[6] = 0x7f + return code + }}, + {name: "overlapping_instruction", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x64}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.edit(x86RawPackedLogicalLiteral("VPXORD", true, 64, 0, false, false)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe packed logical RIP source %x", code) + } + }) + } +} + +func TestDecodeX86RawPackedLogicalRIPDataCompleteFamily(t *testing.T) { + for _, op := range []Op{"VPAND", "VPANDN", "VPOR", "VPXOR"} { + for _, width := range []int{16, 32} { + name := fmt.Sprintf("%s/%d", op, width) + t.Run(name, func(t *testing.T) { + code := x86RawPackedLogicalLiteral(op, false, width, 0, false, false) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != op || + len(decoded[0].Args) != 3 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s literal source", code, decoded, op) + } + }) + } + } + for _, op := range []Op{ + "VPANDD", "VPANDQ", "VPANDND", "VPANDNQ", + "VPORD", "VPORQ", "VPXORD", "VPXORQ", + } { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + name := fmt.Sprintf("%s/%d/bcst%t", op, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedLogicalLiteral(op, true, width, 7, true, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := op + if broadcast { + want += ".BCST" + } + want += ".Z" + if len(decoded) != 2 || decoded[0].Op != want || + len(decoded[0].Args) != 4 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s literal source", code, decoded, want) + } + }) + } + } + } +} diff --git a/amd64_raw_mask_blend.go b/amd64_raw_mask_blend.go new file mode 100644 index 000000000..6268203b0 --- /dev/null +++ b/amd64_raw_mask_blend.go @@ -0,0 +1,41 @@ +package plan9asm + +import "fmt" + +// decodedX86RawMaskBlendInstruction covers all six Go 1.27 _yvblendmpd +// operations across EVEX X/Y/Z, optional K mask and zeroing, and permitted +// D/Q scalar broadcasts. The shared typed operand decoder handles ModRM/SIB. +func decodedX86RawMaskBlendInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 { + return Instr{}, 0, false, nil + } + var op Op + var laneBits int + switch p.opcode { + case 0x65: + if p.w { + op, laneBits = "VBLENDMPD", 64 + } else { + op, laneBits = "VBLENDMPS", 32 + } + case 0x64: + if p.w { + op, laneBits = "VPBLENDMQ", 64 + } else { + op, laneBits = "VPBLENDMD", 32 + } + case 0x66: + if p.w { + op, laneBits = "VPBLENDMW", 16 + } else { + op, laneBits = "VPBLENDMB", 8 + } + default: + return Instr{}, 0, false, nil + } + if mode == 32 && p.mask != 0 { + return Instr{}, 0, true, fmt.Errorf("386 masked %s exceeds Go's three-operand frontend", op) + } + return decodedX86BinaryVectorOperands(code, p, mode, op, laneBits) +} diff --git a/amd64_raw_mask_blend_test.go b/amd64_raw_mask_blend_test.go new file mode 100644 index 000000000..f18cb5a62 --- /dev/null +++ b/amd64_raw_mask_blend_test.go @@ -0,0 +1,212 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawMaskBlendEncoding(op Op, width, mask int, zeroing, broadcast, rip bool) []byte { + properties := map[Op]struct { + opcode byte + w bool + }{ + "VBLENDMPS": {0x65, false}, "VBLENDMPD": {0x65, true}, + "VPBLENDMD": {0x64, false}, "VPBLENDMQ": {0x64, true}, + "VPBLENDMB": {0x66, false}, "VPBLENDMW": {0x66, true}, + }[op] + p1 := byte(0x75) // source 2 is register 1, fixed bit and pp=66. + if properties.w { + p1 |= 0x80 + } + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // register 3, destination register 2. + if broadcast { + modRM = 0x13 // 0(BX), destination register 2. + } + if rip { + modRM = 0x15 // RIP+disp32, destination register 2. + } + code := []byte{0x62, 0xf2, p1, p2, properties.opcode, modRM} + if rip { + code = append(code, 1, 0, 0, 0) + } + return code +} + +func TestDecodeX86RawMaskBlend386FrontendBoundary(t *testing.T) { + plain := x86RawMaskBlendEncoding("VBLENDMPS", 16, 0, false, false, false) + if _, _, ok, err := decodedX86RawMaskBlendInstruction(plain, 32); !ok || err != nil { + t.Fatalf("386 unmasked EVEX form: ok=%v err=%v", ok, err) + } + masked := x86RawMaskBlendEncoding("VBLENDMPS", 16, 1, false, false, false) + if _, _, ok, err := decodedX86RawMaskBlendInstruction(masked, 32); !ok || err == nil { + t.Fatalf("386 masked EVEX form accepted: ok=%v err=%v", ok, err) + } +} + +func x86RawMaskBlendLiteral(op Op, width, mask int, zeroing, broadcast bool) []byte { + code := x86RawMaskBlendEncoding(op, width, mask, zeroing, broadcast, true) + code = append(code, 0xc3) + dataWidth := width + if broadcast { + dataWidth = amd64MaskBlendSpecs[op].laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*7+5)) + } + return code +} + +func TestDecodeX86RawMaskBlendCompleteGo127Family(t *testing.T) { + if len(amd64MaskBlendSpecs) != 6 { + t.Fatalf("modeled %d mask-blend operations, want six", len(amd64MaskBlendSpecs)) + } + for op, spec := range amd64MaskBlendSpecs { + for _, width := range []int{16, 32, 64} { + for _, masking := range []struct { + mask int + zero bool + }{ + {}, {mask: 5}, {mask: 7, zero: true}, + } { + for _, broadcast := range []bool{false, true} { + if broadcast && !spec.broadcast { + continue + } + code := x86RawMaskBlendEncoding(op, width, masking.mask, masking.zero, broadcast, false) + got, length, ok, err := decodedX86RawMaskBlendInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %s %x = %+v, length=%d, ok=%v, err=%v", op, code, got, length, ok, err) + } + want := op + if broadcast { + want += ".BCST" + } + if masking.zero { + want += ".Z" + } + if got.Op != want || len(got.Args) != 3+map[bool]int{false: 0, true: 1}[masking.mask != 0] || + got.Args[1].Reg != "Z1" && width == 64 { + t.Fatalf("decode %s %x = %+v, want %s", op, code, got, want) + } + } + } + } + } +} + +func TestDecodeX86RawMaskBlendArgReduceAVX512Regression(t *testing.T) { + code := []byte{0x62, 0xf2, 0x75, 0x4d, 0x65, 0xdb, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "argreduce AVX512 mask blend", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VBLENDMPS" || + decoded[0].Raw != "VBLENDMPS Z3, Z1, K5, Z3 /* decoded from argreduce AVX512 mask blend */" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawMaskBlendRIPDataCompleteFamily(t *testing.T) { + for op, spec := range amd64MaskBlendSpecs { + for _, width := range []int{16, 32, 64} { + for _, broadcast := range []bool{false, true} { + if broadcast && !spec.broadcast { + continue + } + name := fmt.Sprintf("%s/%d/bcst%t", op, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawMaskBlendLiteral(op, width, 7, true, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + if len(decoded) != 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET || + !bytes.Equal(decoded[0].x86RIPLiteralData, code[len(code)-dataWidth:]) { + t.Fatalf("decoded mask-blend literal %x as %#v", code, decoded) + } + }) + } + } + } +} + +func TestDecodeX86RawMaskBlendRejectsUnsafeForms(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated literal": func(code []byte) []byte { return code[:len(code)-1] }, + "outside directive group": func(code []byte) []byte { code[6] = 127; return code }, + "overlapping instruction": func(code []byte) []byte { code[6] = 0; return code }, + "segment override": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address override": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(x86RawMaskBlendLiteral("VBLENDMPS", 64, 7, true, true)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe mask-blend literal %x", code) + } + }) + } + for _, code := range [][]byte{ + x86RawMaskBlendEncoding("VPBLENDMB", 16, 3, false, true, false), + x86RawMaskBlendEncoding("VBLENDMPS", 16, 0, true, false, false), + } { + if _, _, ok, err := decodedX86RawMaskBlendInstruction(code, 64); !ok || err == nil { + t.Fatalf("accepted reserved mask blend %x: ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawMaskBlendRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for op, spec := range amd64MaskBlendSpecs { + for _, broadcast := range []bool{false, true} { + if broadcast && !spec.broadcast { + continue + } + name := fmt.Sprintf("maskBlendLiteral%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range x86RawMaskBlendLiteral(op, 64, 7, true, broadcast) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-mask-blend-literal.ll", "raw-mask-blend-literal.o", ir) + }) + } +} diff --git a/amd64_raw_mask_broadcast_test.go b/amd64_raw_mask_broadcast_test.go new file mode 100644 index 000000000..ea2033f75 --- /dev/null +++ b/amd64_raw_mask_broadcast_test.go @@ -0,0 +1,206 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +// This physical directory is independent of the decoder and is also checked +// against actual cmd/asm output below. K is a data operand, not a writemask. +func x86MaskBroadcastPhysicalBytes(op string, width, source, destination int) []byte { + p0 := byte(0xf2 ^ ((destination >> 3 & 1) << 7) ^ ((destination >> 4 & 1) << 4)) + p1, opcode := byte(0x7e), byte(0x3a) + if op == "VPBROADCASTMB2Q" { + p1, opcode = 0xfe, 0x2a + } + return []byte{0x62, p0, p1, byte(width<<5 | 8), opcode, byte(0xc0 | destination&7<<3 | source)} +} + +func TestX86BroadcastRawMaskDataOriginalGoBytes(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + var expected []byte + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + for width, bank := range []string{"X", "Y", "Z"} { + for _, input := range []int{0, 7} { + destinations := []int{0, 7} + if arch == "amd64" { + destinations = append(destinations, 31) + } + for _, destination := range destinations { + fmt.Fprintf(&source, "%s K%d,%s%d\n", op, input, bank, destination) + expected = append(expected, x86MaskBroadcastPhysicalBytes(op, width, input, destination)...) + } + } + } + } + source.WriteString("RET\n") + expected = append(expected, 0xc3) + actual := assembleX87ControlBytes(t, arch, source.String()) + if !bytes.Equal(actual, expected) { + t.Fatalf("%s actual Go mask broadcast bytes %x, expected %x", arch, actual, expected) + } + for offset := 0; offset < len(actual)-1; offset += 6 { + mode := 64 + if arch == "386" { + mode = 32 + } + ins, length, known, err := decodedX86VEXPackedBroadcastInstruction(actual[offset:offset+6], mode) + if err != nil || !known || length != 6 { + t.Fatalf("%s actual Go bytes %x: known%t length%d err%v", arch, actual[offset:offset+6], known, length, err) + } + form, recognized, err := x86BroadcastProperties(ins.Op, ins, arch) + if err != nil || !recognized || form.kind != x86BroadcastMask { + t.Fatalf("%s physical mask grammar %+v: known%t err%v", arch, form, recognized, err) + } + // Decoding an initialized K input is not a new numerical proof. + for _, consumer := range x86ResultGPSIMDConsumers { + state := x86ResultState{} + for index := range state.masks { + state.masks[index] = true + } + if consumer.step(ins, &state, nil, 0, arch) { + t.Fatalf("%s admitted mask-data numerical authority", consumer.name) + } + } + } + } +} + +func TestX86BroadcastRawMaskDataCompletePhysicalBanks(t *testing.T) { + count := 0 + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + for width, bank := range []string{"X", "Y", "Z"} { + for input := 0; input < 8; input++ { + for destination := 0; destination < 32; destination++ { + code := x86MaskBroadcastPhysicalBytes(op, width, input, destination) + ins, length, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + want := fmt.Sprintf("%s K%d, %s%d", op, input, bank, destination) + if err != nil || !known || length != 6 || ins.Raw != want { + t.Fatalf("decode %x: %+v length%d known%t err%v; want %q", code, ins, length, known, err, want) + } + _, _, known32, err32 := decodedX86VEXPackedBroadcastInstruction(code, 32) + if !known32 || (err32 != nil) != (destination >= 8) { + t.Fatalf("mode32 destination%d: known%t err%v", destination, known32, err32) + } + count++ + } + } + } + } + if count != 1536 { + t.Fatalf("physical directory covered %d encodings", count) + } +} + +func TestX86BroadcastRawMaskDataReservedAndUnprovedForms(t *testing.T) { + for _, op := range []string{"VPBROADCASTMB2Q", "VPBROADCASTMW2D"} { + valid := x86MaskBroadcastPhysicalBytes(op, 0, 0, 0) + for name, mutate := range map[string]func([]byte){ + "wrong W": func(code []byte) { code[2] ^= 0x80 }, + "missing fixed bit": func(code []byte) { code[2] &^= 4 }, + "used vvvv": func(code []byte) { code[2] &^= 8 }, + "used high vvvv": func(code []byte) { code[3] &^= 8 }, + "source extension B": func(code []byte) { code[1] &^= 0x20 }, + "source extension X": func(code []byte) { code[1] &^= 0x40 }, + "writemask": func(code []byte) { code[3] |= 1 }, + "zeroing": func(code []byte) { code[3] |= 0x80 }, + "broadcast bit": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x60 }, + "memory input": func(code []byte) { code[5] &^= 0xc0 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64); !known || err == nil { + t.Fatalf("%s %s %x: known%t err%v", op, name, code, known, err) + } + } + for _, prefix := range []byte{0x64, 0x65, 0x67} { + code := append([]byte{prefix}, valid...) + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64); !known || err == nil { + t.Fatalf("%s forbidden prefix%x: known%t err%v", op, prefix, known, err) + } + } + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(valid[:5], 64); !known || err == nil { + t.Fatalf("%s missing ModRM: known%t err%v", op, known, err) + } + for length := 0; length < 5; length++ { + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(valid[:length], 64); known || err != nil { + t.Fatalf("%s incomplete prefix length%d: known%t err%v", op, length, known, err) + } + } + for name, mutate := range map[string]func([]byte){ + "other map": func(code []byte) { code[1] ^= 1 }, + "other prefix": func(code []byte) { code[2] ^= 1 }, + "other opcode": func(code []byte) { code[4] ^= 1 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, known, err := decodedX86VEXPackedBroadcastInstruction(code, 64); known || err != nil { + t.Fatalf("%s %s outside grammar: known%t err%v", op, name, known, err) + } + } + } +} + +func TestX86BroadcastRawMaskDataMandatoryPrefixCollision(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + mode := 64 + if arch == "386" { + mode = 32 + } + for _, row := range []string{"VPBROADCASTMB2Q K0,X0", "VPBROADCASTMW2D K7,Z3", "VPMINUW Z1,Z2,Z3"} { + code := assembleX87ControlBytes(t, arch, "TEXT f(SB),4,$0-0\n"+row+"\nRET\n") + if len(code) != 7 || code[0] != 0x62 || code[1]&15 != 2 { + t.Fatalf("%s original Go opcode-map witness %q: %x", arch, row, code) + } + want := Op(strings.Fields(row)[0]) + prefix := byte(2) + if want == "VPMINUW" { + prefix = 1 + } + if code[2]&3 != prefix || want != "VPBROADCASTMB2Q" && code[4] != 0x3a { + t.Fatalf("%s original mandatory-prefix/opcode witness %q: %x", arch, row, code) + } + var source strings.Builder + source.WriteString("TEXT f(SB),4,$0-0\n") + for _, value := range code[:6] { + fmt.Fprintf(&source, "BYTE $%#x\n", value) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := decodeX86RawDirectives(file.Funcs[0], arch) + if err != nil { + t.Fatalf("%s actual Go %q directive route: %v", arch, row, err) + } + ins := normalized.Instrs[1] + if ins.Op != want || ins.x86SourceRaw == nil || ins.x86SourceRaw.op != want { + t.Fatalf("%s %q lost original decoded source identity: %+v", arch, row, ins) + } + if want == "VPMINUW" { + if len(ins.Args) != 3 || ins.Args[0].String() != "Z1" || ins.Args[1].String() != "Z2" || ins.Args[2].String() != "Z3" { + t.Fatalf("legal min/max sibling changed: %+v", ins) + } + continue + } + for name, mutate := range map[string]func([]byte){ + "reserved mandatory prefix": func(code []byte) { code[2] ^= 1 }, + "reserved fixed bit": func(code []byte) { code[2] &^= 4 }, + "reserved W": func(code []byte) { code[2] ^= 0x80 }, + "unproved memory": func(code []byte) { code[5] &^= 0xc0 }, + } { + invalid := append([]byte(nil), code[:6]...) + mutate(invalid) + if _, err := decodeX86RawDirectiveGroup(invalid, mode, 0, name, map[string]bool{}); err == nil { + t.Fatalf("%s %q %s accepted through another decoder: %x", arch, row, name, invalid) + } + } + } + } +} diff --git a/amd64_raw_mask_compare.go b/amd64_raw_mask_compare.go new file mode 100644 index 000000000..01ccb98b7 --- /dev/null +++ b/amd64_raw_mask_compare.go @@ -0,0 +1,88 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawMaskCompareForm struct { + op Op + laneBytes int +} + +// The eight signed and unsigned Go 1.27 _yvpcmpb opcode rows are selected +// by the opcode and EVEX.W, not by a separate decoder for each mnemonic. +var x86RawMaskCompareForms = map[[2]int]x86RawMaskCompareForm{ + {0x3f, 0}: {op: "VPCMPB", laneBytes: 1}, + {0x3f, 1}: {op: "VPCMPW", laneBytes: 2}, + {0x1f, 0}: {op: "VPCMPD", laneBytes: 4}, + {0x1f, 1}: {op: "VPCMPQ", laneBytes: 8}, + {0x3e, 0}: {op: "VPCMPUB", laneBytes: 1}, + {0x3e, 1}: {op: "VPCMPUW", laneBytes: 2}, + {0x1e, 0}: {op: "VPCMPUD", laneBytes: 4}, + {0x1e, 1}: {op: "VPCMPUQ", laneBytes: 8}, +} + +func decodedX86RawMaskCompareInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 { + return Instr{}, 0, false, nil + } + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawMaskCompareForms[[2]int{p.opcode, w}] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX mask compare: %s", message) + } + if mode != 64 { + return fail("Go's 386 assembler frontend cannot express mask compare operands") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 || p.zero || p.r != 0 { + return fail("invalid EVEX fixed, address, vector length, zeroing, or K destination extension") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + if p.broadcast && (form.laneBytes < 4 || modRM>>6 == 3) { + return fail("broadcast requires a D/Q scalar-memory first source") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + disp8Scale := 16 << p.vectorLength + if p.broadcast { + disp8Scale = form.laneBytes + } + first, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := p.modRM + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + op := form.op + if p.broadcast { + op += ".BCST" + } + args := []Operand{ + {Kind: OpImm, Imm: int64(code[immIndex])}, + first, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, p.upper))}, + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", modRM>>3&7))}) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, immIndex + 1, true, nil +} diff --git a/amd64_raw_mask_compare_test.go b/amd64_raw_mask_compare_test.go new file mode 100644 index 000000000..cfa3ddf79 --- /dev/null +++ b/amd64_raw_mask_compare_test.go @@ -0,0 +1,184 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawMaskCompare(opcode, widthBit, vectorBits, writeMask int, broadcast, memory bool) []byte { + p1 := byte(0x5d) // EVEX.vvvv = vector register 4, pp = 66. + if widthBit != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | writeMask) + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // first register 3, K destination 2. + if memory { + modRM = 0x50 // disp8(AX), K destination 2. + } + code := []byte{0x62, 0xf3, p1, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return append(code, 6) +} + +func TestDecodeX86RawMaskCompareSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPCMPUB $6, 0(SI)(R8*1), Z0, K0. + code := []byte{0x62, 0xb3, 0x7d, 0x48, 0x3e, 0x04, 0x06, 0x06, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "maskBitsLess AVX512 VPCMPUB", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPCMPUB" || + decoded[0].Args[0].Imm != 6 || decoded[0].Args[2].Reg != "Z0" || + decoded[0].Args[3].Reg != "K0" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawMaskCompareCompleteGoForms(t *testing.T) { + if len(x86RawMaskCompareForms) != 8 { + t.Fatalf("modeled %d EVEX mask-compare rows, want eight", len(x86RawMaskCompareForms)) + } + for encoding, form := range x86RawMaskCompareForms { + for vectorBits := 0; vectorBits < 3; vectorBits++ { + vector := [...]string{"X", "Y", "Z"}[vectorBits] + for _, variant := range []struct { + name string + writeMask int + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", writeMask: 3}, + {name: "memory", memory: true}, + {name: "masked memory", writeMask: 7, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + } { + if variant.broadcast && form.laneBytes < 4 { + continue + } + name := fmt.Sprintf("%s/%s/%s", form.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawMaskCompare(encoding[0], encoding[1], vectorBits, variant.writeMask, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86RawMaskCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := form.op + if variant.broadcast { + wantOp += ".BCST" + } + if got.Op != wantOp || got.Args[0].Imm != 6 || got.Args[2].String() != vector+"4" || + got.Args[len(got.Args)-1].String() != "K2" || + !variant.memory && got.Args[1].String() != vector+"3" || + len(got.Args) != 4+map[bool]int{true: 1}[variant.writeMask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawMaskCompareRIPLiteral(t *testing.T) { + for encoding, form := range x86RawMaskCompareForms { + for _, broadcast := range []bool{false, true} { + if broadcast && form.laneBytes < 4 { + continue + } + code := encodeX86RawMaskCompare(encoding[0], encoding[1], 2, 1, broadcast, false) + code = code[:6] + code[5] = 0x15 // RIP-relative first source, K destination 2. + code = append(code, 1, 0, 0, 0, 6, 0xc3) + literalWidth := 64 + if broadcast { + literalWidth = form.laneBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/broadcast%t", form.op, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + wantOp := form.op + if broadcast { + wantOp += ".BCST" + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth || got[0].Op != wantOp { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawMaskCompareRejectsInvalidForms(t *testing.T) { + base := encodeX86RawMaskCompare(0x3e, 0, 2, 0, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // no EVEX zeroing + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + func(code []byte) { code[3] |= 0x10 }, // byte broadcast unavailable + func(code []byte) { code[1] &^= 0x80 }, // K destination extension + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawMaskCompareInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + if _, _, ok, err := decodedX86RawMaskCompareInstruction(base, 32); !ok || err == nil { + t.Fatalf("accepted 386 mask compare: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawMaskCompareLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawMaskCompare(SB),$0-0\n") + for encoding, form := range x86RawMaskCompareForms { + code := encodeX86RawMaskCompare(encoding[0], encoding[1], 0, 0, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + if form.laneBytes >= 4 { + code = encodeX86RawMaskCompare(encoding[0], encoding[1], 0, 3, true, true) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawMaskCompare": {Name: "rawMaskCompare", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-mask-compare.ll", "raw-mask-compare.o", ir) + }) + } +} diff --git a/amd64_raw_mask_move_test.go b/amd64_raw_mask_move_test.go new file mode 100644 index 000000000..95104f471 --- /dev/null +++ b/amd64_raw_mask_move_test.go @@ -0,0 +1,93 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawMaskMoveCompleteGo127Family(t *testing.T) { + // Go 1.27's shared _ykmovb table gives KMOVB/W/D/Q the same four + // encoding rows: K->memory, K->GP, K-or-memory->K, and GP->K. Cover + // every row at every width, plus the extended GP form reported by + // github.com/minio/sha256-simd. + tests := []struct { + name string + code []byte + want string + }{ + {name: "b k to k", code: []byte{0xc5, 0xf9, 0x90, 0xcd}, want: "KMOVB K5, K1"}, + {name: "b memory to k", code: []byte{0xc5, 0xf9, 0x90, 0x08}, want: "KMOVB 0(AX), K1"}, + {name: "b k to memory", code: []byte{0xc5, 0xf9, 0x91, 0x00}, want: "KMOVB K0, 0(AX)"}, + {name: "b k to gp", code: []byte{0xc5, 0xf9, 0x93, 0xc5}, want: "KMOVB K5, AX"}, + {name: "b gp to k", code: []byte{0xc5, 0xf9, 0x92, 0xe8}, want: "KMOVB AX, K5"}, + {name: "w k to k", code: []byte{0xc5, 0xf8, 0x90, 0xcd}, want: "KMOVW K5, K1"}, + {name: "w memory to k", code: []byte{0xc5, 0xf8, 0x90, 0x08}, want: "KMOVW 0(AX), K1"}, + {name: "w k to memory", code: []byte{0xc5, 0xf8, 0x91, 0x00}, want: "KMOVW K0, 0(AX)"}, + {name: "w k to gp", code: []byte{0xc5, 0xf8, 0x93, 0xc5}, want: "KMOVW K5, AX"}, + {name: "w gp to k", code: []byte{0xc5, 0xf8, 0x92, 0xe8}, want: "KMOVW AX, K5"}, + {name: "d k to k", code: []byte{0xc4, 0xe1, 0xf9, 0x90, 0xcd}, want: "KMOVD K5, K1"}, + {name: "d memory to k", code: []byte{0xc4, 0xe1, 0xf9, 0x90, 0x08}, want: "KMOVD 0(AX), K1"}, + {name: "d k to memory", code: []byte{0xc4, 0xe1, 0xf9, 0x91, 0x00}, want: "KMOVD K0, 0(AX)"}, + {name: "d k to gp", code: []byte{0xc5, 0xfb, 0x93, 0xc5}, want: "KMOVD K5, AX"}, + {name: "d gp to k", code: []byte{0xc5, 0xfb, 0x92, 0xe8}, want: "KMOVD AX, K5"}, + {name: "q k to k", code: []byte{0xc4, 0xe1, 0xf8, 0x90, 0xcd}, want: "KMOVQ K5, K1"}, + {name: "q memory to k", code: []byte{0xc4, 0xe1, 0xf8, 0x90, 0x08}, want: "KMOVQ 0(AX), K1"}, + {name: "q k to memory", code: []byte{0xc4, 0xe1, 0xf8, 0x91, 0x00}, want: "KMOVQ K0, 0(AX)"}, + {name: "q k to gp", code: []byte{0xc4, 0xe1, 0xfb, 0x93, 0xc5}, want: "KMOVQ K5, AX"}, + {name: "q gp to k", code: []byte{0xc4, 0xe1, 0xfb, 0x92, 0xe8}, want: "KMOVQ AX, K5"}, + {name: "q reported extended gp to k", code: []byte{0xc4, 0xc1, 0xfb, 0x92, 0xce}, want: "KMOVQ R14, K1"}, + {name: "q extended k to gp", code: []byte{0xc4, 0x61, 0xfb, 0x93, 0xd7}, want: "KMOVQ K7, R10"}, + {name: "q extended sib memory to k", code: []byte{0xc4, 0x81, 0xf8, 0x90, 0x4c, 0x88, 0x20}, want: "KMOVQ 32(R8)(R9*4), K1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || !strings.HasPrefix(got.Instrs[0].Raw, test.want+" ") { + t.Fatalf("decoded %#x as %#v, want %q", test.code, got.Instrs, test.want) + } + }) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch+"/lower", func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_mask_move(SB),NOSPLIT,$0-0\n") + for _, test := range tests[:20] { + for _, value := range test.code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_mask_move": {Name: "raw_mask_move", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-mask-move-"+target.goarch+".ll", "raw-mask-move-"+target.goarch+".o", ir) + }) + } +} diff --git a/amd64_raw_mask_vector_move.go b/amd64_raw_mask_vector_move.go new file mode 100644 index 000000000..b9d7fdbc7 --- /dev/null +++ b/amd64_raw_mask_vector_move.go @@ -0,0 +1,113 @@ +package plan9asm + +import "fmt" + +type decodedX86MaskVectorMoveProperties struct { + op Op + maskToVector bool +} + +func decodedX86MaskVectorMoveOpcode(opcode byte, width64 bool) (decodedX86MaskVectorMoveProperties, bool) { + width := byte(0) + if width64 { + width = 1 + } + properties := map[[2]byte]decodedX86MaskVectorMoveProperties{ + {0x28, 0}: {op: "VPMOVM2B", maskToVector: true}, + {0x28, 1}: {op: "VPMOVM2W", maskToVector: true}, + {0x38, 0}: {op: "VPMOVM2D", maskToVector: true}, + {0x38, 1}: {op: "VPMOVM2Q", maskToVector: true}, + {0x29, 0}: {op: "VPMOVB2M"}, + {0x29, 1}: {op: "VPMOVW2M"}, + {0x39, 0}: {op: "VPMOVD2M"}, + {0x39, 1}: {op: "VPMOVQ2M"}, + }[[2]byte{opcode, width}] + return properties, properties.op != "" +} + +// decodedX86MaskVectorMoveInstruction recognizes Go 1.27's complete +// VPMOVM2{B,W,D,Q} and VPMOV{B,W,D,Q}2M family. These EVEX-only instructions +// have register-only K/X/Y/Z forms and do not permit writemasking, zeroing, or +// EVEX.b. Recover all vector registers and widths before the generic decoder. +func decodedX86MaskVectorMoveInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64, 0x65: + i++ + case 0x67: + addressOverride = true + i++ + default: + goto evex + } + } + +evex: + if mode != 32 && mode != 64 || len(code) < i+6 || code[i] != 0x62 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + if p0&0x0f != 2 || p1&3 != 2 { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86MaskVectorMoveOpcode(opcode, p1&0x80 != 0) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if p2&0x97 != 0 { + return Instr{}, 0, true, fmt.Errorf("mask/vector moves do not support masking, zeroing, or EVEX.b") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + if modRM>>6 != 3 { + return Instr{}, 0, true, fmt.Errorf("mask/vector moves require register operands") + } + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + vectorNumber := 0 + maskNumber := 0 + if properties.maskToVector { + if xExt != 0 || bExt != 0 { + return Instr{}, 0, true, fmt.Errorf("K source register cannot be extended") + } + vectorNumber = int(modRM>>3&7) + rExt*8 + rHighExt*16 + maskNumber = int(modRM & 7) + } else { + if rExt != 0 || rHighExt != 0 { + return Instr{}, 0, true, fmt.Errorf("K destination register cannot be extended") + } + vectorNumber = int(modRM&7) + bExt*8 + xExt*16 + maskNumber = int(modRM >> 3 & 7) + } + if mode == 32 && vectorNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + vectorPrefix := [...]string{"X", "Y", "Z"}[vectorBits] + vector := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, vectorNumber))} + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))} + args := []Operand{vector, mask} + if properties.maskToVector { + args[0], args[1] = args[1], args[0] + } + return Instr{Op: properties.op, Args: args, Raw: fmt.Sprintf("%s %s, %s", properties.op, args[0].String(), args[1].String())}, modRMIndex + 1, true, nil +} diff --git a/amd64_raw_mask_vector_move_test.go b/amd64_raw_mask_vector_move_test.go new file mode 100644 index 000000000..01c2b64f9 --- /dev/null +++ b/amd64_raw_mask_vector_move_test.go @@ -0,0 +1,164 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type rawMaskVectorMoveSpec struct { + opcode byte + width64 bool + op Op + laneBits int + maskToVector bool +} + +var rawMaskVectorMoveSpecs = []rawMaskVectorMoveSpec{ + {opcode: 0x28, op: "VPMOVM2B", laneBits: 8, maskToVector: true}, + {opcode: 0x28, width64: true, op: "VPMOVM2W", laneBits: 16, maskToVector: true}, + {opcode: 0x38, op: "VPMOVM2D", laneBits: 32, maskToVector: true}, + {opcode: 0x38, width64: true, op: "VPMOVM2Q", laneBits: 64, maskToVector: true}, + {opcode: 0x29, op: "VPMOVB2M", laneBits: 8}, + {opcode: 0x29, width64: true, op: "VPMOVW2M", laneBits: 16}, + {opcode: 0x39, op: "VPMOVD2M", laneBits: 32}, + {opcode: 0x39, width64: true, op: "VPMOVQ2M", laneBits: 64}, +} + +func encodeX86EVEXMaskVectorMove(spec rawMaskVectorMoveSpec, vectorBits byte, vector, mask int) []byte { + p0 := byte(2) + regField, rmField := vector, mask + if spec.maskToVector { + p0 |= byte((1-(vector>>3)&1)<<7 | 1<<6 | 1<<5 | (1-(vector>>4)&1)<<4) + } else { + regField, rmField = mask, vector + p0 |= byte(1<<7 | (1-(vector>>4)&1)<<6 | (1-(vector>>3)&1)<<5 | 1<<4) + } + p1 := byte(15<<3 | 1<<2 | 2) + if spec.width64 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 1<<3) + modRM := byte(0xc0 | (regField&7)<<3 | rmField&7) + return []byte{0x62, p0, p1, p2, spec.opcode, modRM} +} + +func TestDecodedX86EVEXMaskVectorMoveCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawMaskVectorMoveSpecs { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + for vector := 0; vector < 32; vector++ { + for mask := 0; mask < 8; mask++ { + code := encodeX86EVEXMaskVectorMove(spec, vectorBits, vector, mask) + got, length, ok, err := decodedX86MaskVectorMoveInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, vector), fmt.Sprintf("K%d", mask)} + if spec.maskToVector { + wantArgs[0], wantArgs[1] = wantArgs[1], wantArgs[0] + } + if got.Op != spec.op || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 6144 { + t.Fatalf("covered %d EVEX mask/vector move encodings, want 6144", count) + } +} + +func TestDecodedX86EVEXMaskVectorMoveGoAndWeaviateEncodings(t *testing.T) { + tests := []struct { + code []byte + op Op + args []string + }{ + {code: []byte{0x62, 0x72, 0x7e, 0x48, 0x38, 0xc0}, op: "VPMOVM2D", args: []string{"K0", "Z8"}}, + {code: []byte{0x62, 0x62, 0x7e, 0x08, 0x28, 0xd4}, op: "VPMOVM2B", args: []string{"K4", "X26"}}, + {code: []byte{0x62, 0xd2, 0x7e, 0x48, 0x29, 0xc9}, op: "VPMOVB2M", args: []string{"Z9", "K1"}}, + {code: []byte{0x62, 0x92, 0xfe, 0x28, 0x29, 0xe3}, op: "VPMOVW2M", args: []string{"Y27", "K4"}}, + } + for _, test := range tests { + got, length, ok, err := decodedX86MaskVectorMoveInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != 2 || got.Args[0].String() != test.args[0] || got.Args[1].String() != test.args[1] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } +} + +func TestDecodedX86EVEXMaskVectorMoveInvalidForms(t *testing.T) { + spec := rawMaskVectorMoveSpecs[0] + base := encodeX86EVEXMaskVectorMove(spec, 0, 0, 0) + invalid := make([][]byte, 0, 9) + reservedVVVV := append([]byte(nil), base...) + reservedVVVV[2] &^= 0x08 + reservedHighV := append([]byte(nil), base...) + reservedHighV[3] &^= 0x08 + reservedLength := append([]byte(nil), base...) + reservedLength[3] |= 0x60 + masking := append([]byte(nil), base...) + masking[3] |= 0x01 + zeroing := append([]byte(nil), base...) + zeroing[3] |= 0x80 + broadcast := append([]byte(nil), base...) + broadcast[3] |= 0x10 + memory := append([]byte(nil), base...) + memory[5] &^= 0xc0 + extendedKSource := append([]byte(nil), base...) + extendedKSource[1] &^= 0x20 + vectorToMask := encodeX86EVEXMaskVectorMove(rawMaskVectorMoveSpecs[4], 0, 0, 0) + extendedKDestination := append([]byte(nil), vectorToMask...) + extendedKDestination[1] &^= 0x80 + invalid = append(invalid, reservedVVVV, reservedHighV, reservedLength, masking, zeroing, broadcast, memory, extendedKSource, extendedKDestination) + for _, code := range invalid { + if _, _, ok, err := decodedX86MaskVectorMoveInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateRawEVEXMaskToVectorWeaviateBlockRegression(t *testing.T) { + const source = `TEXT rawMaskToVectorBlock(SB), $0-0 + LONG $0x487e7262; WORD $0xc038 + LONG $0x4875d162; WORD $0xc8fa + LONG $0x487e7262; WORD $0xc738 + LONG $0x4875d162; WORD $0xc8fa + LONG $0x487e7262; WORD $0xc138 + LONG $0x486dd162; WORD $0xd0fa + LONG $0x487e7262; WORD $0xc238 + LONG $0x4865d162; WORD $0xd8fa + LONG $0x487e7262; WORD $0xc338 + LONG $0x485dd162; WORD $0xe0fa + LONG $0x487e7262; WORD $0xc438 + LONG $0x4855d162; WORD $0xe8fa + LONG $0x487e7262; WORD $0xc538 + LONG $0x484dd162; WORD $0xf0fa + LONG $0x487e7262; WORD $0xc638 + LONG $0x4845d162; WORD $0xf8fa + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawMaskToVectorBlock": {Name: "rawMaskToVectorBlock", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if strings.Count(ir, "select i1") < 8*16 { + t.Fatalf("raw VPMOVM2D block omitted mask expansion:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-mask-to-vector.ll", "amd64-raw-mask-to-vector.o", ir) +} diff --git a/amd64_raw_memory_decoder.go b/amd64_raw_memory_decoder.go new file mode 100644 index 000000000..75de6c9bd --- /dev/null +++ b/amd64_raw_memory_decoder.go @@ -0,0 +1,23 @@ +package plan9asm + +// The zero value retains the raw source decoder's layout restrictions. Only +// original native byte observers select physicalPC: they observe whole native +// instruction boundaries and discards these operands. A physical RIP+disp32 +// is not a source symbol, bounded literal, numerical value, or read permission. +type x86RawMemoryDecoder struct { + physicalPC bool +} + +func (d x86RawMemoryDecoder) evex(code []byte, mode, bExt, xExt int, segment Reg, bank string, scale int) (Operand, int, error) { + if len(code) == 0 || code[0]>>6 == 3 { + return decodedX86EVEXRMOperand(code, mode, bExt, xExt, segment, bank, scale) + } + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, scale, d.physicalPC) +} + +func (d x86RawMemoryDecoder) vex(code []byte, mode, bExt, xExt int, segment Reg, bank string) (Operand, int, error) { + if len(code) == 0 || code[0]>>6 == 3 { + return decodedX86VEXVectorRMOperand(code, mode, bExt, xExt, segment, bank) + } + return decodedX86RMSourceWithPC(code, mode, bExt, xExt, segment, 1, d.physicalPC) +} diff --git a/amd64_raw_minmax.go b/amd64_raw_minmax.go new file mode 100644 index 000000000..e2fb161b0 --- /dev/null +++ b/amd64_raw_minmax.go @@ -0,0 +1,26 @@ +package plan9asm + +// VEX B/W/D ignores W. EVEX B/W also ignores W; D/Q shares an opcode with +// fixed W0/W1, and Q has no VEX encoding. The remaining operand grammar is +// shared with per-lane shifts, not inferred from the legacy disassembler. +func decodedX86PackedIntegerMinMaxInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + + for name, spec := range amd64PackedIntegerMinMaxSpecs { + if p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + continue + } + if !p.evex && spec.laneBits == 64 { + continue + } + if p.evex && spec.laneBits >= 32 && p.w != (spec.laneBits == 64) { + continue + } + + return decodedX86BinaryVectorOperands(code, p, mode, Op(name), spec.laneBits) + } + return Instr{}, 0, false, nil +} diff --git a/amd64_raw_minmax_literal_test.go b/amd64_raw_minmax_literal_test.go new file mode 100644 index 000000000..8d00d1dad --- /dev/null +++ b/amd64_raw_minmax_literal_test.go @@ -0,0 +1,174 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawMinMaxLiteral(op string, evex bool, width, mask int, zeroing, broadcast bool) []byte { + spec := amd64PackedIntegerMinMaxSpecs[op] + code := encodeRawScalarMove(evex, 1, spec.opcode, 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | byte(spec.mapNumber) + code[2] &^= 0x80 + if spec.laneBits == 64 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*13+9)) + } + return code +} + +func TestDecodeX86RawMinMaxRIPDataCompleteFamily(t *testing.T) { + for _, op := range x86MinMaxTestOps { + spec := amd64PackedIntegerMinMaxSpecs[op] + for _, evex := range []bool{false, true} { + if !evex && spec.laneBits == 64 { + continue + } + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && spec.broadcast { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawMinMaxLiteral(op, evex, width, mask, zeroing, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(op) + if broadcast { + want += ".BCST" + } + if zeroing { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawMinMaxRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawMinMaxLiteral("VPMINSD", true, 64, 7, true, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawMinMaxRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, op := range x86MinMaxTestOps { + spec := amd64PackedIntegerMinMaxSpecs[op] + evex := spec.laneBits == 64 + width := 32 + if evex { + width = 64 + } + broadcast := evex && spec.broadcast + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + code := x86RawMinMaxLiteral(op, evex, width, mask, zeroing, broadcast) + name := fmt.Sprintf("minMaxLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "minmax-literal.ll", "minmax-literal.o", ir) + }) + } +} diff --git a/amd64_raw_minmax_test.go b/amd64_raw_minmax_test.go new file mode 100644 index 000000000..4794aed42 --- /dev/null +++ b/amd64_raw_minmax_test.go @@ -0,0 +1,460 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86MinMaxTestOps = []string{ + "VPMINSB", "VPMINUB", "VPMAXSB", "VPMAXUB", + "VPMINSW", "VPMINUW", "VPMAXSW", "VPMAXUW", + "VPMINSD", "VPMINUD", "VPMAXSD", "VPMAXUD", + "VPMINSQ", "VPMINUQ", "VPMAXSQ", "VPMAXUQ", +} + +func TestX86MinMaxGrammarMatchesGoEncoder(t *testing.T) { + path := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go") + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + pattern := `(?s)\{as: A(VPM(?:IN|AX)[SU][BWDQ]), ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(string(data), -1) + if len(rows) != 16 || len(amd64PackedIntegerMinMaxSpecs) != len(rows) { + t.Fatalf("Go family=%d, typed family=%d", len(rows), len(amd64PackedIntegerMinMaxSpecs)) + } + + for _, row := range rows { + spec, ok := amd64PackedIntegerMinMaxSpecs[row[1]] + if !ok { + t.Fatalf("missing %s", row[1]) + } + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[row[1][len(row[1])-1]] + if spec.laneBits != bits || spec.signed != (row[1][5] == 'S') || spec.minimum != strings.HasPrefix(row[1], "VPMIN") { + t.Fatalf("incorrect typed semantics for %s: %+v", row[1], spec) + } + if (row[2] == "_yvandnpd") != (bits != 64) || strings.Contains(row[3], "| vex128") != (bits != 64) { + t.Fatalf("incorrect encoding generation for %s", row[1]) + } + if strings.Contains(row[3], "0F38") != (spec.mapNumber == 2) || strings.Contains(row[3], "evexBcst") != spec.broadcast { + t.Fatalf("incorrect map/broadcast for %s", row[1]) + } + opcodes := regexp.MustCompile(`0x[0-9A-F]+`).FindAllString(row[3], -1) + wantRows := 5 + if bits == 64 { + wantRows = 3 + } + if len(opcodes) != wantRows { + t.Fatalf("%s has %d encodings, want %d", row[1], len(opcodes), wantRows) + } + for _, opcode := range opcodes { + if opcode != fmt.Sprintf("0x%X", spec.opcode) { + t.Fatalf("%s opcode=%s, typed=%x", row[1], opcode, spec.opcode) + } + } + } + + operandTables := map[string][]string{ + "_yvandnpd": { + "Yxm, Yxr, Yxr", "Yym, Yyr, Yyr", + "YxmEvex, YxrEvex, YxrEvex", "YxmEvex, YxrEvex, Yknot0, YxrEvex", + "YymEvex, YyrEvex, YyrEvex", "YymEvex, YyrEvex, Yknot0, YyrEvex", + "Yzm, Yzr, Yzr", "Yzm, Yzr, Yknot0, Yzr", + }, + "_yvblendmpd": { + "YxmEvex, YxrEvex, YxrEvex", "YxmEvex, YxrEvex, Yknot0, YxrEvex", + "YymEvex, YyrEvex, YyrEvex", "YymEvex, YyrEvex, Yknot0, YyrEvex", + "Yzm, Yzr, Yzr", "Yzm, Yzr, Yknot0, Yzr", + }, + } + for name, want := range operandTables { + pattern := `(?s)var ` + name + ` = \[\]ytab\{(.*?)\n\}` + table := regexp.MustCompile(pattern).FindStringSubmatch(string(data)) + if len(table) != 2 { + t.Fatal("missing Go operand table", name) + } + args := regexp.MustCompile(`argList\{([^}]+)\}`).FindAllStringSubmatch(table[1], -1) + var got []string + for _, arg := range args { + got = append(got, arg[1]) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Go %s rows=%v, want %v", name, got, want) + } + } +} + +func TestX86MinMaxGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86MinMaxTestOps { + lines := []string{ + op + " X0,Y1,Y2", + op + " X0,X1,(AX)", + op + ".Z X0,X1,X2", + op + ".SAE X0,X1,X2", + op + " X0,X1,K0,X2", + op + ".BCST X0,X1,X2", + op + ".Z.BCST (AX),X1,K1,X2", + op + " $1,X1,X2", + } + if strings.HasSuffix(op, "B") || strings.HasSuffix(op, "W") { + lines = append(lines, op+".BCST (AX),X1,X2") + } + if arch == "386" { + lines = append(lines, op+" Z8,Z0,Z1", op+" X0,X1,K1,X2") + } + + for _, line := range lines { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + source := "TEXT bad(SB),4,$0-0\n" + line + "\nRET\n" + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + out, err := cmd.CombinedOutput() + historicalZero := strings.HasPrefix(runtime.Version(), "go1.20") && strings.Contains(line, ".Z X0") + if historicalZero && err != nil || !historicalZero && err == nil { + t.Fatalf("Go %s unexpected result for %s: %v %s", arch, line, err, out) + } + + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86PackedIntegerMinMaxRejected(t, arch, triple, line) + } + } + } +} + +func TestX86MinMaxZeroMaskDoesNotAccessMemory(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } else if runtime.GOARCH != "amd64" { + t.Skip("execution requires amd64 or Rosetta; five-target objects remain required") + } + + file, err := Parse(ArchAMD64, `TEXT zero_mask(SB),4,$0-24 +MOVQ source+0(FP), AX +MOVQ out+8(FP), DI +KMOVQ mask+16(FP), K1 +VPMINUD (AX), Y0, K1, Y1 +VMOVUPS Y1, (DI) +RET +`) + if err != nil { + t.Fatal(err) + } + sig := FuncSig{ + Name: "zero_mask", + Args: []LLVMType{Ptr, Ptr, I64}, + Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: I64, Index: 2, Field: -1}, + }}, + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: triple, + Sigs: map[string]FuncSig{"zero_mask": sig}, + }) + if err != nil { + t.Fatal(err) + } + + mainC := ` +#include +#include + +extern void zero_mask(const void *source, void *out, uint64_t mask); + +int main(void) { + uint8_t out[32]; + zero_mask(NULL, out, 0); + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "minmax_zero_mask", triple, ir, mainC, prefix) +} + +func TestX86RawMinMaxEncodingFields(t *testing.T) { + // The shared operand decoder already has the exhaustive shift-register + // product test. Here each min/max member checks every register field and + // W interpretation independently, without repeating that product 16 times. + opcodes := map[string]byte{ + "VPMINSB": 0x38, "VPMINUB": 0xda, "VPMAXSB": 0x3c, "VPMAXUB": 0xde, + "VPMINSW": 0xea, "VPMINUW": 0x3a, "VPMAXSW": 0xee, "VPMAXUW": 0x3e, + "VPMINSD": 0x39, "VPMINUD": 0x3b, "VPMAXSD": 0x3d, "VPMAXUD": 0x3f, + "VPMINSQ": 0x39, "VPMINUQ": 0x3b, "VPMAXSQ": 0x3d, "VPMAXUQ": 0x3f, + } + count := 0 + for _, op := range x86MinMaxTestOps { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + mapNumber := byte(2) + if op == "VPMINUB" || op == "VPMAXUB" || op == "VPMINSW" || op == "VPMAXSW" { + mapNumber = 1 + } + for _, evex := range []bool{false, true} { + if !evex && bits == 64 { + continue + } + limit, widths := 16, 2 + if evex { + limit, widths = 32, 3 + } + for width := 0; width < widths; width++ { + for axis := 0; axis < 3; axis++ { + for register := 0; register < limit; register++ { + regs := [3]int{0, 1, 2} // source2, source1, destination. + regs[axis] = register + variants := 1 + if !evex || bits < 32 { + variants = 2 + } + for variant := 0; variant < variants; variant++ { + mask, zero := 0, false + if evex { + mask = register % 8 + zero = mask != 0 && register&1 != 0 + } + code := encodeRawScalarMove(evex, 1, 0, regs[0], regs[1], regs[2], mask, zero) + code[len(code)-2] = opcodes[op] + code[1] = code[1]&0xf0 | mapNumber + code[2] &^= 0x80 + if variant != 0 || bits == 64 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(width) << 5 + } else { + code[2] |= byte(width) << 2 + } + + prefix := []string{"X", "Y", "Z"}[width] + args := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regs[0]))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regs[1]))}, + } + if mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, regs[2]))}) + wantOp := Op(op) + if zero { + wantOp += ".Z" + } + + got, size, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64) + if err != nil || !matched || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x: %+v %d %v %v; want %s %+v", code, got, size, matched, err, wantOp, args) + } + _, _, matched, err = decodedX86PackedIntegerMinMaxInstruction(code, 32) + if !matched || (err != nil) != (register >= 8) { + t.Fatalf("386 decode %x: matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 9216 { + t.Fatalf("covered %d field cases, want 9216", count) + } +} + +func TestX86RawMinMax386MaskedObjects(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT masked_minmax(SB),4,$0-0\n") + for _, op := range x86MinMaxTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, input := range []string{width + "0", "(AX)"} { + for _, suffix := range []string{"", ".Z"} { + line := fmt.Sprintf("%s%s %s,%s1,K7,%s2", op, suffix, input, width, width) + emitX86GoEncodedTestInstruction(t, &source, line, true) + } + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + line := fmt.Sprintf("%s.BCST.Z (AX),%s1,K7,%s2", op, width, width) + emitX86GoEncodedTestInstruction(t, &source, line, true) + } + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir, err := Translate(file, Options{ + Goarch: "386", + TargetTriple: triple, + Sigs: map[string]FuncSig{"masked_minmax": {Name: "masked_minmax", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "masked-minmax-386.ll", "masked-minmax-386.o", ir) + } +} + +func TestX86RawMinMaxGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 31 + } + + var source strings.Builder + source.WriteString("TEXT minmax(SB),4,$0-0\n") + for _, op := range x86MinMaxTestOps { + broadcast := strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") + for _, width := range []string{"X", "Y", "Z"} { + for _, reg := range []int{0, last} { + inputs := []string{ + fmt.Sprintf("%s%d", width, last-reg), + "(AX)", "-7(BX)(CX*4)", "64(SI)", "8192(DI)", + } + for _, input := range inputs { + fmt.Fprintf(&source, "%s %s,%s%d,%s%d\n", op, input, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s,%s%d,K1,%s%d\n", op, input, width, reg, width, last-reg) + fmt.Fprintf(&source, "%s.Z %s,%s%d,K7,%s%d\n", op, input, width, reg, width, last-reg) + } + } + if broadcast { + fmt.Fprintf(&source, "%s.BCST -8(BX)(CX*4),%s%d,%s%d\n", op, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST.Z 64(SI),%s%d,K7,%s%d\n", op, width, reg, width, last-reg) + } + } + } + if width != "Z" { + fmt.Fprintf(&source, "%s %s0,%s1,%s2\n", op, width, width, width) + fmt.Fprintf(&source, "%s (AX),%s1,%s2\n", op, width, width) + } + } + } + source.WriteString("RET\n") + + code := assembleX87ControlBytes(t, arch, source.String()) + got, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(got.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(got.Instrs), len(want)) + } + for i, instruction := range got.Instrs { + if instruction.Op != want[i].Op || !reflect.DeepEqual(instruction.Args, want[i].Args) { + t.Fatalf("instruction %d: %+v, want %+v", i, instruction, want[i]) + } + } + + file := x86RawUnprovedReturnFormProbeFile(t, arch, "minmax", code) + targets := []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + "i386-unknown-linux-gnu", "i686-pc-windows-msvc", + } + for _, triple := range targets { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir, err := Translate(file, Options{ + Goarch: arch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"minmax": {Name: "minmax", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "minmax.ll", "minmax.o", ir) + } + t.Logf("%s: %d actual Go encodings", arch, len(want)-1) + }) + } +} + +func TestX86RawMinMaxInvalidAndMemoryForms(t *testing.T) { + invalid := [][]byte{ + {0x62, 0xf2, 0x71, 0x0b, 0x39, 0xe0}, // EVEX fixed bit clear. + {0x62, 0xf2, 0x75, 0x6b, 0x39, 0xe0}, // Reserved vector length. + {0x62, 0xf2, 0x75, 0x88, 0x39, 0xe0}, // Zeroing without mask. + {0x62, 0xf2, 0x75, 0x1b, 0x39, 0xe0}, // Register broadcast. + {0x62, 0xf2, 0x75, 0x1b, 0x38, 0x00}, // Byte broadcast. + {0x62, 0xf2, 0x74, 0x0b, 0x39, 0xe0}, // Wrong mandatory prefix. + {0x62, 0xf2, 0x75, 0x0b, 0x39}, // Missing ModRM. + {0x62, 0xf2, 0x75, 0x0b, 0x39, 0x04}, // Missing SIB. + {0x62, 0xf2, 0x75, 0x0b, 0x39, 0x40}, // Missing disp8. + {0x67, 0xc5, 0xf1, 0xda, 0xc0}, // Address-size override. + {0xc5, 0xf1, 0xda, 0x05, 0, 0, 0, 0}, // Unrelocated RIP source. + } + for _, code := range invalid { + _, _, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64) + if !matched || err == nil { + t.Fatalf("accepted invalid %x", code) + } + } + if _, _, matched, err := decodedX86PackedIntegerMinMaxInstruction([]byte{0xc5, 0xf1, 0xda, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted 16-bit mode") + } + for _, code := range [][]byte{nil, {0x62}, {0x90}, {0xc4, 0xe3, 0x79, 0x39, 0xc0}, {0xc4, 0xe2, 0x79, 0x40, 0xc0}} { + if _, _, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64); matched || err != nil { + t.Fatalf("claimed unrelated %x", code) + } + } + + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, broadcast := range []bool{false, true} { + code := []byte{segment.prefix, 0x62, 0x92, 0xf5, 0xcb, 0x39, 0x64, 0x88, 0xfe} + offset, op := int64(-128), Op("VPMINSQ.Z") + if broadcast { + code[4] |= 0x10 + offset, op = -16, "VPMINSQ.BCST.Z" + } + got, size, matched, err := decodedX86PackedIntegerMinMaxInstruction(code, 64) + want := MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: offset} + if err != nil || !matched || size != len(code) || got.Op != op || !reflect.DeepEqual(got.Args[0].Mem, want) { + t.Fatalf("decode %x: %+v, err=%v", code, got, err) + } + } + } +} diff --git a/amd64_raw_minpos.go b/amd64_raw_minpos.go new file mode 100644 index 000000000..32455ed50 --- /dev/null +++ b/amd64_raw_minpos.go @@ -0,0 +1,41 @@ +package plan9asm + +import "fmt" + +// Intel XED's AVX table specifies VL128, V66, V0F38 and NOVSR; W is ignored. +// There are no Y/Z, EVEX, mask, immediate or three-operand forms. +func decodedX86MinimumPositionInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + spec := amd64MinimumPositionSpecs["VPHMINPOSUW"] + if !matched || p.mapNumber != spec.mapNumber || p.opcode != spec.opcode { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("minimum position: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.vectorLength != 0 || p.pp != 1 || p.upper != 0 { + return fail("requires VEX.128.66 with reserved vvvv") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.x != 0 || p.b != 0) { + return fail("extended register in 32-bit mode") + } + source, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + if err != nil { + return Instr{}, 0, true, err + } + dst := Reg(fmt.Sprintf("X%d", int(code[p.modRM]>>3&7)+p.r*8)) + return Instr{ + Op: "VPHMINPOSUW", + Args: []Operand{source, {Kind: OpReg, Reg: dst}}, + Raw: fmt.Sprintf("VPHMINPOSUW %s, %s", source.String(), dst), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_minpos_address_test.go b/amd64_raw_minpos_address_test.go new file mode 100644 index 000000000..44a088903 --- /dev/null +++ b/amd64_raw_minpos_address_test.go @@ -0,0 +1,204 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +func x86MinimumPositionOriginalGoBytes(t *testing.T, arch, source string) []byte { + t.Helper() + listing, err := x86ScalarLiteralGoListing(t, arch, source) + if err != nil { + t.Fatal(err) + } + var code []byte + for _, line := range strings.Split(listing, "\n") { + text := strings.TrimLeft(line, "\t ") + fields := strings.Fields(text) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, err := strconv.ParseUint(fields[0][2:], 16, 32) + start := strings.IndexByte(text, ' ') + if err != nil || int(offset) != len(code) || start < 0 || len(text) < start+48 { + t.Fatalf("incomplete/noncontiguous stock Go hex row: %s", line) + } + gap := false + for column := 0; column < 16; column++ { + field := text[start+3*column : start+3*(column+1)] + if field == " " { + gap = true + continue + } + value, err := strconv.ParseUint(field[1:], 16, 8) + if gap || field[0] != ' ' || err != nil { + t.Fatalf("invalid stock Go hex column: %q", field) + } + code = append(code, byte(value)) + } + } + if len(code) == 0 { + t.Fatal("stock Go omitted the original instruction bytes") + } + return code +} + +func TestX86RawMinimumPositionInactiveSIBScalePhysicalWitness(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + mode := 64 + if arch == "386" { + mode = 32 + } + for shift := byte(0); shift < 4; shift++ { + // Original legacy PHMINPOSUW 8192(SP),X0. SIB.index=4 + // suppresses the index; all four scale bit patterns are ignored. + code := []byte{0x66, 0x0f, 0x38, 0x41, 0x84, 0x24 | shift<<6, 0, 0x20, 0, 0} + original := append([]byte(nil), code...) + x86HandoffOriginalBytes(t, arch, code) + physical, err := x86asm.Decode(code, mode) + memory, ok := physical.Args[1].(x86asm.Mem) + base := x86asm.RSP + if mode == 32 { + base = x86asm.ESP + } + if err != nil || physical.Len != len(code) || physical.Op != x86asm.PHMINPOSUW || + !ok || memory.Base != base || memory.Index != 0 || memory.Scale != 1<4 GiB allocation. The original object's llc O0 gate rejects its frame offsets; +// default O2 instead only warns. Neither grants native stack/return permission. +func TestX86RawMinimumPositionHugeSPAllocatorBoundaryObservation(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Keep the complete mode32 address inventory in both source forms. The + // original encoded RET remains a negative stack/continuation control; + // the raw compile-only probe replaces only that RET with encoded UD2. + source, _, count, _ := x86MinimumPositionAddressSources("386", "giant", "UD2") + if count != 240 { + t.Fatalf("complete mode32 address denominator changed: %d", count) + } + namedCode := x86MinimumPositionOriginalGoBytes(t, "386", source) + namedFile, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + rawSource, _, _, _ := x86MinimumPositionAddressSources("386", "giant", "RET") + rawCode := x86MinimumPositionOriginalGoBytes(t, "386", rawSource) + rawFile := x86RawStackObservingFormProbeFile(t, "386", "giant", rawCode) + for _, fixture := range []struct { + name string + file *File + source string + code []byte + }{ + {"named", namedFile, source, namedCode}, + {"original-byte-probe", rawFile, rawSource, rawCode}, + } { + t.Run(fixture.name, func(t *testing.T) { + file := fixture.file + before := fmt.Sprintf("%#v", file.Funcs) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + for _, api := range []string{"text", "module", "context"} { + t.Run(triple+"/"+api, func(t *testing.T) { + options := Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"giant": {Name: "giant", Ret: Void}}} + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + module, callErr := TranslateModule(file, options) + err = callErr + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "context": + ctx := llvm.NewContext() + defer ctx.Dispose() + module, callErr := TranslateModuleInContext(ctx, file, options) + err = callErr + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + } + // The exact source envelope is [-2^31, 2^31-1+16), with + // two existing 64-byte synthetic guards: 4,294,967,439 bytes. + if err != nil || !strings.Contains(ir, "%local_stack = alloca [4294967439 x i8]") || before != fmt.Sprintf("%#v", file.Funcs) { + t.Fatalf("allocator observation changed: %v; complete source unchanged=%t", err, before == fmt.Sprintf("%#v", file.Funcs)) + } + dir := t.TempDir() + input, object := filepath.Join(dir, "giant.ll"), filepath.Join(dir, "giant.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, llc, "-O0", "-filetype=obj", "-mtriple="+triple, "-o", object, input) + output, err := runtimeOracleCombinedOutput(ctx, command) + if ctx.Err() != nil { + t.Fatalf("bounded compiler observation expired: %v: %s", err, output) + } + exit, ordinaryExit := err.(*exec.ExitError) + if !ordinaryExit || exit.ExitCode() != 1 { + t.Fatalf("expected plain LLVM32 O0 offset failure, got %v: %s", err, output) + } + diagnostic := "error: :0:0: 64-bit offset calculated but target is 32-bit" + warning := regexp.MustCompile(`^warning: :0:0: stack frame size \(([0-9]+)\) exceeds limit \(4294967295\) in function 'giant'$`) + errors, warnings := 0, 0 + for _, line := range strings.Split(strings.TrimSpace(string(output)), "\n") { + if line == diagnostic { + errors++ + continue + } + match := warning.FindStringSubmatch(line) + if len(match) != 2 { + t.Fatalf("unknown diagnostic is not the observed allocator boundary: %s", output) + } + frameBytes, err := strconv.ParseUint(match[1], 10, 64) + if err != nil || frameBytes <= 4294967295 { + t.Fatalf("compiler warning did not observe an oversized frame: %s", line) + } + warnings++ + } + if warnings != 1 || errors == 0 { + t.Fatalf("missing/changed exact allocator diagnostics: errors=%d warnings=%d", errors, warnings) + } + t.Logf("LLVM_ALLOCATOR_BOUNDARY_OBSERVATION: source SHA=%x; original Go code SHA=%x; %s %s emitted local_stack=4294967439 bytes; compiler O0 offset errors=%d, frame warnings=%d; no object/stack correctness/runtime or source-N/A credit", sha256.Sum256([]byte(fixture.source)), sha256.Sum256(fixture.code), fixture.name, api, errors, warnings) + }) + } + } + }) + } +} diff --git a/amd64_raw_minpos_test.go b/amd64_raw_minpos_test.go new file mode 100644 index 000000000..60145b458 --- /dev/null +++ b/amd64_raw_minpos_test.go @@ -0,0 +1,245 @@ +package plan9asm + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestX86MinimumPositionGrammarMatchesGoEncoder(t *testing.T) { + root := filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86") + for _, form := range []struct { + file, op, table, operands string + vector bool + }{ + {"asm6.go", "PHMINPOSUW", "yxm_q4", "Yxm, Yxr", false}, + {"avx_optabs.go", "VPHMINPOSUW", "_yvaesimc", "Yxm, Yxr", true}, + } { + data, err := os.ReadFile(filepath.Join(root, form.file)) + if err != nil { + t.Fatal(err) + } + text := string(data) + spec, ok := amd64MinimumPositionSpecs[Op(form.op)] + if !ok || spec.vector != form.vector || spec.mapNumber != 2 || spec.opcode != 0x41 { + t.Fatalf("incorrect typed spec for %s: %+v", form.op, spec) + } + pattern := `(?s)var ` + form.table + ` = \[\]ytab\{(.*?)\n\}` + table := regexp.MustCompile(pattern).FindStringSubmatch(text) + if len(table) != 2 || strings.Count(table[1], "argList{") != 1 || + !strings.Contains(table[1], "argList{"+form.operands+"}") { + t.Fatalf("Go %s operand grammar changed", form.op) + } + if form.vector { + pattern = `(?s)\{as: AVPHMINPOSUW, ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}` + rows := regexp.MustCompile(pattern).FindAllStringSubmatch(text, -1) + if len(rows) != 1 || rows[0][1] != form.table || strings.Count(rows[0][2], "0x41") != 1 { + t.Fatal("Go VPHMINPOSUW opcode forms changed") + } + for _, token := range []string{"vex128", "vex66", "vex0F38", "vexW0"} { + if !strings.Contains(rows[0][2], token) { + t.Fatalf("Go VPHMINPOSUW missing %s", token) + } + } + } else if !strings.Contains(text, "{APHMINPOSUW, yxm_q4, Pq4, opBytes{0x41}}") { + t.Fatal("Go PHMINPOSUW encoding changed") + } + } + if len(amd64MinimumPositionSpecs) != 2 { + t.Fatal("minimum-position family must match both Go opcodes") + } +} + +func TestX86RawMinimumPositionEncodingFields(t *testing.T) { + for _, wide := range []bool{false, true} { + for src := 0; src < 16; src++ { + for dst := 0; dst < 16; dst++ { + code := encodeRawScalarMove(false, 1, 0x41, src, 0, dst, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + for _, mode := range []int{32, 64} { + got, size, matched, err := decodedX86MinimumPositionInstruction(code, mode) + if mode == 32 && (src >= 8 || dst >= 8) { + if !matched || err == nil { + t.Fatalf("386 accepted %x", code) + } + continue + } + want := []Operand{ + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", src))}, + {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}, + } + if err != nil || !matched || size != len(code) || got.Op != "VPHMINPOSUW" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %+v", code, mode, got, err, want) + } + } + } + } + } +} + +func TestX86RawMinimumPositionInvalidAndSegmentForms(t *testing.T) { + invalid := [][]byte{ + {0xc4, 0xe2, 0x7d, 0x41, 0xc0}, // VEX.L1. + {0xc4, 0xe2, 0x71, 0x41, 0xc0}, // vvvv is not reserved. + {0xc4, 0xe2, 0x78, 0x41, 0xc0}, // No 66 prefix. + {0x62, 0xf2, 0x7d, 0x08, 0x41, 0xc0}, // No EVEX form. + {0xc4, 0xe2, 0x79, 0x41}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0x41, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0x41, 0x80}, // Missing displacement. + {0xc4, 0xe2, 0x79, 0x41, 0x05, 0, 0, 0, 0}, // RIP relative. + {0x67, 0xc4, 0xe2, 0x79, 0x41, 0x00}, // Address override. + } + for _, code := range invalid { + if _, _, matched, err := decodedX86MinimumPositionInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x", code) + } + } + if _, _, matched, err := decodedX86MinimumPositionInstruction([]byte{0xc4, 0xe2, 0x79, 0x41, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted invalid x86 mode") + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe1, 0x79, 0x41, 0xc0}, {0xc4, 0xe2, 0x79, 0x40, 0xc0}} { + if _, _, matched, _ := decodedX86MinimumPositionInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{segment.prefix, 0xc4, 0x02, 0x79, 0x41, 0x64, 0x88, 0xf9} + got, size, matched, err := decodedX86MinimumPositionInstruction(code, 64) + want := []Operand{ + {Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, + {Kind: OpReg, Reg: "X12"}, + } + if err != nil || !matched || size != len(code) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x: %+v %v, want %+v", code, got, err, want) + } + } +} + +func TestX86MinimumPositionGoRejectedForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + for _, op := range []string{"PHMINPOSUW", "VPHMINPOSUW"} { + forms := []string{ + " X0", " X0,X1,X2", " X0,K1,X1", " X0,(AX)", + " M0,M1", " Y0,Y1", " Z0,Z1", " X16,X0", " X0,X16", + ".Z X0,X1", ".BCST (AX),X1", ".SAE X0,X1", + } + if arch == "386" && op == "PHMINPOSUW" { + forms = append(forms, " X8,X0", " X0,X8") + } + for _, operands := range forms { + line := op + operands + requireX86GoAssemblerResult(t, arch, "TEXT bad(SB),4,$0-0\n"+line+"\nRET\n", false) + assertX86PackedUnsignedWordMinimumPositionRejected(t, arch, triple, line) + } + } + } +} + +func TestX86RawMinimumPositionGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + limit := 16 + if arch == "386" { + limit = 8 + } + memories := []string{"(AX)", "-7(BP)(CX*4)", "8192(SP)"} + if arch == "amd64" { + memories = append(memories, "(R8)", "-7(R13)(R15*8)") + } + var source strings.Builder + source.WriteString("TEXT minpos(SB),4,$0-0\n") + for _, op := range []string{"PHMINPOSUW", "VPHMINPOSUW"} { + for dst := 0; dst < limit; dst++ { + for src := 0; src < limit; src++ { + fmt.Fprintf(&source, "%s X%d,X%d\n", op, src, dst) + } + for _, memory := range memories { + fmt.Fprintf(&source, "%s %s,X%d\n", op, memory, dst) + } + } + } + source.WriteString("RET\n") + code := x86MinimumPositionOriginalGoBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !x86RawOperandsMatchGoSource(got.Args, want[i].Args) { + t.Fatalf("instruction %d %s operands=%#v, want source %#v", i, got.Op, got.Args, want[i].Args) + } + } + file := x86RawStackObservingFormProbeFile(t, arch, "minpos", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{ + Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{"minpos": {Name: "minpos", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-minpos.ll", "raw-minpos.o", ir) + } + t.Logf("checked %d actual Go encodings", len(want)-1) + }) + } +} + +func TestX86MinimumPositionGoAccepted386HighNamedRegisters(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT minpos(SB),4,$0-0\n") + for reg := 8; reg < 16; reg++ { + fmt.Fprintf(&source, "VPHMINPOSUW X%d,X0\nVPHMINPOSUW X0,X%d\nVPHMINPOSUW (AX),X%d\n", reg, reg, reg) + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir, err := Translate(file, Options{ + Goarch: "386", TargetTriple: triple, + Sigs: map[string]FuncSig{"minpos": {Name: "minpos", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "named-minpos.ll", "named-minpos.o", ir) + } +} diff --git a/amd64_raw_mixed_layout.go b/amd64_raw_mixed_layout.go new file mode 100644 index 000000000..8c13c150f --- /dev/null +++ b/amd64_raw_mixed_layout.go @@ -0,0 +1,51 @@ +package plan9asm + +// These legacy SSE/SSE2 symbol loads occur between generated raw instruction +// spans. Their exact Go-assembler lengths are needed only to locate raw +// PC-relative targets; the named instructions still lower semantically. +var x86RawMixedSSESymbolReadOps = map[Op]bool{ + "ADDPD": true, "ADDPS": true, "MULPD": true, "MULPS": true, + "ANDPD": true, "ANDPS": true, "ANDNPD": true, "ANDNPS": true, + "XORPD": true, "XORPS": true, "CMPPD": true, "CMPPS": true, + "PAND": true, "PANDN": true, "POR": true, "PXOR": true, + "PADDB": true, "PADDW": true, "PADDL": true, "PADDQ": true, + "PSUBL": true, "PCMPGTL": true, "PCMPEQL": true, + "PMULHUW": true, "PMADDWL": true, "PMINUB": true, "PMAXUB": true, + "MOVQ": true, +} + +func x86RawMixedSSESymbolReadSize(ins Instr) (int, bool) { + if !x86RawMixedSSESymbolReadOps[ins.Op] || len(ins.Args) < 2 || + ins.Args[0].Kind != OpSym || ins.Args[1].Kind != OpReg { + return 0, false + } + if _, _, ok := parseSBRef(ins.Args[0].Sym); !ok { + return 0, false + } + register, ok := amd64ParseXReg(ins.Args[1].Reg) + if !ok || register >= 16 { + return 0, false + } + compare := ins.Op == "CMPPD" || ins.Op == "CMPPS" + if compare { + if len(ins.Args) != 3 || ins.Args[2].Kind != OpImm || ins.Args[2].Imm < 0 || ins.Args[2].Imm > 255 { + return 0, false + } + } else if len(ins.Args) != 2 { + return 0, false + } + // 0F opcode, ModRM and disp32 = 7 bytes. Packed-double/integer and + // MOVQ forms have one mandatory prefix; X8..X15 add one REX byte. + length := 7 + if ins.Op == "MOVQ" || len(ins.Op) > 0 && ins.Op[0] == 'P' || + len(ins.Op) >= 2 && ins.Op[len(ins.Op)-2:] == "PD" { + length++ + } + if register >= 8 { + length++ + } + if compare { + length++ + } + return length, true +} diff --git a/amd64_raw_mixed_layout_test.go b/amd64_raw_mixed_layout_test.go new file mode 100644 index 000000000..68754640d --- /dev/null +++ b/amd64_raw_mixed_layout_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestDecodeX86RawByteIncrementWidth(t *testing.T) { + code := []byte{0xfe, 0xc1, 0xc3} + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + if inst.Op != x86asm.INC || x86asm.GoSyntax(inst, 0, nil) != "INCL CL" { + t.Fatalf("x/arch INC baseline changed: %+v", inst) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "byte increment", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "INCB" || decoded[0].Args[0].Reg != CL { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawIncrementDecrementCompleteGoForms(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + var source strings.Builder + source.WriteString("TEXT rawIncDec(SB),4,$0-0\n") + for _, op := range []string{"INC", "DEC"} { + for _, form := range []struct { + suffix, register string + }{ + {"B", "CL"}, {"W", "CX"}, {"L", "CX"}, {"Q", "CX"}, + } { + if goarch == "386" && form.suffix == "Q" { + continue + } + fmt.Fprintf(&source, "\t%s%s %s\n", op, form.suffix, form.register) + fmt.Fprintf(&source, "\t%s%s 8(AX)\n", op, form.suffix) + } + } + source.WriteString("\tRET\n") + code := assembleX87ControlBytes(t, goarch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), goarch) + if err != nil { + t.Fatalf("%s: %v", goarch, err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("%s decoded %d instructions, want %d", goarch, len(decoded.Instrs), len(want)) + } + for index, ins := range decoded.Instrs { + if ins.Op != want[index].Op || !reflect.DeepEqual(ins.Args, want[index].Args) { + t.Fatalf("%s instruction %d=%+v, want %+v", goarch, index, ins, want[index]) + } + } + } +} + +func TestX86MixedSSESymbolReadLengthsMatchGoAssembler(t *testing.T) { + for op := range x86RawMixedSSESymbolReadOps { + for _, register := range []string{"X0", "X10"} { + t.Run(string(op)+"/"+register, func(t *testing.T) { + instruction := fmt.Sprintf("%s pool(SB), %s", op, register) + if op == "CMPPS" || op == "CMPPD" { + instruction += ", $5" + } + source := "TEXT width(SB),4,$0-0\n\t" + instruction + "\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + length, ok := x86RawMixedSSESymbolReadSize(file.Funcs[0].Instrs[1]) + if !ok || length != len(code)-1 { + t.Fatalf("%s length=%d, recognized=%t, Go bytes=%x", instruction, length, ok, code) + } + }) + } + } +} + +func TestTranslateX86MixedRawJumpAcrossNamedSSERead(t *testing.T) { + const source = `TEXT mixedRawJump(SB),4,$0-0 + BYTE $0xe9 + LONG $8 + POR pool(SB), X0 + BYTE $0xc3 +` + code := assembleX87ControlBytes(t, "amd64", source) + if len(code) != 14 || code[0] != 0xe9 || code[13] != 0xc3 { + t.Fatalf("unexpected Go assembler layout: %x", code) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"mixedRawJump": {Name: "mixedRawJump", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, amd64LLVMBlockName("raw_jcc_")) { + t.Fatal("cross-segment branch label missing") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, triple, "mixed-raw-jump.ll", "mixed-raw-jump.o", ir) + }) + } +} diff --git a/amd64_raw_movbe_width_test.go b/amd64_raw_movbe_width_test.go new file mode 100644 index 000000000..fc02d5315 --- /dev/null +++ b/amd64_raw_movbe_width_test.go @@ -0,0 +1,77 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func x86RawMOVBE(width int, store bool, register int) []byte { + var code []byte + if width == 16 { + code = append(code, 0x66) + } + rex := byte(0x40 | (register>>3&1)<<2) + if width == 64 { + rex |= 0x08 + } + if rex != 0x40 { + code = append(code, rex) + } + opcode := byte(0xf0) + if store { + opcode = 0xf1 + } + code = append(code, 0x0f, 0x38, opcode, byte((register&7)<<3|0x02), 0xc3) + return code +} + +func TestDecodeX86RawMOVBECompleteWidthsAndDirections(t *testing.T) { + for _, width := range []int{16, 32, 64} { + for _, store := range []bool{false, true} { + for _, register := range []int{0, 7, 8, 15} { + name := fmt.Sprintf("%d/store%t/reg%d", width, store, register) + t.Run(name, func(t *testing.T) { + code := x86RawMOVBE(width, store, register) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := map[int]Op{16: "MOVBEW", 32: "MOVBEL", 64: "MOVBEQ"}[width] + if len(got) != 2 || got[0].Op != want || got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s", code, got, want) + } + }) + } + } + } +} + +func TestTranslateX86RawMOVBEObjects(t *testing.T) { + file := &File{Arch: ArchAMD64} + sigs := make(map[string]FuncSig) + index := 0 + for _, width := range []int{16, 32, 64} { + for _, store := range []bool{false, true} { + name := fmt.Sprintf("rawMOVBE%d", index) + probe := x86RawUnprovedReturnFormProbeFile(t, "amd64", name, x86RawMOVBE(width, store, 8)) + file.Funcs = append(file.Funcs, probe.Funcs...) + sigs[name] = FuncSig{Name: name, Ret: Void} + index++ + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-movbe.ll", "raw-movbe.o", ir) + }) + } +} diff --git a/amd64_raw_movemask.go b/amd64_raw_movemask.go new file mode 100644 index 000000000..38a5d3dfe --- /dev/null +++ b/amd64_raw_movemask.go @@ -0,0 +1,60 @@ +package plan9asm + +import "fmt" + +// Decode Go's VEX _yvmovmskpd family before x/arch can mistake opcode 50/D7 +// for a legacy PUSH/XLATB. The source is always a vector register; ModRM.reg +// names the GP destination and vvvv is reserved. +func decodedX86VEXMoveMaskInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.evex || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + var op Op + for name, spec := range amd64MoveMaskSpecs { + if spec.vector && spec.opcode == p.opcode && spec.prefix == p.pp { + op = name + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VEX move mask: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 || p.vectorLength > 1 { + return fail("reserved vvvv or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 != 3 { + return fail("source must be a vector register") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + width := "X" + if p.vectorLength == 1 { + width = "Y" + } + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]&7)+p.b*8))} + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + destinationReg, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return fail("invalid GP destination register") + } + destination := Operand{Kind: OpReg, Reg: destinationReg} + return Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + }, p.modRM + 1, true, nil +} diff --git a/amd64_raw_movemask_test.go b/amd64_raw_movemask_test.go new file mode 100644 index 000000000..f41ecd2a9 --- /dev/null +++ b/amd64_raw_movemask_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +var x86RawMoveMaskForms = []struct { + name string + code []byte + op Op + src Reg + dst Reg +}{ + {"PMOVMSKB/MMX", []byte{0x0f, 0xd7, 0xc1}, "PMOVMSKB", "M1", AX}, + {"PMOVMSKB/X", []byte{0x66, 0x0f, 0xd7, 0xc1}, "PMOVMSKB", "X1", AX}, + {"MOVMSKPS/X", []byte{0x0f, 0x50, 0xc1}, "MOVMSKPS", "X1", AX}, + {"MOVMSKPD/X", []byte{0x66, 0x0f, 0x50, 0xc1}, "MOVMSKPD", "X1", AX}, + {"VPMOVMSKB/X", []byte{0xc5, 0xf9, 0xd7, 0xc1}, "VPMOVMSKB", "X1", AX}, + {"VPMOVMSKB/Y", []byte{0xc5, 0xfd, 0xd7, 0xc1}, "VPMOVMSKB", "Y1", AX}, + {"VMOVMSKPS/X", []byte{0xc5, 0xf8, 0x50, 0xc1}, "VMOVMSKPS", "X1", AX}, + {"VMOVMSKPS/Y", []byte{0xc5, 0xfc, 0x50, 0xc1}, "VMOVMSKPS", "Y1", AX}, + {"VMOVMSKPD/X", []byte{0xc5, 0xf9, 0x50, 0xc1}, "VMOVMSKPD", "X1", AX}, + {"VMOVMSKPD/Y", []byte{0xc5, 0xfd, 0x50, 0xc1}, "VMOVMSKPD", "Y1", AX}, +} + +func TestX86RawMoveMaskCompleteGo127Family(t *testing.T) { + if len(amd64MoveMaskSpecs) != 6 { + t.Fatalf("move-mask grammar has %d operations, want six", len(amd64MoveMaskSpecs)) + } + for _, goarch := range []string{"386", "amd64"} { + for _, form := range x86RawMoveMaskForms { + t.Run(goarch+"/"+form.name, func(t *testing.T) { + fn := Func{} + for _, value := range form.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != form.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", form.code, got.Instrs, form.op) + } + args := got.Instrs[0].Args + if args[0].Kind != OpReg || args[0].Reg != form.src || args[1].Kind != OpReg || args[1].Reg != form.dst { + t.Fatalf("decoded %#x as %#v, want %s, %s", form.code, args, form.src, form.dst) + } + }) + } + } +} + +func TestX86RawMoveMaskRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"reserved-vvvv", []byte{0xc5, 0xe9, 0xd7, 0xc1}, 64}, + {"memory-source", []byte{0xc5, 0xfd, 0xd7, 0x01}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfd, 0xd7, 0xc1}, 64}, + {"missing-modrm", []byte{0xc5, 0xfd, 0xd7}, 64}, + {"extended-386", []byte{0xc4, 0x61, 0x7d, 0xd7, 0xc1}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86VEXMoveMaskInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } + for _, tc := range []struct { + name string + code []byte + src Reg + dst Reg + }{ + {"W-ignored", []byte{0xc4, 0xe1, 0xfd, 0xd7, 0xc1}, "Y1", AX}, + {"extended-vector-and-GP", []byte{0xc4, 0x41, 0x7d, 0xd7, 0xc8}, "Y8", "R9"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86VEXMoveMaskInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) { + t.Fatalf("encoding %#x: instruction=%+v length=%d recognized=%v error=%v", tc.code, got, length, recognized, err) + } + if got.Args[0].Reg != tc.src || got.Args[1].Reg != tc.dst { + t.Fatalf("encoding %#x operands = %+v, want %s, %s", tc.code, got.Args, tc.src, tc.dst) + } + }) + } +} + +func TestX86RawMoveMaskCompilesEveryGoForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawmovemaskforms(SB),4,$0-0\n") + for _, form := range x86RawMoveMaskForms { + for _, value := range form.code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawmovemaskforms": {Name: "rawmovemaskforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-movemask.ll", "raw-movemask.o", ir) + }) + } +} + +func TestX86RawMoveMaskRuntime(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution requires amd64 or Rosetta") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const source = `TEXT rawmovemasksemantics(SB),4,$0-16 + MOVQ input+0(FP), DI + MOVQ out+8(FP), SI + VMOVDQU (DI), Y1 + BYTE $0xc5; BYTE $0xfd; BYTE $0xd7; BYTE $0xc1 + MOVL AX, 0(SI) + BYTE $0xc5; BYTE $0xf8; BYTE $0x50; BYTE $0xc1 + MOVL AX, 4(SI) + BYTE $0xc5; BYTE $0xfd; BYTE $0x50; BYTE $0xc1 + MOVL AX, 8(SI) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawmovemasksemantics": { + Name: "rawmovemasksemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void rawmovemasksemantics(const uint8_t *, uint32_t *); +int main(void) { + uint8_t input[32] = {0}; + uint32_t out[3] = {0}; + input[3] = 0x80; input[7] = 0x80; input[15] = 0x80; input[31] = 0x80; + rawmovemasksemantics(input, out); + if (out[0] != ((1u << 3) | (1u << 7) | (1u << 15) | (1u << 31))) return 1; + if (out[1] != ((1u << 0) | (1u << 1) | (1u << 3))) return 2; + if (out[2] != ((1u << 0) | (1u << 1) | (1u << 3))) return 3; + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_move_mask", triple, ir, mainC, runPrefix) +} diff --git a/amd64_raw_movhlps.go b/amd64_raw_movhlps.go new file mode 100644 index 000000000..d5332dbde --- /dev/null +++ b/amd64_raw_movhlps.go @@ -0,0 +1,65 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +var x86RawVectorHighLowMoveOps = map[int]Op{ + 0x12: "VMOVHLPS", + 0x16: "VMOVLHPS", +} + +// The Go 1.27 _yvmovhlps table has exactly two register-only 128-bit rows. +// ModRM memory encodings belong to the separate VMOVLPS/VMOVHPS family. +func decodedX86RawVectorHighLowMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.mapNumber != 1 || p.pp != 0 { + return Instr{}, 0, false, nil + } + op, owned := x86RawVectorHighLowMoveOps[p.opcode] + if !owned { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector high/low move: %s", message) + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 != 3 { + return Instr{}, 0, false, nil + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength != 0 || p.evex && (!p.fixed || p.w || p.mask != 0 || p.zero || p.broadcast) { + return fail("invalid 128-bit VEX/EVEX register form") + } + if !p.evex && mode == 32 && (p.r != 0 || p.upper >= 8) { + return fail("extended VEX register in 32-bit mode") + } + var first Operand + var consumed int + var err error + if p.evex { + first, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", 16) + } else { + first, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))} + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", destinationNumber))} + args := []Operand{first, second, destination} + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_movhlps_test.go b/amd64_raw_movhlps_test.go new file mode 100644 index 000000000..8d8a9a664 --- /dev/null +++ b/amd64_raw_movhlps_test.go @@ -0,0 +1,168 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVectorHighLowMove(opcode byte, encoding string, first, second, destination int, wig bool) []byte { + modRM := byte(0xc0 | (destination&7)<<3 | first&7) + switch encoding { + case "VEX2": + p1 := byte((1-destination/8)<<7 | (^second&15)<<3) + return []byte{0xc5, p1, opcode, modRM} + case "VEX3": + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-first/8)<<5 | 1) + p1 := byte((^second & 15) << 3) + if wig { + p1 |= 0x80 + } + return []byte{0xc4, p0, p1, opcode, modRM} + case "EVEX": + p0 := byte((1-destination/8&1)<<7 | (1-first/16)<<6 | + (1-first/8&1)<<5 | (1-destination/16)<<4 | 1) + p1 := byte(4 | (^second&15)<<3) + p2 := byte((1 - second/16) << 3) + return []byte{0x62, p0, p1, p2, opcode, modRM} + default: + panic("unknown high/low move encoding") + } +} + +func TestTranslateRawVectorHighLowMoveGorseRegression(t *testing.T) { + const source = `TEXT rawMoveLowToHigh(SB), $0-0 + LONG $0xc016f0c5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawMoveLowToHigh": {Name: "rawMoveLowToHigh", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <2 x i64>") { + t.Fatalf("raw VMOVLHPS omitted half-lane move:\n%s", ir) + } +} + +func TestDecodedX86RawVectorHighLowMoveGoRowsAndEncodings(t *testing.T) { + if len(x86RawVectorHighLowMoveOps) != 2 || + x86RawVectorHighLowMoveOps[0x12] != "VMOVHLPS" || + x86RawVectorHighLowMoveOps[0x16] != "VMOVLHPS" { + t.Fatal("Go 1.27 _yvmovhlps raw opcode grammar changed") + } + for opcode, op := range x86RawVectorHighLowMoveOps { + for _, test := range []struct { + encoding string + first, second, dest int + wig bool + }{ + {"VEX2", 1, 2, 3, false}, + {"VEX3", 12, 13, 14, false}, + {"VEX3", 12, 13, 14, true}, // VEX.W is ignored by the hardware. + {"EVEX", 20, 21, 22, false}, + } { + code := encodeX86RawVectorHighLowMove(byte(opcode), test.encoding, test.first, test.second, test.dest, test.wig) + got, length, recognized, err := decodedX86RawVectorHighLowMoveInstruction(code, 64) + if err != nil || !recognized || length != len(code) || got.Op != op { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + want := []string{ + fmt.Sprintf("X%d", test.first), + fmt.Sprintf("X%d", test.second), + fmt.Sprintf("X%d", test.dest), + } + if len(got.Args) != len(want) { + t.Fatalf("decode %x = %+v, want %v", code, got, want) + } + for index, operand := range got.Args { + if operand.String() != want[index] { + t.Fatalf("decode %x = %+v, want %v", code, got, want) + } + } + } + } +} + +func TestDecodedX86RawVectorHighLowMoveRejectsNonGoForms(t *testing.T) { + base := encodeX86RawVectorHighLowMove(0x16, "EVEX", 1, 2, 3, false) + for _, test := range []struct { + name string + code []byte + }{ + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"EVEX W", []byte{base[0], base[1], base[2] | 0x80, base[3], base[4], base[5]}}, + {"vector length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"mask", []byte{base[0], base[1], base[2], base[3] | 1, base[4], base[5]}}, + {"zero", []byte{base[0], base[1], base[2], base[3] | 0x80, base[4], base[5]}}, + {"broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawVectorHighLowMoveInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + memory := append([]byte(nil), base...) + memory[5] = 0x43 + if _, _, recognized, err := decodedX86RawVectorHighLowMoveInstruction(memory, 64); recognized || err != nil { + t.Fatalf("claimed VMOVHPS memory encoding %x: recognized=%v err=%v", memory, recognized, err) + } +} + +func TestTranslateRawVectorHighLowMoveLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawHighLowMoveFamily(SB), $0-0\n") + for opcode := range x86RawVectorHighLowMoveOps { + for _, code := range [][]byte{ + encodeX86RawVectorHighLowMove(byte(opcode), "VEX2", 1, 2, 3, false), + encodeX86RawVectorHighLowMove(byte(opcode), "VEX3", 4, 5, 6, true), + encodeX86RawVectorHighLowMove(byte(opcode), "EVEX", 20, 21, 22, false), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawHighLowMoveFamily": {Name: "rawHighLowMoveFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-high-low-move.ll", "raw-high-low-move.o", ir) + }) + } +} diff --git a/amd64_raw_movsxd.go b/amd64_raw_movsxd.go new file mode 100644 index 000000000..96fb826b0 --- /dev/null +++ b/amd64_raw_movsxd.go @@ -0,0 +1,89 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Opcode 63 is ARPL outside 64-bit mode. In 64-bit mode its default +// operation is a 32-bit copy, 66 selects a 16-bit copy, and effective +// REX.W selects a 32-to-64-bit sign extension. See Intel SDM volume 2B, +// MOVSX/MOVSXD. A 16-bit memory operand remains Context: Intel's r/m16 +// and AMD/x86asm's r/m32 descriptions do not establish one read extent. +func decodedX86MOVSXD(inst x86asm.Inst, encoding []byte) (string, error) { + fail := func(reason string) (string, error) { + return "", fmt.Errorf("%w: raw MOVSXD %s", ErrProbeNeedsContext, reason) + } + if inst.Op != x86asm.MOVSXD || inst.Mode != 64 || inst.Len != len(encoding) { + return fail("requires its complete 64-bit-mode physical encoding") + } + physical, decodeErr := x86asm.Decode(encoding, 64) + if decodeErr != nil || physical != inst { + return fail("does not match the independently decoded original bytes") + } + // Bind the effective size and register extension to actual bytes, not a + // public DataSize or a widthless mnemonic. Deliberately leave unregistered + // ignored/repeated/segment/address prefixes outside this closed grammar. + index, bits := 0, 32 + if index < len(encoding) && encoding[index] == 0x66 { + bits = 16 + index++ + } + if index < len(encoding) && encoding[index]&0xf0 == 0x40 { + if encoding[index]&8 != 0 { + bits = 64 + } + index++ + } + if index+1 >= len(encoding) || encoding[index] != 0x63 || inst.DataSize != bits || inst.AddrSize != 64 { + return fail("has an unregistered prefix, opcode or effective size") + } + destination, register := inst.Args[0].(x86asm.Reg) + if !register || decodedX86RegisterBits(destination) != bits || inst.Args[2] != nil || inst.Args[3] != nil { + return fail("has inconsistent destination or extra operands") + } + switch source := inst.Args[1].(type) { + case x86asm.Reg: + // The pinned decoder describes every 63 source as r/m32, including + // 66 register forms. The latter consume only its low sixteen bits. + if decodedX86RegisterBits(source) != 32 || encoding[index+1]&0xc0 != 0xc0 { + return fail("has inconsistent register source or ModRM") + } + if bits == 16 { + inst.Args[1] = x86asm.AX + (source - x86asm.EAX) + } + case x86asm.Mem: + if encoding[index+1]&0xc0 == 0xc0 || inst.MemBytes != 4 || source.Segment != 0 { + return fail("has inconsistent memory source or native read extent") + } + if bits == 16 { + return fail("16-bit memory needs a source-bound vendor read-width contract") + } + default: + return fail("requires a register or memory source") + } + if bits == 64 { + // Go has a width-specific name for the genuine sign extension. + syntax := x86asm.GoSyntax(inst, 0, nil) + return "MOVLQSX" + syntax[len("MOVSXD"):], nil + } + inst.Op = x86asm.MOV + if bits == 16 { + inst.MemBytes = 2 + } + return x86asm.GoSyntax(inst, 0, nil), nil +} + +func x86RawMOVSXDLiteralInput(ins Instr) bool { + // This is only the source-stack transfer's input-availability check. The + // final consumer must still bind the exact bytes to the allocated DATA + // object through x86RawReturnDecodedReadBound; no GP input is inferred. + read := ins.x86RIPMemoryRead + if ins.Op != "MOVLQSX" || read.source != 0 || read.width != 4 || len(ins.Args) != 2 || + ins.Args[1].Kind != OpReg || !x86RawReturnDecodedReadIdentity(ins) { + return false + } + result, valid := x86CallerGP(ins.Args[1].Reg) + return valid && result != SP && result != BP && result == ins.Args[1].Reg +} diff --git a/amd64_raw_movsxd_runtime_test.go b/amd64_raw_movsxd_runtime_test.go new file mode 100644 index 000000000..f4cf66e9f --- /dev/null +++ b/amd64_raw_movsxd_runtime_test.go @@ -0,0 +1,325 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +type x86MOVSXDNumericalForm struct { + name string + encoding []byte + bits int + offset int + setup string + result string + inPlace bool + flagSeed int +} + +func x86MOVSXDNumericalForms() []x86MOVSXDNumericalForm { + forms := []x86MOVSXDNumericalForm{ + {name: "wordBX", encoding: []byte{0x66, 0x63, 0xc3}, bits: 16}, + {name: "wordSI", encoding: []byte{0x66, 0x63, 0xc6}, bits: 16, setup: "MOVQ BX,SI\n"}, + {name: "wordR9", encoding: []byte{0x66, 0x41, 0x63, 0xc1}, bits: 16, setup: "MOVQ BX,R9\n"}, + {name: "wordR8", encoding: []byte{0x66, 0x44, 0x63, 0xc3}, bits: 16, result: "MOVQ R8,AX\n"}, + {name: "wordAlias", encoding: []byte{0x66, 0x63, 0xc0}, bits: 16, setup: "MOVQ BX,AX\n", inPlace: true}, + {name: "longBX", encoding: []byte{0x63, 0xc3}, bits: 32}, + {name: "longR9", encoding: []byte{0x41, 0x63, 0xc1}, bits: 32, setup: "MOVQ BX,R9\n"}, + {name: "longR8", encoding: []byte{0x44, 0x63, 0xc3}, bits: 32, result: "MOVQ R8,AX\n"}, + {name: "longAlias", encoding: []byte{0x63, 0xc0}, bits: 32, setup: "MOVQ BX,AX\n", inPlace: true}, + {name: "longMemory", encoding: []byte{0x63, 0x07}, bits: 32}, + {name: "longUnaligned", encoding: []byte{0x63, 0x47, 0x01}, bits: 32, offset: 1}, + {name: "longOffset4", encoding: []byte{0x63, 0x47, 0x04}, bits: 32, offset: 4}, + {name: "longIndexed", encoding: []byte{0x63, 0x04, 0x8f}, bits: 32, setup: "MOVQ $0,CX\n"}, + {name: "quadBX", encoding: []byte{0x48, 0x63, 0xc3}, bits: 64}, + {name: "quadR9", encoding: []byte{0x49, 0x63, 0xc1}, bits: 64, setup: "MOVQ BX,R9\n"}, + {name: "quadR8", encoding: []byte{0x4c, 0x63, 0xc3}, bits: 64, result: "MOVQ R8,AX\n"}, + {name: "quadAlias", encoding: []byte{0x48, 0x63, 0xc0}, bits: 64, setup: "MOVQ BX,AX\n", inPlace: true}, + {name: "quadMemory", encoding: []byte{0x48, 0x63, 0x07}, bits: 64}, + {name: "quadUnaligned", encoding: []byte{0x48, 0x63, 0x47, 0x01}, bits: 64, offset: 1}, + {name: "quadOffset4", encoding: []byte{0x48, 0x63, 0x47, 0x04}, bits: 64, offset: 4}, + {name: "quadIndexed", encoding: []byte{0x48, 0x63, 0x04, 0x8f}, bits: 64, setup: "MOVQ $0,CX\n"}, + {name: "quadOperandOverride", encoding: []byte{0x66, 0x48, 0x63, 0x07}, bits: 64}, + } + var seeded []x86MOVSXDNumericalForm + for _, form := range forms { + for seed, suffix := range []string{"", "Carry", "Overflow"} { + variant := form + variant.name += suffix + variant.flagSeed = seed + seeded = append(seeded, variant) + } + } + return seeded +} + +func x86MOVSXDFlagSeed(seed int) string { + switch seed { + case 1: + // 0-1: CF=1, OF=0, SF=1, PF=1, ZF=0. + return "MOVQ $0,R15\nCMPQ R15,$1\n" + case 2: + // INT64_MIN-1: CF=0, OF=1, SF=0, PF=1, ZF=0. + return "MOVQ $0x8000000000000000,R15\nCMPQ R15,$1\n" + default: + return "CMPQ BX,$0\n" + } +} + +func x86MOVSXDNumericalSource() (string, string) { + var declarations, assembly strings.Builder + declarations.WriteString("package caller\nimport \"unsafe\"\n") + for _, form := range x86MOVSXDNumericalForms() { + declaration := strings.ReplaceAll(x86CallerFixtureSource, "float32", "uint64") + declaration = declaration[strings.Index(declaration, "//go:noescape"):] + declaration = strings.ReplaceAll(declaration, "kernel", form.name) + declaration = strings.Replace(declaration, "func Call(", "func Call"+form.name+"(", 1) + declarations.WriteString(declaration) + body := "MOVQ (DI),BX\nMOVQ $0x5a6b7c8d9eaf1021,AX\nMOVQ AX,R8\n" + form.setup + body += x86MOVSXDFlagSeed(form.flagSeed) + x86MOVSXDBytes(form.encoding) + form.result + // Neutral MOVs capture each modeled flag before any arithmetic. The + // branch joins preserve definite numeric bytes, not guessed values. + body += `MOVQ $0,R9 +MOVQ $0,R10 +MOVQ $0,R11 +MOVQ $0,R12 +MOVQ $0,R13 +JCC noCarry +MOVQ $1,R9 +noCarry: +JOC noOverflow +MOVQ $2,R10 +noOverflow: +JPL noSign +MOVQ $4,R11 +noSign: +JPC noParity +MOVQ $8,R12 +noParity: +JNE noZero +MOVQ $16,R13 +noZero: +JE done +ADDQ $7,AX +done: +ORQ R9,R10 +ORQ R11,R10 +ORQ R12,R10 +ORQ R13,R10 +SHLQ $48,R10 +XORQ R10,AX +MOVQ AX,(DX)` + machine := strings.ReplaceAll(x86CallerFixtureAssembly, "·kernel", "·"+form.name) + assembly.WriteString(strings.Replace(machine, "MOVL $7,AX\nMOVL AX,(DX)", body, 1)) + } + return declarations.String(), assembly.String() +} + +func x86MOVSXDGoOracle() string { + var source strings.Builder + source.WriteString(`package caller +import "testing" +func reference(input uint64, bits, offset uint, inPlace bool, flagSeed uint) uint64 { + value := uint64(uint32(input >> (offset*8))) + if bits == 16 { + seed := uint64(0x5a6b7c8d9eaf1021) + if inPlace { seed = input } + value = seed & ^uint64(65535) | input & 65535 + } else if bits == 64 { + value = uint64(int64(int32(value))) + } + if input != 0 || flagSeed != 0 { value += 7 } + flags := uint64(0) + if input>>63 != 0 { flags |= 4 } + var ones uint + for byteValue := uint8(input); byteValue != 0; byteValue >>= 1 { ones += uint(byteValue&1) } + if ones%2 == 0 { flags |= 8 } + if input == 0 { flags |= 16 } + if flagSeed == 1 { flags = 1 | 4 | 8 } + if flagSeed == 2 { flags = 2 | 8 } + return value ^ (flags<<48) +} +func TestOriginalMOVSXDNumerical(t *testing.T) { +`) + for _, form := range x86MOVSXDNumericalForms() { + fmt.Fprintf(&source, ` + for iteration := 0; iteration < 16; iteration++ { + for _, input := range []uint64{0, 1, 127, 128, 65535, 1<<31, 1<<63, ^uint64(0)} { + x := []uint64{input, 0x1122334455667788} + y := []uint64{0x8877665544332211, 23} + canary := [4]uint64{11, 23, 37, 41} + got := Call%[1]s(x, y) + want := reference(input, %[2]d, %[3]d, %[4]t, %[5]d) + if got != want || x[0] != input || x[1] != 0x1122334455667788 || + y[0] != 0x8877665544332211 || y[1] != 23 || canary != [4]uint64{11,23,37,41} { + t.Fatalf("%[1]s got=%%#x want=%%#x input=%%#x canary=%%v", got, want, input, canary) + } + } + } +`, form.name, form.bits, form.offset, form.inPlace, form.flagSeed) + } + source.WriteString("}\n") + return source.String() +} + +func x86MOVSXDActualGo(t *testing.T, execute bool) { + t.Helper() + declaration, assembly := x86MOVSXDNumericalSource() + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module caller\n\ngo 1.20\n", "value.go": declaration, + "value_amd64.s": assembly, "value_test.go": x86MOVSXDGoOracle(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + binary := filepath.Join(dir, "original.test") + command := exec.CommandContext(ctx, "go", "test", "-p=2", "-vet=off", "-c", "-o", binary, ".") + command.Dir = dir + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("actual Go original raw MOVSXD build: %v\n%s", err, output) + } + if !execute { + return // Object/Go entry transport evidence is not runtime success. + } + command = exec.CommandContext(ctx, binary, "-test.v", "-test.timeout=2m") + if output, err := runtimeOracleCombinedOutput(ctx, command); err != nil { + t.Fatalf("original Linux/amd64 MOVSXD mathematical/canary oracle: %v\n%s", err, output) + } +} + +func x86MOVSXDCOracle() string { + var declarations, checks strings.Builder + for index, form := range x86MOVSXDNumericalForms() { + fmt.Fprintf(&declarations, "extern void %s(void*,void*,void*,void*) __asm__(LABEL(\"caller.%s\"));\n", form.name, form.name) + fmt.Fprintf(&checks, " if (check(%s, %d, %d, %t, %d)) return %d;\n", + form.name, form.bits, form.offset, form.inPlace, form.flagSeed, index+1) + } + return `#include +#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +` + declarations.String() + ` +static uint64_t reference(uint64_t input, unsigned bits, unsigned offset, bool in_place, unsigned flag_seed) { + uint64_t value = (uint32_t)(input >> (offset*8)); + if (bits == 16) { + uint64_t seed = in_place ? input : UINT64_C(0x5a6b7c8d9eaf1021); + value = (seed & ~UINT64_C(65535)) | (input & UINT64_C(65535)); + } else if (bits == 64) { + value = (uint64_t)(int64_t)(int32_t)value; + } + value += input != 0 || flag_seed != 0 ? 7 : 0; + uint64_t flags = input >> 63 ? 4 : 0; + unsigned ones = 0; + for (unsigned byte_value = (unsigned char)input; byte_value; byte_value >>= 1) ones += byte_value & 1; + if (!(ones & 1)) flags |= 8; + if (!input) flags |= 16; + if (flag_seed == 1) flags = 1 | 4 | 8; + if (flag_seed == 2) flags = 2 | 8; + return value ^ (flags << 48); +} +static int check(void (*fn)(void*,void*,void*,void*), unsigned bits, unsigned offset, bool in_place, unsigned flag_seed) { + const uint64_t inputs[] = {0, 1, 127, 128, 65535, UINT64_C(1)<<31, UINT64_C(1)<<63, UINT64_MAX}; + for (unsigned iteration = 0; iteration < 16; iteration++) { + for (unsigned index = 0; index < sizeof(inputs)/sizeof(inputs[0]); index++) { + uint64_t x[] = {inputs[index], UINT64_C(0x1122334455667788)}; + uint64_t y[] = {UINT64_C(0x8877665544332211), 23}, length = 2; + unsigned char output[10]; + memset(output, 0xa5, sizeof(output)); + fn(x, y, output+1, &length); + uint64_t got; + memcpy(&got, output+1, sizeof(got)); + if (got != reference(inputs[index], bits, offset, in_place, flag_seed) || + output[0] != 0xa5 || output[9] != 0xa5 || length != 2 || + x[0] != inputs[index] || x[1] != UINT64_C(0x1122334455667788) || + y[0] != UINT64_C(0x8877665544332211) || y[1] != 23) return 1; + } + } + return 0; +} +int main(void) { +` + checks.String() + " return 0;\n}\n" +} + +func x86MOVSXDLLVMNumerical(t *testing.T, runner []string) { + t.Helper() + llc, clang, found := findLlcAndClang(t) + if !found { + t.Fatal("LLVM22 llc/clang required") + } + declaration, assembly := x86MOVSXDNumericalSource() + pkg := x86CallerRuntimePackage(t, declaration, assembly) + file := pkg.SourceSelection.Assembly["value_amd64.s"] + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, "amd64") + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "movsxd-text", triple, ir, x86MOVSXDCOracle(), runner) + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "movsxd-module", triple, module.String(), x86MOVSXDCOracle(), runner) + ctx = llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(assembly), GoModuleOptions{ + FileName: "value_amd64.s", GOARCH: "amd64", TargetTriple: triple, ResolveSym: resolve, + }) + if translation != nil && !translation.Module.IsNil() { + defer translation.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "movsxd-go-module", triple, translation.Module.String(), x86MOVSXDCOracle(), runner) +} + +func TestX86RawMOVSXDOriginalGoBuild(t *testing.T) { + x86MOVSXDActualGo(t, false) +} + +func TestX86RawMOVSXDLLVMHostNumerical(t *testing.T) { + var runner []string + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() { + runner = []string{"/usr/bin/arch", "-x86_64"} + t.Log("LLVM numerical diagnostic uses Rosetta; the original no-REX 63 environment failure is retained separately") + } else if runtime.GOARCH != "amd64" { + t.Skip("requires the mandatory Linux/amd64 original-Go plus LLVM counterpart") + } + x86MOVSXDLLVMNumerical(t, runner) +} + +func TestCrossLinuxRuntimeMatrixAMD64MOVSXDNativeWidths(t *testing.T) { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Skip("required actual Linux/amd64 original-Go and LLVM counterpart; Darwin/Rosetta is not equivalent") + } + x86MOVSXDActualGo(t, true) + x86MOVSXDLLVMNumerical(t, nil) +} diff --git a/amd64_raw_movsxd_test.go b/amd64_raw_movsxd_test.go new file mode 100644 index 000000000..f4e159f69 --- /dev/null +++ b/amd64_raw_movsxd_test.go @@ -0,0 +1,464 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +func x86MOVSXDBytes(encoding []byte) string { + var source strings.Builder + for _, value := range encoding { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func TestX86RawMOVSXDNativeWidths(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + want string + }{ + {"long register", []byte{0x63, 0xc3}, "MOVL BX, AX"}, + {"long high source", []byte{0x41, 0x63, 0xc1}, "MOVL R9, AX"}, + {"long high result", []byte{0x44, 0x63, 0xc3}, "MOVL BX, R8"}, + {"long alias", []byte{0x63, 0xc0}, "MOVL AX, AX"}, + {"long memory", []byte{0x63, 0x07}, "MOVL 0(DI), AX"}, + {"long indexed", []byte{0x63, 0x44, 0x8f, 0x03}, "MOVL 0x3(DI)(CX*4), AX"}, + {"word register", []byte{0x66, 0x63, 0xc3}, "MOVW BX, AX"}, + {"word physical SI", []byte{0x66, 0x63, 0xc6}, "MOVW SI, AX"}, + {"word high source", []byte{0x66, 0x41, 0x63, 0xc1}, "MOVW R9, AX"}, + {"word high result", []byte{0x66, 0x44, 0x63, 0xc3}, "MOVW BX, R8"}, + {"quad register", []byte{0x48, 0x63, 0xc3}, "MOVLQSX BX, AX"}, + {"quad memory", []byte{0x48, 0x63, 0x07}, "MOVLQSX 0(DI), AX"}, + } { + t.Run(test.name, func(t *testing.T) { + inst, err := x86asm.Decode(test.encoding, 64) + if err != nil || inst.Op != x86asm.MOVSXD || inst.Len != len(test.encoding) { + t.Fatalf("physical decode: %+v %v", inst, err) + } + got, err := decodedX86GoSyntax(inst, test.encoding) + if err != nil || got != test.want { + t.Fatalf("physical width: got %q, want %q: %v", got, test.want, err) + } + }) + } +} + +func TestX86RawMOVSXDLongCallerOriginalBytes(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL $7,AX\nMOVL AX,(DX)", + "MOVQ $0x80000000,BX\nWORD $0xc363\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, _ := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + if err := x86CallerCoreTranslate(t, pkg, assembly, "x86_64-unknown-linux-gnu"); err != nil { + t.Fatal(err) + } +} + +func TestX86RawMOVSXDCompleteRegisterModRM(t *testing.T) { + registers := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + for _, bits := range []int{16, 32, 64} { + for result := range registers { + for source := range registers { + var encoding []byte + if bits == 16 { + encoding = append(encoding, 0x66) + } + rex := byte(0x40) + if bits == 64 { + rex |= 8 + } + if result >= 8 { + rex |= 4 + } + if source >= 8 { + rex |= 1 + } + if rex != 0x40 { + encoding = append(encoding, rex) + } + encoding = append(encoding, 0x63, byte(0xc0+(result%8)*8+source%8)) + inst, err := x86asm.Decode(encoding, 64) + if err != nil { + t.Fatal(err) + } + before := inst + got, err := decodedX86GoSyntax(inst, encoding) + op := map[int]string{16: "MOVW", 32: "MOVL", 64: "MOVLQSX"}[bits] + want := fmt.Sprintf("%s %s, %s", op, registers[source], registers[result]) + if err != nil || got != want || !reflect.DeepEqual(inst, before) { + t.Fatalf("bits=%d bytes=%x: got %q want %q changed=%t: %v", bits, encoding, got, want, inst != before, err) + } + } + } + } +} + +func x86MOVSXDDenied(t *testing.T, source string) { + t.Helper() + requireX86GoAssemblerResult(t, "amd64", source, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, source) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved raw MOVSXD Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved raw MOVSXD module: %v", err) + } + if err := x86CallerCoreTranslate(t, pkg, source, triple); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved raw MOVSXD Go module: %v", err) + } + } +} + +func TestX86RawMOVSXDInputAndVendorBoundaries(t *testing.T) { + for _, test := range []struct{ name, body string }{ + {"undefined long source", "WORD $0xc363"}, + {"partial long source", "MOVW $7,BX\nWORD $0xc363"}, + {"undefined word source", "BYTE $0x66\nWORD $0xc363"}, + {"partial word source", "MOVB $7,BX\nBYTE $0x66\nWORD $0xc363"}, + {"caller pointer long source", "WORD $0xc763"}, + {"caller pointer word source", "BYTE $0x66\nWORD $0xc763"}, + {"word overwrites surviving pointer bits", "MOVQ DI,AX\nBYTE $0x66\nWORD $0xc363"}, + {"native SP source", "WORD $0xc463"}, + {"native BP source", "BYTE $0x66\nWORD $0xc563"}, + {"native SP result", "WORD $0xe363"}, + {"native BP result", "BYTE $0x66\nWORD $0xeb63"}, + {"g source", "BYTE $0x41\nWORD $0xc663"}, + {"g result", "BYTE $0x44\nWORD $0xf363"}, + {"unknown memory", "WORD $0x0363"}, + {"partial FP pointer slot", "WORD $0x4463\nWORD $0x0824"}, + {"return PC memory", "WORD $0x0463\nBYTE $0x24"}, + {"word memory vendor identity absent", "BYTE $0x66\nWORD $0x0763"}, + {"word indexed vendor identity absent", "BYTE $0x66\nLONG $0x038f4463"}, + {"address-size memory", "BYTE $0x67\nWORD $0x0763"}, + {"different initialized bytes join", "CMPQ AX,$0\nJE other\nMOVL $7,BX\nother:\nWORD $0xc363"}, + } { + t.Run(test.name, func(t *testing.T) { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", test.body+"\nMOVL AX,(DX)", 1) + x86MOVSXDDenied(t, assembly) + }) + } +} + +func TestX86RawMOVSXDRequiresExactEncodingIdentity(t *testing.T) { + encoding := []byte{0x63, 0xc3} + physical, err := x86asm.Decode(encoding, 64) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"source", "result", "width", "mode", "extra operand", "bytes", "truncated", "extended"} { + t.Run(change, func(t *testing.T) { + inst, bytes := physical, append([]byte(nil), encoding...) + switch change { + case "source": + inst.Args[1] = x86asm.ECX + case "result": + inst.Args[0] = x86asm.ECX + case "width": + inst.DataSize = 64 + case "mode": + inst.Mode = 32 + case "extra operand": + inst.Args[2] = x86asm.EAX + case "bytes": + bytes[1] = 0xc1 + case "truncated": + bytes = bytes[:1] + case "extended": + bytes = append(bytes, 0x90) + } + if _, err := decodedX86GoSyntax(inst, bytes); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("mismatched native MOVSXD width acquired permission: %v", err) + } + }) + } +} + +func TestX86RawMOVSXDNoncallerDoesNotAcquireInputProof(t *testing.T) { + for _, prefix := range []string{"", "MOVW $7,BX\n", "MOVQ x+0(FP),BX\n"} { + for _, raw := range []string{"WORD $0xc363", "BYTE $0x66\nWORD $0xc363"} { + source, sig, pkg := x86SourceStackNumericalFixture("amd64", prefix+raw+"\nMOVQ AX,ret+8(FP)\nBYTE $0xc3") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig) + if _, err := Translate(file, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller raw scalar width acquired input proof: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller raw scalar module acquired input proof: %v", err) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + }) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("noncaller raw scalar Go module acquired input proof: %v", err) + } + } + } +} + +func TestX86RawMOVSXDLegacyModeRemainsARPL(t *testing.T) { + for _, encoding := range [][]byte{{0x63, 0xc3}, {0x66, 0x63, 0xc3}} { + inst, err := x86asm.Decode(encoding, 32) + if err != nil || inst.Op != x86asm.ARPL { + t.Fatalf("legacy mode opcode identity: %+v %v", inst, err) + } + syntax, err := decodedX86GoSyntax(inst, encoding) + if err != nil || !strings.HasPrefix(syntax, "ARPL ") { + t.Fatalf("legacy ARPL must not become a numerical copy: %q %v", syntax, err) + } + } +} + +func TestX86RawMOVSXDUnregisteredPrefixesStayContext(t *testing.T) { + for _, encoding := range [][]byte{ + {0x66, 0x66, 0x63, 0xc3}, {0x40, 0x41, 0x63, 0xc3}, + {0xf2, 0x63, 0xc3}, {0xf3, 0x63, 0xc3}, + {0x67, 0x63, 0x07}, {0x64, 0x63, 0x07}, {0x65, 0x48, 0x63, 0x07}, + } { + inst, err := x86asm.Decode(encoding, 64) + if err != nil || inst.Op != x86asm.MOVSXD { + continue // Decoder rejection is also closed; it is not object/runtime P. + } + if _, err := decodedX86GoSyntax(inst, encoding); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unregistered physical prefix %x was silently discarded: %v", encoding, err) + } + } + if inst, err := x86asm.Decode([]byte{0xf0, 0x63, 0xc3}, 64); err == nil && inst.Op == x86asm.MOVSXD { + if _, err := decodedX86GoSyntax(inst, []byte{0xf0, 0x63, 0xc3}); err == nil || errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("machine-invalid LOCK MOVSXD was accepted or misclassified as missing context", err) + } + } +} + +func x86MOVSXDModuleObject(t *testing.T, file *File, options Options, triple, name string) { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, options) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, name+".ll", name+".o", module.String()) +} + +func x86MOVSXDGoModuleObject(t *testing.T, pkg GoPackage, source, triple, name string) { + t.Helper() + ctx := llvm.NewContext() + defer ctx.Dispose() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: triple, + }) + if translation != nil { + defer translation.Module.Dispose() + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, name+".ll", name+".o", translation.Module.String()) +} + +func TestX86RawMOVSXDSafeFormatsThreeAPIObjects(t *testing.T) { + for _, body := range []string{ + "MOVL $0x80000000,BX\nWORD $0xc363", + "MOVL $0x80000000,R9\nBYTE $0x41\nWORD $0xc163", + "MOVL $0x80000000,BX\nBYTE $0x44\nWORD $0xc363\nMOVL R8,AX", + "MOVL $0x80000000,AX\nWORD $0xc063", + "WORD $0x0763", "WORD $0x4763\nBYTE $1", "MOVL $0,CX\nLONG $0x038f4463", + "MOVL $0,CX\nLONG $0x008f4463", + "MOVQ $0x1122334455667788,AX\nMOVW $0x8000,BX\nBYTE $0x66\nWORD $0xc363", + "MOVQ $0x1122334455667788,AX\nMOVQ $0x8000,SI\nBYTE $0x66\nWORD $0xc663", + "MOVQ $0x1122334455667788,AX\nMOVW $0x8000,R9\nWORD $0x4166\nWORD $0xc163", + "MOVL $0x80000000,BX\nBYTE $0x48\nWORD $0xc363", + "BYTE $0x48\nWORD $0x0763", "WORD $0x4866\nWORD $0x0763", + } { + assembly := strings.Replace(x86CallerFixtureAssembly, "MOVL AX,(DX)", body+"\nMOVL AX,(DX)", 1) + requireX86GoAssemblerResult(t, "amd64", assembly, true) + pkg, file := x86CallerCoreFixture(t, x86CallerFixtureSource, assembly) + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "caller.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, "amd64", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, ResolveSym: resolve} + ir, err := Translate(file, options) + if err != nil { + t.Fatalf("Translate %s: %v\n%s", triple, err, body) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + compileLLVMToObject(t, llc, triple, "movsxd-text.ll", "movsxd-text.o", ir) + x86MOVSXDModuleObject(t, file, options, triple, "movsxd-module") + if err := x86CallerCoreTranslate(t, pkg, assembly, triple); err != nil { + t.Fatal(err) + } + } + } +} + +func TestX86RawMOVSXDLiteralExactReadExtent(t *testing.T) { + for _, prefix := range []string{"", "BYTE $0x48\n"} { + // The literal lies after the native C3, twelve/thirteen bytes from the + // entry. Displacement six spans the encoded full result-slot store. + body := prefix + "WORD $0x0563\nLONG $6\nWORD $0x8948\nWORD $0x2444\nBYTE $16\nBYTE $0xc3\nLONG $0x80000000" + source, sig, pkg := x86SourceStackNumericalFixture("amd64", body) + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + options := x86SourceStackOptions("amd64", triple, sig) + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, findLLVM22Tool("llc"), triple, "movsxd-literal.ll", "movsxd-literal.o", ir) + x86MOVSXDModuleObject(t, file, options, triple, "movsxd-literal-module") + x86MOVSXDGoModuleObject(t, pkg, source, triple, "movsxd-literal-go") + } + // Preserve the real opcode and displacement; only truncate the object. + truncated := strings.TrimSuffix(source, "LONG $0x80000000\n") + "WORD $0x8000\n" + truncatedFile, err := Parse(ArchAMD64, truncated) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(truncatedFile, x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig)); err == nil { + t.Fatal("four-byte MOVSXD literal read borrowed a truncated object") + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, truncatedFile, x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig)) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatal("module borrowed a truncated MOVSXD literal") + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(truncated), GoModuleOptions{ + GOARCH: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + }) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if err == nil { + t.Fatal("Go module borrowed a truncated MOVSXD literal") + } + } +} + +func TestX86RawMOVSXDOriginalGoByteWitness(t *testing.T) { + for _, encoding := range [][]byte{ + {0x63, 0xc3}, {0x66, 0x63, 0xc3}, {0x48, 0x63, 0xc3}, + {0x41, 0x63, 0xc1}, {0x44, 0x63, 0xc3}, + {0x66, 0x41, 0x63, 0xc1}, {0x66, 0x44, 0x63, 0xc3}, + {0x63, 0x07}, {0x63, 0x47, 0x01}, {0x63, 0x44, 0x8f, 0x03}, + {0x48, 0x63, 0x07}, {0x66, 0x48, 0x63, 0x07}, + {0x66, 0x63, 0x07}, // Go accepts the raw bytes; native extent remains Context. + } { + listing, err := x86ExtensionGoListing(t, "amd64", x86MOVSXDBytes(encoding)) + var bytes strings.Builder + for _, value := range append(append([]byte(nil), encoding...), 0xc3) { + fmt.Fprintf(&bytes, "%02x ", value) + } + if err != nil || !strings.Contains(listing, strings.TrimSpace(bytes.String())) { + t.Fatalf("actual Go %x byte witness: %v\n%s", encoding, err, listing) + } + } +} + +func TestX86RawMOVSXDLiteralCannotBorrowStaleCertificate(t *testing.T) { + body := "BYTE $0x48\nWORD $0x0563\nLONG $6\nWORD $0x8948\nWORD $0x2444\nBYTE $16\nBYTE $0xc3\nLONG $0x80000000" + source, sig, _ := x86SourceStackNumericalFixture("amd64", body) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + options := x86SourceStackOptions("amd64", "x86_64-unknown-linux-gnu", sig) + normalized, err := normalizeX86RawFile(file, "amd64", options) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"source width", "opcode", "result", "source", "payload", "missing object", "g result"} { + t.Run(change, func(t *testing.T) { + mutated := *normalized + mutated.Data = append([]DataStmt(nil), normalized.Data...) + mutated.Funcs = append([]Func(nil), normalized.Funcs...) + mutated.Funcs[0].Instrs = append([]Instr(nil), normalized.Funcs[0].Instrs...) + instruction := &mutated.Funcs[0].Instrs[1] + instruction.Args = append([]Operand(nil), instruction.Args...) + switch change { + case "source width": + instruction.x86RIPMemoryRead.width = 8 + case "opcode": + instruction.Op = "MOVQ" + case "result": + instruction.Args[1].Reg = BX + case "source": + instruction.Args[0] = Operand{Kind: OpReg, Reg: BX} + case "g result": + instruction.Args[1].Reg = "R14" + case "payload": + mutated.Data[0].Payload = []byte{1, 2, 3, 4} + case "missing object": + mutated.Data = nil + } + if _, err := Translate(&mutated, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed literal Translate: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, &mutated, options) + if !module.IsNil() { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed literal module: %v", err) + } + // Go-module input is original bytes, not a File carrying a private + // certificate. Its independently regenerated, truncated source is + // tested above; it cannot accept this mutated certificate as input. + }) + } +} diff --git a/amd64_raw_mul_dword_qword.go b/amd64_raw_mul_dword_qword.go new file mode 100644 index 000000000..4d7f82dfe --- /dev/null +++ b/amd64_raw_mul_dword_qword.go @@ -0,0 +1,243 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86DwordToQwordMultiplyInstruction recognizes every VEX and EVEX +// encoding of VPMULDQ and VPMULUDQ in Go 1.27's _yvandnpd table. x/arch +// v0.14 does not decode these encodings, so raw BYTE/LONG directive users +// need the complete X/Y/Z, memory, mask, zeroing, and broadcast family here. +func decodedX86DwordToQwordMultiplyInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if i >= len(code) { + return Instr{}, 0, false, nil + } + + var ( + opcodeIndex int + modRMIndex int + mapSelect int + width int + rExt int + rHigh int + xExt int + bExt int + vRegister int + vHigh int + rmHigh int + masked bool + zeroing bool + broadcast bool + mask int + evex bool + ) + + switch code[i] { + case 0xc5: + if len(code) < i+3 { + return Instr{}, 0, false, nil + } + vexByte := code[i+1] + opcodeIndex, modRMIndex = i+2, i+3 + mapSelect = 1 + if vexByte&3 != 1 || code[opcodeIndex] != 0xf4 { + return Instr{}, 0, false, nil + } + rExt = int(^vexByte>>7) & 1 + vRegister = int(^vexByte>>3) & 15 + width = 16 + if vexByte&4 != 0 { + width = 32 + } + case 0xc4: + if len(code) < i+4 { + return Instr{}, 0, false, nil + } + vexExt, vexByte := code[i+1], code[i+2] + opcodeIndex, modRMIndex = i+3, i+4 + mapSelect = int(vexExt & 0x1f) + if vexByte&3 != 1 || !x86DwordToQwordMultiplyOpcode(mapSelect, code[opcodeIndex]) { + return Instr{}, 0, false, nil + } + ok = true + if vexByte&0x80 != 0 { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + rExt = int(^vexExt>>7) & 1 + xExt = int(^vexExt>>6) & 1 + bExt = int(^vexExt>>5) & 1 + vRegister = int(^vexByte>>3) & 15 + width = 16 + if vexByte&4 != 0 { + width = 32 + } + case 0x62: + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcodeIndex, modRMIndex = i+4, i+5 + mapSelect = int(p0 & 3) + if p0&0x0c != 0 || p1&3 != 1 || !x86DwordToQwordMultiplyOpcode(mapSelect, code[opcodeIndex]) { + return Instr{}, 0, false, nil + } + ok, evex = true, true + if p1&0x84 != 0x84 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W must be one and reserved bit must be set") + } + rExt = int(^p0>>7) & 1 + rHigh = int(^p0>>4) & 1 + xExt = int(^p0>>6) & 1 + bExt = int(^p0>>5) & 1 + vRegister = int(^p1>>3) & 15 + vHigh = int(^p2>>3) & 1 + rmHigh = xExt + zeroing = p2&0x80 != 0 + broadcast = p2&0x10 != 0 + mask = int(p2 & 7) + masked = mask != 0 + switch p2 >> 5 & 3 { + case 0: + width = 16 + case 1: + width = 32 + case 2: + width = 64 + default: + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + default: + return Instr{}, 0, false, nil + } + + if !ok { + ok = true + } + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRM := code[modRMIndex] + if evex && broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + if zeroing && !masked { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + destinationNumber := int(modRM>>3&7) + rExt*8 + rHigh*16 + secondNumber := vRegister + vHigh*16 + if mode == 32 && (destinationNumber >= 8 || secondNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector or address register in 32-bit mode") + } + destination, valid := decodedX86VectorRegister(destinationNumber, width) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination vector register %d", destinationNumber) + } + second, valid := decodedX86VectorRegister(secondNumber, width) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid second-source vector register %d", secondNumber) + } + disp8Scale := 1 + if evex { + disp8Scale = width + if broadcast { + disp8Scale = 8 + } + } + first, consumed, err := decodedX86VectorRMSource(code[modRMIndex:], mode, bExt, xExt, rmHigh, width, segment, disp8Scale) + if err != nil { + return Instr{}, 0, true, err + } + + op := "VPMULUDQ" + if mapSelect == 2 { + op = "VPMULDQ" + } + var suffixes []string + if broadcast { + suffixes = append(suffixes, "BCST") + } + if zeroing { + suffixes = append(suffixes, "Z") + } + if len(suffixes) != 0 { + op += "." + strings.Join(suffixes, ".") + } + args := []Operand{first, {Kind: OpReg, Reg: second}} + if masked { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: destination}) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: Op(op), Args: args, Raw: op + " " + strings.Join(rawArgs, ", "), x86Encoded: true}, modRMIndex + consumed, true, nil +} + +func x86DwordToQwordMultiplyOpcode(mapSelect int, opcode byte) bool { + return mapSelect == 1 && opcode == 0xf4 || mapSelect == 2 && opcode == 0x28 +} + +func decodedX86VectorRMSource(code []byte, mode, bExt, xExt, highExt, width int, segment Reg, disp8Scale int) (Operand, int, error) { + if len(code) == 0 { + return Operand{}, 0, fmt.Errorf("missing ModRM byte") + } + if code[0]>>6 != 3 { + return decodedX86RMSource(code, mode, bExt, xExt, segment, disp8Scale) + } + number := int(code[0]&7) + bExt*8 + highExt*16 + if mode == 32 && number >= 8 { + return Operand{}, 0, fmt.Errorf("extended vector register in 32-bit mode") + } + register, ok := decodedX86VectorRegister(number, width) + if !ok { + return Operand{}, 0, fmt.Errorf("invalid first-source vector register %d", number) + } + return Operand{Kind: OpReg, Reg: register}, 1, nil +} + +func decodedX86VectorRegister(number, width int) (Reg, bool) { + prefix := "" + switch width { + case 16: + prefix = "X" + case 32: + prefix = "Y" + case 64: + prefix = "Z" + default: + return "", false + } + if number < 0 || number >= 32 { + return "", false + } + return Reg(fmt.Sprintf("%s%d", prefix, number)), true +} diff --git a/amd64_raw_mul_dword_qword_test.go b/amd64_raw_mul_dword_qword_test.go new file mode 100644 index 000000000..cb997191a --- /dev/null +++ b/amd64_raw_mul_dword_qword_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestX86RawDwordToQwordMultiplyReportedEncoding(t *testing.T) { + const source = ` +TEXT rawVPMULUDQ(SB),$0-0 + BYTE $0xc5 + BYTE $0xfd + BYTE $0xf4 + BYTE $0xc2 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "386", triple: "i686-unknown-linux-gnu"}, + {goarch: "386", triple: "i686-pc-windows-msvc"}, + {goarch: "amd64", triple: "x86_64-apple-darwin"}, + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawVPMULUDQ": {Name: "rawVPMULUDQ", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-vpmuludq.ll", "raw-vpmuludq.o", ir) + }) + } +} + +func TestX86RawDwordToQwordMultiplyCompleteVEXRegisterFields(t *testing.T) { + for _, family := range []struct { + name string + mapID byte + opcode byte + }{ + {name: "VPMULUDQ", mapID: 1, opcode: 0xf4}, + {name: "VPMULDQ", mapID: 2, opcode: 0x28}, + } { + for _, width := range []struct { + prefix string + l byte + }{ + {prefix: "X", l: 0}, + {prefix: "Y", l: 1}, + } { + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + p0 := byte((^destination>>3)&1)<<7 | 1<<6 | byte((^first>>3)&1)<<5 | family.mapID + p1 := byte((^second)&15)<<3 | width.l<<2 | 1 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0xc4, p0, p1, family.opcode, modRM} + instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("%s %s registers %d,%d,%d: ok=%v length=%d err=%v", family.name, width.prefix, first, second, destination, ok, length, err) + } + want := []Reg{ + Reg(fmt.Sprintf("%s%d", width.prefix, first)), + Reg(fmt.Sprintf("%s%d", width.prefix, second)), + Reg(fmt.Sprintf("%s%d", width.prefix, destination)), + } + if instruction.Op != Op(family.name) || len(instruction.Args) != len(want) { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + for index := range want { + if instruction.Args[index].Kind != OpReg || instruction.Args[index].Reg != want[index] { + t.Fatalf("encoding %x arg %d = %#v, want %s", encoding, index, instruction.Args[index], want[index]) + } + } + } + } + } + } + } +} + +func TestX86RawDwordToQwordMultiplyCompleteEVEXRegisterFields(t *testing.T) { + for _, family := range []struct { + name string + mapID byte + opcode byte + }{ + {name: "VPMULUDQ", mapID: 1, opcode: 0xf4}, + {name: "VPMULDQ", mapID: 2, opcode: 0x28}, + } { + for _, width := range []struct { + prefix string + ll byte + }{ + {prefix: "X", ll: 0}, + {prefix: "Y", ll: 1}, + {prefix: "Z", ll: 2}, + } { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + p0 := byte((^destination>>3)&1)<<7 | byte((^first>>4)&1)<<6 | byte((^first>>3)&1)<<5 | byte((^destination>>4)&1)<<4 | family.mapID + p1 := byte(0x85 | ((^second)&15)<<3) + p2 := width.ll<<5 | byte((^second>>4)&1)<<3 | 1 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, family.opcode, modRM} + instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("%s %s registers %d,%d,%d: ok=%v length=%d err=%v", family.name, width.prefix, first, second, destination, ok, length, err) + } + want := []Reg{ + Reg(fmt.Sprintf("%s%d", width.prefix, first)), + Reg(fmt.Sprintf("%s%d", width.prefix, second)), + "K1", + Reg(fmt.Sprintf("%s%d", width.prefix, destination)), + } + if instruction.Op != Op(family.name) || len(instruction.Args) != len(want) { + t.Fatalf("encoding %x decoded as %#v", encoding, instruction) + } + for index := range want { + if instruction.Args[index].Kind != OpReg || instruction.Args[index].Reg != want[index] { + t.Fatalf("encoding %x arg %d = %#v, want %s", encoding, index, instruction.Args[index], want[index]) + } + } + } + } + } + } + } +} + +func TestX86RawDwordToQwordMultiplyMemoryMaskAndBroadcastForms(t *testing.T) { + for _, test := range []struct { + name string + encoding []byte + wantOp Op + wantArgs int + wantMem bool + }{ + {name: "vex signed base", encoding: []byte{0xc4, 0xe2, 0x31, 0x28, 0x13}, wantOp: "VPMULDQ", wantArgs: 3, wantMem: true}, + {name: "vex unsigned extended base", encoding: []byte{0xc4, 0xc1, 0x31, 0xf4, 0x13}, wantOp: "VPMULUDQ", wantArgs: 3, wantMem: true}, + {name: "evex masked memory", encoding: []byte{0x62, 0x71, 0xfd, 0x0e, 0xf4, 0xbc, 0x75, 0xef, 0xff, 0xff, 0xff}, wantOp: "VPMULUDQ", wantArgs: 4, wantMem: true}, + {name: "evex broadcast", encoding: []byte{0x62, 0x71, 0xfd, 0x1e, 0xf4, 0xbc, 0x75, 0xef, 0xff, 0xff, 0xff}, wantOp: "VPMULUDQ.BCST", wantArgs: 4, wantMem: true}, + {name: "evex broadcast zero", encoding: []byte{0x62, 0x71, 0xfd, 0x9e, 0xf4, 0xbc, 0x75, 0xef, 0xff, 0xff, 0xff}, wantOp: "VPMULUDQ.BCST.Z", wantArgs: 4, wantMem: true}, + {name: "evex unmasked", encoding: []byte{0x62, 0x31, 0xfd, 0x08, 0xf4, 0xf8}, wantOp: "VPMULUDQ", wantArgs: 3}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86DwordToQwordMultiplyInstruction(test.encoding, 64) + if !ok || err != nil || length != len(test.encoding) { + t.Fatalf("encoding %x: ok=%v length=%d err=%v", test.encoding, ok, length, err) + } + if instruction.Op != test.wantOp || len(instruction.Args) != test.wantArgs || (instruction.Args[0].Kind == OpMem) != test.wantMem { + t.Fatalf("encoding %x decoded as %#v", test.encoding, instruction) + } + }) + } +} + +func TestX86RawDwordToQwordMultiplyRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + }{ + {name: "vex w", mode: 64, encoding: []byte{0xc4, 0xe1, 0xfd, 0xf4, 0xc2}}, + {name: "evex w", mode: 64, encoding: []byte{0x62, 0x31, 0x7d, 0x0e, 0xf4, 0xf8}}, + {name: "evex reserved length", mode: 64, encoding: []byte{0x62, 0x31, 0xfd, 0x6e, 0xf4, 0xf8}}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0x31, 0xfd, 0x88, 0xf4, 0xf8}}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0x31, 0xfd, 0x1e, 0xf4, 0xf8}}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc5, 0xfd, 0xf4, 0xc2}}, + {name: "rip relative", mode: 64, encoding: []byte{0xc5, 0xfd, 0xf4, 0x05, 0, 0, 0, 0}}, + {name: "extended 386", mode: 32, encoding: []byte{0xc4, 0x61, 0x79, 0xf4, 0xc0}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86DwordToQwordMultiplyInstruction(test.encoding, test.mode) + if !ok || err == nil { + t.Fatalf("encoding %x: ok=%v err=%v, want recognized rejection", test.encoding, ok, err) + } + }) + } +} diff --git a/amd64_raw_mul_low.go b/amd64_raw_mul_low.go new file mode 100644 index 000000000..b19a3982a --- /dev/null +++ b/amd64_raw_mul_low.go @@ -0,0 +1,190 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's packed-multiply rows share _yvandnpd's operand grammar. The +// opcode/map selects the lane operation; EVEX.W selects Q for opcode 40/0F38. +// VEX.W is ignored, including VPMULLD; it cannot select a VEX qword form. +// The word operations have no broadcast row, whereas D/Q permit it. +type x86PackedMultiplyForm struct { + mapID byte + opcode byte + vexOp Op + evexW0Op Op + evexW1Op Op + broadcastByte int +} + +var x86PackedMultiplyForms = [...]x86PackedMultiplyForm{ + {mapID: 1, opcode: 0xd5, vexOp: "VPMULLW", evexW0Op: "VPMULLW", evexW1Op: "VPMULLW"}, + {mapID: 1, opcode: 0xe5, vexOp: "VPMULHW", evexW0Op: "VPMULHW", evexW1Op: "VPMULHW"}, + {mapID: 1, opcode: 0xe4, vexOp: "VPMULHUW", evexW0Op: "VPMULHUW", evexW1Op: "VPMULHUW"}, + {mapID: 2, opcode: 0x0b, vexOp: "VPMULHRSW", evexW0Op: "VPMULHRSW", evexW1Op: "VPMULHRSW"}, + {mapID: 2, opcode: 0x40, vexOp: "VPMULLD", evexW0Op: "VPMULLD", evexW1Op: "VPMULLQ", broadcastByte: 4}, +} + +func x86PackedMultiplyFormFor(mapID, opcode byte) (x86PackedMultiplyForm, bool) { + for _, form := range x86PackedMultiplyForms { + if form.mapID == mapID && form.opcode == opcode { + return form, true + } + } + return x86PackedMultiplyForm{}, false +} + +// Keep the direct decoder used by the exhaustive VPMULLD/Q tests separate +// from the dispatcher, while sharing its parser with the word-multiply rows. +func decodedX86PackedMultiplyLowInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + instruction, length, ok, err = decodedX86PackedMultiplyInstruction(code, mode) + if !ok || err != nil || instruction.Op == "VPMULLD" || strings.HasPrefix(string(instruction.Op), "VPMULLD.") || + instruction.Op == "VPMULLQ" || strings.HasPrefix(string(instruction.Op), "VPMULLQ.") { + return instruction, length, ok, err + } + return Instr{}, 0, false, nil +} + +func decodedX86PackedMultiplyInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized { + return Instr{}, 0, false, nil + } + if p.evex { + return decodedX86EVEXPackedMultiplyInstruction( + code, p.modRM-5, mode, p.segment, p.addressOverride, + ) + } + if p.pp != 1 { + return Instr{}, 0, false, nil + } + form, recognized := x86PackedMultiplyFormFor(byte(p.mapNumber), byte(p.opcode)) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p.addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + modRMIndex := p.modRM + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorName := "X" + if p.vectorLength != 0 { + vectorName = "Y" + } + secondSourceNumber := p.upper + destinationNumber := int(code[modRMIndex]>>3&7) + p.r*8 + if mode == 32 && (secondSourceNumber >= 8 || destinationNumber >= 8 || + code[modRMIndex]>>6 == 3 && int(code[modRMIndex]&7)+p.b*8 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + firstSource, consumed, decodeErr := decodedX86VEXVectorRMOperand( + code[modRMIndex:], mode, p.b, p.x, p.segment, vectorName, + ) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + instruction = Instr{ + Op: form.vexOp, + Args: []Operand{firstSource, secondSource, destination}, + Raw: fmt.Sprintf("%s %s, %s, %s", form.vexOp, firstSource.String(), secondSource.String(), destination.String()), + } + instruction.x86Encoded = true + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXPackedMultiplyInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+5 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + if p1&0x07 != 5 { + return Instr{}, 0, false, nil + } + form, recognized := x86PackedMultiplyFormFor(p0&0x0f, code[i+4]) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + vectorBits := (p2 >> 5) & 3 + if vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth := [...]int{16, 32, 64}[vectorBits] + width64 := p1&0x80 != 0 + op := form.evexW0Op + if width64 { + op = form.evexW1Op + } + laneBytes := form.broadcastByte + if width64 && laneBytes != 0 { + laneBytes = 8 + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + modRMIndex := i + 5 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + modRM := code[modRMIndex] + broadcast := p2&0x10 != 0 + if broadcast && laneBytes == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast is not available for %s", op) + } + if broadcast && modRM>>6 == 3 { + return Instr{}, 0, true, fmt.Errorf("EVEX broadcast requires a memory source") + } + secondSourceNumber := (int(^p1>>3) & 15) + (int(^p2>>3)&1)*16 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + firstSourceNumber := int(modRM&7) + bExt*8 + xExt*16 + if mode == 32 && vectorName == "Z" && (secondSourceNumber >= 8 || destinationNumber >= 8 || modRM>>6 == 3 && firstSourceNumber >= 8) { + return Instr{}, 0, true, fmt.Errorf("high Z register is unavailable in 32-bit mode") + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = laneBytes + } + firstSource, consumed, decodeErr := decodedX86EVEXRMOperand(code[modRMIndex:], mode, bExt, xExt, segment, vectorName, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + secondSource := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, secondSourceNumber))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorName, destinationNumber))} + args := []Operand{firstSource, secondSource} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + if broadcast { + op += ".BCST" + } + if zeroing { + op += ".Z" + } + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), + x86Encoded: true}, modRMIndex + consumed, true, nil +} diff --git a/amd64_raw_mul_low_test.go b/amd64_raw_mul_low_test.go new file mode 100644 index 000000000..2a7a1df15 --- /dev/null +++ b/amd64_raw_mul_low_test.go @@ -0,0 +1,168 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedMultiplyLow(width256 bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 2) + p1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0xc4, p0, p1, 0x40, modRM} +} + +func encodeX86EVEXPackedMultiplyLow(width64 bool, vectorBits byte, mask int, zeroing bool, destination, source1, source2 int) []byte { + p0 := byte((1-destination/8&1)<<7 | (1-source2/16)<<6 | (1-source2/8&1)<<5 | (1-destination/16)<<4 | 2) + p1 := byte(((^source1)&15)<<3 | 5) + if width64 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | byte((^source1>>4)&1)<<3 | byte(mask)) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, 0x40, modRM} +} + +func TestTranslateRawEVEXVPMULLDWeaviateRegression(t *testing.T) { + const source = `TEXT rawVPMULLD(SB), $0-0 + LONG $0x483df262 + WORD $0xe440 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawVPMULLD": {Name: "rawVPMULLD", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <16 x i32>") { + t.Fatalf("raw VPMULLD lowering omitted packed multiply:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evex-vpmulld.ll", "amd64-raw-evex-vpmulld.o", ir) +} + +func TestDecodedX86PackedMultiplyLowCompleteVEXRegisterFamily(t *testing.T) { + count := 0 + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < 16; source2++ { + code := encodeX86VEXPackedMultiplyLow(width256, destination, source1, source2) + got, length, ok, err := decodedX86PackedMultiplyLowInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != "VPMULLD" || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want VPMULLD %s", code, got, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VEX packed multiply-low register encodings, want 8192", count) + } +} + +func TestDecodedX86PackedMultiplyLowCompleteEVEXRegisterFamily(t *testing.T) { + count := 0 + for _, width := range []struct { + width64 bool + op Op + }{ + {op: "VPMULLD"}, + {width64: true, op: "VPMULLQ"}, + } { + for vectorBits, prefix := range []string{"X", "Y", "Z"} { + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + code := encodeX86EVEXPackedMultiplyLow(width.width64, byte(vectorBits), 1, false, destination, source1, source2) + got, length, ok, err := decodedX86PackedMultiplyLowInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != width.op || len(got.Args) != 4 { + t.Fatalf("decode %x = %+v, want %s %s", code, got, width.op, strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, width.op, strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + } + if count != 196608 { + t.Fatalf("covered %d EVEX packed multiply-low register encodings, want 196608", count) + } +} + +func TestDecodedX86PackedMultiplyLowMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + width64 bool + vectorBits byte + op Op + laneBytes int + }{ + {name: "dword", vectorBits: 2, op: "VPMULLD.BCST.Z", laneBytes: 4}, + {name: "qword", width64: true, vectorBits: 2, op: "VPMULLQ.BCST.Z", laneBytes: 8}, + } { + t.Run(test.name, func(t *testing.T) { + code := encodeX86EVEXPackedMultiplyLow(test.width64, test.vectorBits, 3, true, 21, 20, 0) + code[3] |= 0x10 + code[5] = 0x68 + code = append(code, 0x7f) + got, length, ok, err := decodedX86PackedMultiplyLowInstruction(code, 64) + wantSource := fmt.Sprintf("%d(AX)", test.laneBytes*127) + if err != nil || !ok || length != len(code) || got.Op != test.op || len(got.Args) != 4 || got.Args[0].String() != wantSource || got.Args[1].String() != "Z20" || got.Args[2].String() != "K3" || got.Args[3].String() != "Z21" { + t.Fatalf("decode %x = %+v, length=%d ok=%v err=%v; want %s %s, Z20, K3, Z21", code, got, length, ok, err, test.op, wantSource) + } + }) + } + + for name, code := range map[string][]byte{ + "truncated": {0xc4, 0xe2}, + "wrong map": {0xc4, 0xe1, 0x79, 0x40, 0xc0}, + "wrong pp": {0xc4, 0xe2, 0x78, 0x40, 0xc0}, + "wrong opcode": {0xc4, 0xe2, 0x79, 0x41, 0xc0}, + } { + if instruction, _, matched, decodeErr := decodedX86PackedMultiplyLowInstruction(code, 64); matched || decodeErr != nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, code, instruction, matched, decodeErr) + } + } + invalidBroadcast := encodeX86EVEXPackedMultiplyLow(false, 2, 1, false, 0, 0, 0) + invalidBroadcast[3] |= 0x10 + if _, _, matched, decodeErr := decodedX86PackedMultiplyLowInstruction(invalidBroadcast, 64); !matched || decodeErr == nil { + t.Fatalf("register broadcast returned ok=%v err=%v", matched, decodeErr) + } + zeroWithoutMask := encodeX86EVEXPackedMultiplyLow(false, 2, 0, true, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86PackedMultiplyLowInstruction(zeroWithoutMask, 64); !matched || decodeErr == nil { + t.Fatalf("zero without mask returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_muldiv_width_test.go b/amd64_raw_muldiv_width_test.go new file mode 100644 index 000000000..939849c11 --- /dev/null +++ b/amd64_raw_muldiv_width_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawImplicitMulDivWidths(t *testing.T) { + for _, form := range []struct { + name string + regBit byte + }{ + {"MUL", 4}, {"IMUL", 5}, {"DIV", 6}, {"IDIV", 7}, + } { + for _, width := range []struct { + name string + prefix byte + opcode byte + }{ + {"B", 0, 0xf6}, + {"W", 0x66, 0xf7}, + {"L", 0, 0xf7}, + {"Q", 0x48, 0xf7}, + } { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s%s/mem%t", form.name, width.name, memory) + t.Run(name, func(t *testing.T) { + var code []byte + if width.prefix != 0 { + code = append(code, width.prefix) + } + modRM := byte(0xc2 | form.regBit<<3) + if memory { + modRM = 0x02 | form.regBit<<3 + } + code = append(code, width.opcode, modRM, 0xc3) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || string(got[0].Op) != form.name+width.name || got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s", code, got, name) + } + }) + } + } + } +} + +func TestTranslateX86RawImplicitMulDivWidthsObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, code := range [][]byte{ + {0xf6, 0xe2, 0xc3}, + {0x66, 0xf7, 0xea, 0xc3}, + {0xf7, 0xf2, 0xc3}, + {0x48, 0xf7, 0xfa, 0xc3}, + } { + name := fmt.Sprintf("rawMulDivWidth%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-muldiv-width.ll", "raw-muldiv-width.o", ir) + }) + } +} diff --git a/amd64_raw_mulword_test.go b/amd64_raw_mulword_test.go new file mode 100644 index 000000000..f60615409 --- /dev/null +++ b/amd64_raw_mulword_test.go @@ -0,0 +1,273 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86PackedWordMultiplyGoRows = []struct { + mapID byte + opcode byte + op Op +}{ + {1, 0xd5, "VPMULLW"}, + {1, 0xe5, "VPMULHW"}, + {1, 0xe4, "VPMULHUW"}, + {2, 0x0b, "VPMULHRSW"}, +} + +func encodeX86VEXPackedWordMultiply(mapID, opcode byte, width256, w bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | int(mapID)) + p1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if w { + p1 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0xc4, p0, p1, opcode, modRM} +} + +func encodeX86VEX2PackedWordMultiply(opcode byte, width256 bool, destination, source1, source2 int) []byte { + vectorBit := 0 + if width256 { + vectorBit = 1 + } + prefix := byte((1-destination/8)<<7 | (^source1&15)<<3 | vectorBit<<2 | 1) + modRM := byte(0xc0 | destination&7<<3 | source2&7) + return []byte{0xc5, prefix, opcode, modRM} +} + +func encodeX86EVEXPackedWordMultiply(mapID, opcode byte, vectorBits, mask int, w, zeroing bool, destination, source1, source2 int) []byte { + p0 := byte((1-destination/8&1)<<7 | (1-source2/16)<<6 | (1-source2/8&1)<<5 | (1-destination/16)<<4 | int(mapID)) + p1 := byte(((^source1)&15)<<3 | 5) + if w { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | ((^source1>>4)&1)<<3 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestTranslateRawVPMULHWOldCorpusRegression(t *testing.T) { + const source = `TEXT rawVPMULHW(SB), $0-0 + LONG $0xe57161c4 + BYTE $0xc6 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawVPMULHW": {Name: "rawVPMULHW", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <8 x i32>") || !strings.Contains(ir, "ashr <8 x i32>") { + t.Fatalf("raw VPMULHW omitted packed word multiply:\n%s", ir) + } +} + +func TestTranslateRawVEX2PackedWordMultiplyGoccRegression(t *testing.T) { + const source = `TEXT rawVEX2WordMultiply(SB), $0-0 + LONG $0xdbd5cdc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawVEX2WordMultiply": {Name: "rawVEX2WordMultiply", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <16 x i16>") { + t.Fatalf("raw VEX2 VPMULLW omitted packed multiplication:\n%s", ir) + } +} + +func TestDecodedX86PackedWordMultiplyCompleteGoRows(t *testing.T) { + if len(x86PackedMultiplyForms) != len(x86PackedWordMultiplyGoRows)+1 { + t.Fatalf("packed multiply grammar has %d rows, want four word rows plus D/Q", len(x86PackedMultiplyForms)) + } + for _, row := range x86PackedWordMultiplyGoRows { + form, ok := x86PackedMultiplyFormFor(row.mapID, row.opcode) + if !ok || form.vexOp != row.op || form.evexW0Op != row.op || form.evexW1Op != row.op || form.broadcastByte != 0 { + t.Fatalf("Go word-multiply row %s/%02x is not modeled completely: %+v", row.op, row.opcode, form) + } + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, w := range []bool{false, true} { + code := encodeX86VEXPackedWordMultiply(row.mapID, row.opcode, width256, w, 8, 1, 6) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s %s6, %s1, %s8", row.op, prefix, prefix, prefix) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("VEX %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + if row.mapID == 1 { + code := encodeX86VEX2PackedWordMultiply(row.opcode, width256, 8, 1, 6) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s %s6, %s1, %s8", row.op, prefix, prefix, prefix) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("VEX2 %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + } + for vectorBits, prefix := range []string{"X", "Y", "Z"} { + for _, w := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 2 + } + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, vectorBits, mask, w, masked, 21, 20, 22) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + wantOp := string(row.op) + wantArgs := fmt.Sprintf("%s22, %s20, %s21", prefix, prefix, prefix) + if masked { + wantOp += ".Z" + wantArgs = fmt.Sprintf("%s22, %s20, K2, %s21", prefix, prefix, prefix) + } + want := wantOp + " " + wantArgs + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("EVEX %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + } + } + } +} + +func TestDecodedX86PackedWordMultiplyMemoryAndRejectedForms(t *testing.T) { + for _, row := range x86PackedWordMultiplyGoRows { + if row.mapID == 1 { + code := encodeX86VEX2PackedWordMultiply(row.opcode, true, 2, 1, 0) + code[3] = 0x50 + code = append(code, 0x20) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s 32(AX), Y1, Y2", row.op) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("VEX2 memory %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, 2, 0, false, false, 21, 20, 0) + code[5] = 0x68 + code = append(code, 0x7f) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 64) + want := fmt.Sprintf("%s 8128(AX), Z20, Z21", row.op) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("memory %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + broadcast := append([]byte(nil), code...) + broadcast[3] |= 0x10 + if _, _, matched, err := decodedX86PackedMultiplyInstruction(broadcast, 64); !matched || err == nil { + t.Fatalf("word multiply incorrectly accepted EVEX broadcast %x", broadcast) + } + for vectorBits, prefix := range []string{"X", "Y"} { + code := encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, vectorBits, 0, false, false, 21, 20, 22) + got, length, matched, err := decodedX86PackedMultiplyInstruction(code, 32) + want := fmt.Sprintf("%s %s22, %s20, %s21", row.op, prefix, prefix, prefix) + if err != nil || !matched || length != len(code) || got.Raw != want { + t.Fatalf("386 high-register %x = %+v, length=%d matched=%v err=%v; want %s", code, got, length, matched, err, want) + } + } + } + invalidZeroing := encodeX86EVEXPackedWordMultiply(1, 0xe5, 0, 0, false, true, 0, 1, 2) + if _, _, matched, err := decodedX86PackedMultiplyInstruction(invalidZeroing, 64); !matched || err == nil { + t.Fatalf("zero without mask accepted: %x", invalidZeroing) + } + invalidLength := encodeX86EVEXPackedWordMultiply(1, 0xe5, 3, 0, false, false, 0, 1, 2) + if _, _, matched, err := decodedX86PackedMultiplyInstruction(invalidLength, 64); !matched || err == nil { + t.Fatalf("reserved vector length accepted: %x", invalidLength) + } + for _, code := range [][]byte{ + {0xc4, 0xe3, 0x79, 0xe5, 0xc0}, + {0xc4, 0xe1, 0x78, 0xe5, 0xc0}, + {0xc4, 0xe1, 0x79, 0xe6, 0xc0}, + } { + if got, _, matched, err := decodedX86PackedMultiplyInstruction(code, 64); matched || err != nil { + t.Fatalf("invalid opcode form %x decoded as %+v, matched=%v err=%v", code, got, matched, err) + } + } +} + +func TestTranslateRawPackedWordMultiplyLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedWordMultiply(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, row := range x86PackedWordMultiplyGoRows { + for _, width256 := range []bool{false, true} { + appendBytes(encodeX86VEXPackedWordMultiply(row.mapID, row.opcode, width256, false, 2, 1, 0)) + if row.mapID == 1 { + appendBytes(encodeX86VEX2PackedWordMultiply(row.opcode, width256, 2, 1, 0)) + } + } + for vectorBits := 0; vectorBits < 3; vectorBits++ { + mask := 3 + zeroing := true + if target.goarch == "386" { + mask = 0 + zeroing = false + } + destination, second, first := 21, 20, 22 + if target.goarch == "386" { + destination, second, first = 2, 1, 0 + } + appendBytes(encodeX86EVEXPackedWordMultiply(row.mapID, row.opcode, vectorBits, mask, false, zeroing, destination, second, first)) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawPackedWordMultiply": {Name: "rawPackedWordMultiply", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-word-multiply.ll", "raw-packed-word-multiply.o", ir) + }) + } +} diff --git a/amd64_raw_narrow_single_source.go b/amd64_raw_narrow_single_source.go new file mode 100644 index 000000000..e13caebc0 --- /dev/null +++ b/amd64_raw_narrow_single_source.go @@ -0,0 +1,81 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's _yvpmovdb/_yvpmovdw rows form a three-by-six grammar: +// unsigned-saturating, signed-saturating and truncating conversions, each +// narrowing W/D/Q into B/W/D. The low opcode nibble selects the width pair. +var x86RawSingleSourceNarrowOps = [3][6]Op{ + {"VPMOVUSWB", "VPMOVUSDB", "VPMOVUSQB", "VPMOVUSDW", "VPMOVUSQW", "VPMOVUSQD"}, + {"VPMOVSWB", "VPMOVSDB", "VPMOVSQB", "VPMOVSDW", "VPMOVSQW", "VPMOVSQD"}, + {"VPMOVWB", "VPMOVDB", "VPMOVQB", "VPMOVDW", "VPMOVQW", "VPMOVQD"}, +} + +func decodedX86SingleSourceNarrowInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || !p.evex || p.mapNumber != 2 || p.pp != 2 { + return Instr{}, 0, false, nil + } + group := p.opcode>>4 - 1 + width := p.opcode & 15 + if group < 0 || group >= len(x86RawSingleSourceNarrowOps) || width >= 6 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("single-source narrowing: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.w || p.upper != 0 || p.broadcast || p.vectorLength > 2 { + return fail("invalid reserved EVEX encoding bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + + baseOp := x86RawSingleSourceNarrowOps[group][width] + spec := amd64SingleSourceNarrowSpecs[string(baseOp)] + inputBytes := 16 << p.vectorLength + outputBytes := inputBytes * spec.outputBits / spec.inputBits + destinationPrefix := "X" + if outputBytes > 16 { + destinationPrefix = "Y" + } + modRM := code[p.modRM] + registerNumber := int(modRM>>3&7) + p.r*8 + source := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", [...]string{"X", "Y", "Z"}[p.vectorLength], registerNumber))} + if mode == 32 && p.vectorLength == 2 && registerNumber >= 8 { + return fail("386 Z source register exceeds Go's encoder class") + } + if mode == 32 && modRM>>6 != 3 && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + destination, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, destinationPrefix, outputBytes) + if err != nil { + return Instr{}, 0, true, err + } + op := baseOp + if p.zero { + op += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_narrow_single_source_test.go b/amd64_raw_narrow_single_source_test.go new file mode 100644 index 000000000..4f3e19abc --- /dev/null +++ b/amd64_raw_narrow_single_source_test.go @@ -0,0 +1,192 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawSingleSourceNarrow(group, width, vectorBits, source, destination, mask int, zeroing, memory bool) []byte { + if memory { + destination = 0 + } + p0 := byte((1-source/8&1)<<7 | (1-destination/16)<<6 | + (1-destination/8&1)<<5 | (1-source/16)<<4 | 2) + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (source&7)<<3 | destination&7) + if memory { + modRM = byte(0x40 | (source&7)<<3) + } + code := []byte{0x62, p0, 0x7e, p2, byte((group+1)<<4 | width), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestTranslateRawNarrowSingleSourceGorseRegression(t *testing.T) { + const source = `TEXT rawNarrowSingleSource(SB), $0-0 + LONG $0x487ef262 + WORD $0x2c11 + BYTE $0x07 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawNarrowSingleSource": {Name: "rawNarrowSingleSource", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "icmp") { + t.Fatalf("raw VPMOVUSDB omitted saturating conversion:\n%s", ir) + } +} + +func TestDecodedX86RawSingleSourceNarrowCompleteGoRows(t *testing.T) { + if len(amd64SingleSourceNarrowSpecs) != 18 { + t.Fatalf("Go 1.27 single-source narrowing grammar has %d members, want 18", len(amd64SingleSourceNarrowSpecs)) + } + inputBits := [...]int{16, 32, 64, 32, 64, 64} + outputBits := [...]int{8, 8, 8, 16, 16, 32} + seen := make(map[Op]bool) + for group, row := range x86RawSingleSourceNarrowOps { + for width, op := range row { + spec, ok := amd64SingleSourceNarrowSpecs[string(op)] + if !ok || seen[op] || spec.inputBits != inputBits[width] || spec.outputBits != outputBits[width] || spec.saturating != (group != 2) { + t.Fatalf("raw opcode row %d/%d = %s, spec %+v", group, width, op, spec) + } + seen[op] = true + for vectorBits, prefix := range [...]string{"X", "Y", "Z"} { + outputBytes := (16 << vectorBits) * spec.outputBits / spec.inputBits + outputPrefix := "X" + if outputBytes > 16 { + outputPrefix = "Y" + } + for _, memory := range []bool{false, true} { + for _, masked := range []bool{false, true} { + mask := 0 + if masked { + mask = 3 + } + for _, zeroing := range []bool{false, true} { + if zeroing && !masked { + continue + } + code := encodeX86RawSingleSourceNarrow(group, width, vectorBits, 21, 20, mask, zeroing, memory) + got, length, recognized, err := decodedX86SingleSourceNarrowInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := op + if zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[0].String() != fmt.Sprintf("%s21", prefix) { + t.Fatalf("decode %x = %+v, want %s and %s21", code, got, wantOp, prefix) + } + if masked && (len(got.Args) != 3 || got.Args[1].String() != "K3") || !masked && len(got.Args) != 2 { + t.Fatalf("decode %x = %+v, lost mask", code, got) + } + destination := got.Args[len(got.Args)-1] + if memory { + if destination.Kind != OpMem || destination.Mem.Base != AX || destination.Mem.Off != int64(outputBytes) { + t.Fatalf("decode %x = %+v, want compressed displacement %d", code, got, outputBytes) + } + } else if destination.String() != fmt.Sprintf("%s20", outputPrefix) { + t.Fatalf("decode %x = %+v, want %s20", code, got, outputPrefix) + } + } + } + } + } + } + } + if len(seen) != len(amd64SingleSourceNarrowSpecs) { + t.Fatalf("raw grammar has %d distinct ops, named lowerer has %d", len(seen), len(amd64SingleSourceNarrowSpecs)) + } +} + +func TestDecodedX86RawSingleSourceNarrowRejectsReservedBits(t *testing.T) { + base := encodeX86RawSingleSourceNarrow(0, 1, 2, 2, 1, 3, true, false) + for _, test := range []struct { + name string + code []byte + }{ + {"W bit", []byte{base[0], base[1], base[2] | 0x80, base[3], base[4], base[5]}}, + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"vvvv", []byte{base[0], base[1], base[2] &^ 8, base[3], base[4], base[5]}}, + {"V prime", []byte{base[0], base[1], base[2], base[3] &^ 8, base[4], base[5]}}, + {"broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"reserved length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"zero without mask", []byte{base[0], base[1], base[2], base[3] &^ 7, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86SingleSourceNarrowInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted reserved encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + if _, _, recognized, err := decodedX86PerLaneVariableShiftInstruction(base, 64); recognized || err != nil { + t.Fatalf("per-lane shift incorrectly owns F3 narrow opcode: recognized=%v err=%v", recognized, err) + } +} + +func TestTranslateRawSingleSourceNarrowLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawNarrowFamily(SB), $0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for group, row := range x86RawSingleSourceNarrowOps { + for width := range row { + appendBytes(encodeX86RawSingleSourceNarrow(group, width, 0, 2, 1, 0, false, false)) + } + appendBytes(encodeX86RawSingleSourceNarrow(group, 5, 2, 2, 1, 2, true, false)) + appendBytes(encodeX86RawSingleSourceNarrow(group, 1, 1, 2, 0, 3, false, true)) + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawNarrowFamily": {Name: "rawNarrowFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-single-source-narrow.ll", "raw-single-source-narrow.o", ir) + }) + } +} diff --git a/amd64_raw_near_return.go b/amd64_raw_near_return.go new file mode 100644 index 000000000..97afdb0dc --- /dev/null +++ b/amd64_raw_near_return.go @@ -0,0 +1,437 @@ +package plan9asm + +import ( + "bytes" + "encoding/binary" + "fmt" + "reflect" + "strings" +) + +type x86RawNearReturnForm struct { + nativeWidth bool + cleanup uint16 +} + +// x86RawStaticRead is produced only by a source-local RIP decoder after its +// complete encoding grammar and access width have been validated. It is not a +// mnemonic whitelist, a general pointer contract, or permission to store. +type x86RawStaticRead struct { + op Op + source int + width int64 + symbol string + offset int64 + object []byte + args []Operand +} + +// Only the normalizer's symbol relocation may change a proved operand. Keep +// an independent snapshot so a caller's edited File cannot authorize a wider +// vector access, a different mask, or an out-of-range proof index. +func rebindX86RawStaticRead(ins *Instr, source int, symbol string) error { + if source < 0 || source >= len(ins.Args) { + return fmt.Errorf("%w: invalid raw static read operand", ErrProbeNeedsContext) + } + read := ins.x86RIPMemoryRead + if read.width != 0 { + if read.source != source || read.op != ins.Op || !reflect.DeepEqual(read.args, ins.Args) { + return fmt.Errorf("%w: changed raw static read cannot inherit decoder proof", ErrProbeNeedsContext) + } + read.args = append([]Operand(nil), read.args...) + read.args[source].Sym = symbol + read.symbol = symbol + ins.x86RIPMemoryRead = read + } + ins.Args[source].Sym = symbol + return nil +} + +// C3 and C2 imm16 use the target's native return-PC width. Operand-size +// overrides and other prefixes require their own physical entry contract; +// successful decoding alone does not make them ordinary Go/C returns. +func x86RawNearReturnEncoding(code []byte) x86RawNearReturnForm { + if len(code) == 1 && code[0] == 0xc3 { + return x86RawNearReturnForm{nativeWidth: true} + } + if len(code) == 3 && code[0] == 0xc2 { + return x86RawNearReturnForm{nativeWidth: true, cleanup: binary.LittleEndian.Uint16(code[1:])} + } + return x86RawNearReturnForm{} +} + +func validateX86RawNearReturns(fn Func, file *File, opt Options, bindFrame bool) error { + hasReturn := false + for _, ins := range fn.Instrs { + form := ins.x86RawNearReturn + if form == nil { + continue + } + hasReturn = true + if !form.nativeWidth || form.cleanup != 0 { + return fmt.Errorf("%w: raw near RET needs a physical return-width and caller-stack cleanup contract: %q", ErrProbeNeedsContext, ins.Raw) + } + } + if !hasReturn { + return nil + } + // The raw instruction bypasses obj6's RET epilogue. A zero-sized, + // stack-unobserving leaf has the same return-PC location as its caller. + // All other cases need a native frame/continuation contract, not a guessed + // epilogue or an inline RET inside LLVM's independently allocated frame. + if fn.FrameSize != 0 || (!x86RawReturnLeafStackUnobserved(fn) && fn.x86SourceStack == nil) { + diagnostic := "" + if fn.x86SourceStackFailure != "" { + diagnostic = "; " + fn.x86SourceStackFailure + } + return fmt.Errorf("%w: raw near RET requires a proved source stack/continuation; Go's TEXT frame epilogue is not encoded in raw bytes%s", ErrProbeNeedsContext, diagnostic) + } + if !x86RawReturnMemoryBound(fn, file, opt, bindFrame) { + return fmt.Errorf("%w: raw near RET needs bounded typed FP/static memory; ordinary pointers have no native return-PC nonalias contract", ErrProbeNeedsContext) + } + return nil +} + +// This bounded contract does not infer noalias from a Go pointer type or a +// register value. It proves only scalar MOV accesses to bound FP slots or a +// same-file static object, and shared raw decoders' exact source-local reads. +// Other memory forms need their own complete effects and access-range proof. +// In particular, this is not a general store whitelist. +func x86RawReturnMemoryBound(fn Func, file *File, opt Options, bindFrame bool) bool { + name := fn.Sym + if opt.ResolveSym != nil { + name = opt.ResolveSym(name) + } + sig := opt.Sigs[name] + caller := bindFrame && x86CallerNormalizedSourceMatches(fn, sig) + if fn.x86SourceStack != nil && fn.x86SourceStack.caller != nil && !caller { + return false + } + for _, ins := range fn.Instrs { + if ins.x86RawMOVSXDNeedsCaller && !caller { + return false // A native width is not an initialized GP-input proof. + } + read := ins.x86RIPMemoryRead + if read.width != 0 && (opt.Goarch == "386" || !x86RawReturnDecodedReadBound(ins, file)) { + return false + } + op := Op(normalizeInstructionOpcode(ins.Op)) + if _, extension := amd64ScalarExtensionMoveSpecs[op]; extension && !caller && !ins.x86Encoded { + // A named extension gains new source-frame effects only through + // actual typed caller input availability. Keep the old separately + // bound physical/RIP decoder contract distinct from that producer. + return false + } + if _, _, _, memory := x86StringProperties(op); memory { + return false + } + if _, _, memory := x86PortStringProperties(op); memory { + return false + } + if _, memory := amd64ImplicitMaskMoveSpecs[op]; memory || op == "XLAT" { + return false + } + for index, arg := range ins.Args { + switch arg.Kind { + case OpMem: + if caller { + if !x86CallerMemoryCertified(ins, index) && op != "LEAW" && op != "LEAL" && op != "LEAQ" { + return false + } + // Caller pointers require their own exact access certificate, + // including physical-SP imports. The old numerical FP proof + // is not an alternative permission after that certificate is lost. + continue + } + if offset, proved := ins.x86SourceStackFP[index]; bindFrame && fn.x86SourceStack != nil && proved && + x86RawReturnFPBound(offset, x86ScalarMoveMemoryBytes(ins, index, opt.Goarch), sig, opt.Goarch) { + continue // Instruction-local, freshly recomputed physical-SP slot proof. + } + if !bindFrame && fn.x86SourceStack != nil && x86RawReturnStackMemory(arg.Mem) && x86ScalarMoveMemoryBytes(ins, index, opt.Goarch) != 0 { + continue // Pure preparation; final normalization binds the exact slot for lowering. + } + // Even a static symbol plus an index is not a bounded object. + // Address-only register expressions have no dereference, but + // their later uses still have to satisfy this same contract. + if op != "LEAW" && op != "LEAL" && op != "LEAQ" { + return false + } + case OpFP: + if caller { + if !x86CallerMemoryCertified(ins, index) { + return false + } + continue + } + if !bindFrame { + continue // Pure prepartition normalization retains metadata. + } + width := x86ScalarMoveMemoryBytes(ins, index, opt.Goarch) + if width == 0 || !x86RawReturnFPBound(arg.FPOffset, width, sig, opt.Goarch) { + return false + } + case OpSym: + if read.width != 0 && read.source == index { + continue // Exact source and allocated bytes were checked above. + } + symbol := strings.TrimSpace(arg.Sym) + if strings.HasPrefix(symbol, "$") || ins.Op == OpTEXT || ins.Op == OpLABEL { + continue + } + if op == "LEAW" || op == "LEAL" || op == "LEAQ" { + continue // Address formation, like OpMem LEA; not a read. + } + base, offset, static := parseSBRef(symbol) + if !static { + continue // Non-memory control operands retain their own checks. + } + if !x86RawReturnStaticBound(base, offset, x86ScalarMoveMemoryBytes(ins, index, opt.Goarch), file) { + return false + } + } + } + } + return true +} + +func x86RawReturnDecodedReadIdentity(ins Instr) bool { + read := ins.x86RIPMemoryRead + if !ins.x86Encoded || !ins.x86RIPLiteral || + read.op != ins.Op || read.source < 0 || read.source >= len(ins.Args) || + !reflect.DeepEqual(read.args, ins.Args) || + read.width <= 0 || read.offset < 0 || + read.width > int64(len(read.object)) || read.offset > int64(len(read.object))-read.width { + return false + } + operand := ins.Args[read.source] + if operand.Kind != OpSym || operand.Sym != read.symbol { + return false + } + if len(ins.x86RIPLiteralData) != 0 && + !bytes.Equal(ins.x86RIPLiteralData, read.object[read.offset:read.offset+read.width]) { + return false + } + return true +} + +func x86RawReturnDecodedReadBound(ins Instr, file *File) bool { + if file == nil || !x86RawReturnDecodedReadIdentity(ins) { + return false + } + read := ins.x86RIPMemoryRead + symbol, offset, ok := parseSBRef(ins.Args[read.source].Sym) + if !ok || offset != read.offset || !x86RawReturnStaticBound(symbol, offset, read.width, file) { + return false + } + // Re-normalization and bounded module partitioning retain the data. Do + // not use stale constant-specialization bytes after a caller changes it. + matched := false + for _, datum := range file.Data { + if datum.Sym != symbol { + continue + } + if matched || datum.Off != 0 || datum.Width != int64(len(read.object)) || datum.Addr != "" || + !bytes.Equal(datum.Payload, read.object) { + return false + } + matched = true + } + return matched +} + +func x86RawReturnScalarMoveBytes(op Op) int64 { + switch op { + case "MOVB": + return 1 + case "MOVW": + return 2 + case "MOVL": + return 4 + case "MOVQ": + return 8 + default: + return 0 + } +} + +func x86RawReturnFPBound(offset, width int64, sig FuncSig, goarch string) bool { + if offset < 0 || width <= 0 { + return false + } + const maxOffset = int64(1<<63 - 1) + var checked []FrameSlot + matched := false + for group, slots := range [][]FrameSlot{sig.Frame.Params, sig.Frame.Results} { + for _, slot := range slots { + size := x86RawReturnSlotBytes(slot.Type, goarch) + if !x86RawReturnSlotMatchesSignature(slot, group, sig) || size == 0 || + slot.Offset < 0 || slot.Offset > maxOffset-size { + return false + } + for _, previous := range checked { + previousSize := x86RawReturnSlotBytes(previous.Type, goarch) + if slot.Offset < previous.Offset+previousSize && previous.Offset < slot.Offset+size { + return false + } + } + checked = append(checked, slot) + // Exact whole-slot moves avoid relying on generic partial FP writes + // preserving the untouched high bytes. This does not repair those + // writes elsewhere in the translator. + matched = matched || slot.Offset == offset && width == size + } + } + return matched +} + +func x86RawReturnSlotMatchesSignature(slot FrameSlot, group int, sig FuncSig) bool { + if slot.Index < 0 { + return false + } + path := frameSlotFields(slot) + if group == 0 { + if slot.Index >= len(sig.Args) { + return false + } + typ := sig.Args[slot.Index] + if len(path) != 0 { + // Reuse the existing flat literal-aggregate grammar. Nested paths + // and arrays need a richer validated binding, not a new guessed + // layout parser inside the native-return proof. + fields, aggregate := parseLiteralStructFields(typ) + if !aggregate || len(path) != 1 || path[0] < 0 || path[0] >= len(fields) { + return false + } + typ = fields[path[0]] + } + return slot.Type == typ + } + // Classic result slots index the already flattened return tuple. Field + // paths are parameter extraction metadata, not a result-slot binding. + if sig.Ret == Void || sig.Ret == "" || len(path) != 0 { + return false + } + if fields, aggregate := parseLiteralStructFields(sig.Ret); aggregate { + return slot.Index < len(fields) && slot.Type == fields[slot.Index] + } + return slot.Index == 0 && slot.Type == sig.Ret +} + +func x86RawReturnSlotBytes(typ LLVMType, goarch string) int64 { + return x86FrameScalarBytes(typ, goarch) +} + +func x86RawReturnStaticBound(symbol string, offset, width int64, file *File) bool { + if file == nil || offset < 0 || width <= 0 { + return false + } + var size int64 + for _, object := range file.Globl { + if object.Sym != symbol { + continue + } + if object.SizeRaw != "" || object.Size < 0 || object.Size > maxDataGlobalSize { + return false + } + if object.Size > size { + size = object.Size + } + } + // Match the actual DATA allocation consumer. DATA without a GLOBL still + // creates a same-file byte object; holes are initialized to zero. Reuse + // its existing checked extent rather than guessing a fixed object size. + for _, datum := range file.Data { + if datum.Sym != symbol { + continue + } + end, err := dataStmtEnd(datum) + if err != nil { + return false + } + if end > size { + size = end + } + } + return size >= width && offset <= size-width +} + +func x86RawReturnLeafStackUnobserved(fn Func) bool { + hasText := false + for _, ins := range fn.Instrs { + if ins.Op == OpTEXT { + hasText = true + _, rest := splitOpcode(ins.Raw) + parts := strings.Split(rest, ",") + if len(parts) == 3 { + expression := globlFlagName.ReplaceAllStringFunc(parts[1], func(name string) string { + if value, ok := goTextFlagValues[name]; ok { + return value + } + return name + }) + flags, known := parseImmExpr(strings.TrimSpace(expression)) + if !known || flags & ^uint64(2|4|512) != 0 { + return false + } + } else if len(parts) != 2 { + return false + } + continue + } + if x86SourceGOperand(ins) { + // Stack-unobserving does not mean native-context-unobserving. + // Incoming g is opaque; numerical use or clobber needs the same + // complete caller and exact saved-identity proof as a framed leaf. + return false + } + op := strings.ToUpper(string(ins.Op)) + if strings.HasPrefix(op, "PUSH") || strings.HasPrefix(op, "POP") || + strings.HasPrefix(op, "IRET") || strings.HasPrefix(op, "RETF") { + return false + } + switch Op(op) { + case OpCALL, "ADJSP", "ENTER", "ENTERL", "ENTERQ", "ENTERW", "LEAVEL", "LEAVEQ", "LEAVEW": + return false + case OpRET: + if len(ins.Args) != 0 { + return false + } + case OpJMP: + if len(ins.Args) != 1 || (ins.Args[0].Kind != OpIdent && ins.Args[0].Kind != OpLabel) { + return false + } + } + for _, operand := range ins.Args { + if operand.Kind == OpFPAddr || (operand.Kind == OpFP && operand.FPOffset < 0) { + return false + } + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + return false + } + if operand.Kind == OpMem && x86RawReturnStackMemory(operand.Mem) { + return false + } + // Go's effective-address immediate is OpSym, not OpMem. Use the + // same parser and typed register roots as its actual lowering. + if operand.Kind == OpSym { + symbol := strings.TrimSpace(operand.Sym) + if strings.HasPrefix(symbol, "$") { + address, ok := parseMem(strings.TrimSpace(strings.TrimPrefix(symbol, "$"))) + if ok && x86RawReturnStackMemory(address) { + return false + } + } + } + } + } + return hasText +} + +func x86RawReturnStackRegister(reg Reg) bool { + root, known := amd64FullRegBase(reg) + return known && (root == SP || root == BP) +} + +func x86RawReturnStackMemory(memory MemRef) bool { + return x86RawReturnStackRegister(memory.Base) || x86RawReturnStackRegister(memory.Index) +} diff --git a/amd64_raw_near_return_runtime_test.go b/amd64_raw_near_return_runtime_test.go new file mode 100644 index 000000000..0676e27cb --- /dev/null +++ b/amd64_raw_near_return_runtime_test.go @@ -0,0 +1,146 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Observe C2's real caller-SP effect with a source-native continuation. The +// trampoline restores the original SP before returning to the Go runtime. +// It is an independent machine-code witness, not a translated C tail call. +func x86RawNearReturnGoStackOracle(t *testing.T, arch string, runPrefix []string) { + t.Helper() + move, sub, add, address := "MOVQ", "SUBQ", "ADDQ", "LEAQ" + wordBytes := 8 + if arch == "386" { + move, sub, add, address = "MOVL", "SUBL", "ADDL", "LEAL" + wordBytes = 4 + } + var assembly strings.Builder + for _, cleanup := range []int{0, 4} { + fmt.Fprintf(&assembly, `TEXT ·Measure%d(SB),516,$0-%d +%s SP,BX +%s $16,SP +%s resume%d<>(SB),AX +%s AX,0(SP) +JMP native%d<>(SB) +TEXT native%d<>(SB),516,$0-0 +BYTE $0xc2 +WORD $%d +TEXT resume%d<>(SB),516,$0-0 +%s SP,AX +%s BX,AX +%s $%d,AX +%s BX,SP +%s AX,ret+0(FP) +RET +`, cleanup, wordBytes, move, sub, address, cleanup, move, cleanup, cleanup, + cleanup, cleanup, move, sub, add, 16-wordBytes, move, move) + } + for _, encoding := range []struct{ name, code string }{ + {"C3", "BYTE $0xc3"}, {"C2", "BYTE $0xc2\nWORD $0"}, + } { + fmt.Fprintf(&assembly, `TEXT ·Result%s(SB),4,$0-4 +CALL value%s<>(SB) +MOVL AX,ret+0(FP) +RET +TEXT value%s<>(SB),4,$0-0 +MOVL $17,AX +%s +RET +`, encoding.name, encoding.name, encoding.name, encoding.code) + } + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module example.com/rawreturn\n\ngo 1.20\n", + "oracle_" + arch + ".s": assembly.String(), + "oracle_test.go": `package rawreturn +import "testing" +func Measure0() int +func Measure4() int +func ResultC2() int32 +func ResultC3() int32 +func TestActualMachineStack(t *testing.T) { + if delta := Measure0(); delta != 0 { t.Fatalf("C2 zero cleanup: SP delta %d", delta) } + if delta := Measure4(); delta != 4 { t.Fatalf("C2 imm16=4: SP delta %d", delta) } + if ResultC2() != 17 || ResultC3() != 17 { t.Fatal("zero-cleanup raw return lost AX") } +} +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + args := []string{"test", "-vet=off", "-count=1"} + if len(runPrefix) != 0 { + args = append(args, "-exec", strings.Join(runPrefix, " ")) + } + args = append(args, ".") + cmd := exec.Command("go", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOWORK=off", "GOTOOLCHAIN=local", "GOARCH="+arch, "CGO_ENABLED=0", "GODEBUG=asyncpreemptoff=1") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("actual Go %s raw-return stack oracle: %v\n%s", arch, err, out) + } +} + +func TestCrossLinuxRuntimeMatrixX86RawNearReturn(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("x86 source-stack oracle runs in the required Linux/amd64 cross-runtime gate") + } + x86RawNearReturnGoStackOracle(t, "amd64", nil) + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var runPrefix []string + if crossRosetta { + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + var source strings.Builder + for _, encoding := range []struct{ name, code string }{ + {"rawC3", "BYTE $0xc3"}, {"rawC2", "BYTE $0xc2\nWORD $0"}, + } { + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\nMOVL $17,AX\n%s\nRET\n", encoding.name, encoding.code) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawC3": {Name: "rawC3", Ret: I32}, "rawC2": {Name: "rawC2", Ret: I32}}}) + if err != nil { + t.Fatal(err) + } + const main = `#include +extern int32_t rawC2(void), rawC3(void); +int main(void) { + for (unsigned i=0; i<1024; i++) { + volatile uint64_t canary = UINT64_C(0x123456789abcdef0); + if (rawC2()!=17 || rawC3()!=17 || canary!=UINT64_C(0x123456789abcdef0)) return 1; + } + return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_near_return", triple, ir, main, runPrefix) + if os.Getenv("PLAN9ASM_CROSS_EXEC") == "1" { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("PLAN9ASM_CROSS_EXEC raw-return matrix requires Linux/amd64") + } + x86RawNearReturnGoStackOracle(t, "386", []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawC3": {Name: "rawC3", Ret: I32}, "rawC2": {Name: "rawC2", Ret: I32}}}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc", "-no-pie"}, + "raw_near_return386", "i386-unknown-linux-gnu", ir, main, []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"}) + } +} diff --git a/amd64_raw_near_return_test.go b/amd64_raw_near_return_test.go new file mode 100644 index 000000000..bb3ff1e79 --- /dev/null +++ b/amd64_raw_near_return_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestX86RawNearReturnCompleteImmediateEncoding(t *testing.T) { + if form := x86RawNearReturnEncoding([]byte{0xc3}); !form.nativeWidth || form.cleanup != 0 { + t.Fatalf("C3 has no caller cleanup: %+v", form) + } + code := []byte{0xc2, 0, 0} + for cleanup := 0; cleanup <= 65535; cleanup++ { + binary.LittleEndian.PutUint16(code[1:], uint16(cleanup)) + form := x86RawNearReturnEncoding(code) + if !form.nativeWidth || form.cleanup != uint16(cleanup) { + t.Fatalf("C2 imm16=%d was changed: %+v", cleanup, form) + } + } + for _, code := range [][]byte{nil, {0xc2}, {0xc2, 0}, {0xcb}, {0xca, 0, 0}, {0x66, 0xc3}, {0xf2, 0xc3}, {0xf3, 0xc3}, {0x48, 0xc3}} { + if x86RawNearReturnEncoding(code).nativeWidth { + t.Errorf("unproved/truncated near return %x became an ordinary return", code) + } + } +} + +var x86RawReturnTargets = []struct { + arch string + triple string +}{ + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, +} + +func TestX86RawNearReturnNeedsNativeStackContract(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, cleanup := range []uint16{1, 4, 255, 256, 65535} { + t.Run(fmt.Sprintf("%s/pop%d", target.triple, cleanup), func(t *testing.T) { + // Unlike textual RET $n, these are valid Go BYTE/WORD inputs. + // C2 pops the return PC and then adds imm16 to the caller SP. + source := fmt.Sprintf("TEXT rawret(SB),4,$0-0\nBYTE $0xc2\nWORD $%d\nRET\n", cleanup) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawret": {Name: "rawret", Ret: Void}}} + if _, err := Translate(file, opt); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("native caller cleanup=%d must remain Context, got %v", cleanup, err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module API lost native cleanup=%d contract, got %v", cleanup, err) + } + if err := ProbeInstructionSequence(ArchAMD64, target.arch, file.Funcs[0].Instrs[1:3]); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("isolated raw cleanup became an ordinary return: %v", err) + } + }) + } + } +} + +func TestX86RawNearReturnCannotInventGoEpilogue(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, push, pop := "MOVQ", "PUSHQ", "POPQ" + if target.arch == "386" { + move, push, pop = "MOVL", "PUSHL", "POPL" + } + for _, source := range []string{ + "TEXT rawret(SB),4,$8-0\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\n" + move + " $0,0(SP)\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\n" + push + " AX\nBYTE $0xc3\n" + pop + " AX\nRET\n", + "TEXT rawret(SB),4,$0-0\nBYTE $0x66\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\nBYTE $0xf2\nBYTE $0xc3\nRET\n", + "TEXT rawret(SB),4,$0-0\nBYTE $0xf3\nBYTE $0xc3\nRET\n", + } { + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawret": {Name: "rawret", Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s raw machine return cannot acquire a Go epilogue or normal stack width; got %v\n%s", target.triple, err, source) + } + } + } +} + +func x86RawStackObservingFormProbeFile(t *testing.T, arch, symbol string, code []byte) *File { + t.Helper() + return x86RawUnprovedReturnFormProbeFile(t, arch, symbol, code) +} + +// Retain the original Go encoder object as a negative return-contract test. +// A compile-only operand probe ends in UD2: it cannot promise a caller return +// after observing a physical stack or dereferencing unbound native pointers. +// Every preceding instruction byte is unchanged. These objects prove operand +// lowering, not execution of the original machine-entry function. +func x86RawUnprovedReturnFormProbeFile(t *testing.T, arch, symbol string, code []byte) *File { + t.Helper() + if len(code) == 0 || code[len(code)-1] != 0xc3 { + t.Fatalf("Go form fixture must end in its ordinary encoded RET, got %x", code) + } + makeFile := func(bytes []byte) *File { + var source strings.Builder + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", symbol) + for _, value := range bytes { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + return file + } + original := makeFile(code) + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + _, err := Translate(original, Options{Goarch: arch, TargetTriple: triple, + Sigs: map[string]FuncSig{symbol: {Name: symbol, Ret: Void}}}) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("original format fixture must retain its native return-contract failure, got %v", err) + } + if !strings.Contains(err.Error(), "source stack/continuation") && + !strings.Contains(err.Error(), "bounded typed FP/static memory") { + t.Fatalf("unrelated Context error is not a native return-contract witness: %v", err) + } + probe := append(append([]byte(nil), code[:len(code)-1]...), 0x0f, 0x0b) + return makeFile(probe) +} + +func TestX86RawUnprovedReturnFormProbePreservesInstructions(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, instruction := range []string{"MOVL (AX),CX", "MOVL CX,(AX)"} { + t.Run(target.triple+"/"+instruction, func(t *testing.T) { + code := assembleX87ControlBytes(t, target.arch, + "TEXT form(SB),4,$0-0\n"+instruction+"\nRET\n") + original := append([]byte(nil), code...) + file := x86RawUnprovedReturnFormProbeFile(t, target.arch, "form", code) + if string(code) != string(original) { + t.Fatal("form probe modified its independent Go encoder input") + } + body := file.Funcs[0].Instrs[1:] + if len(body) != len(code)+1 { + t.Fatalf("probe directives=%d, want %d", len(body), len(code)+1) + } + want := append(append([]byte(nil), code[:len(code)-1]...), 0x0f, 0x0b) + for index, ins := range body { + if ins.Op != OpBYTE || len(ins.Args) != 1 || + ins.Args[0].Kind != OpImm || ins.Args[0].Imm != int64(want[index]) { + t.Fatalf("changed operand-form byte %d: %+v, want %#x", index, ins, want[index]) + } + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"form": {Name: "form", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `asm sideeffect "ud2"`) || !strings.Contains(ir, "unreachable") { + t.Fatal("compile-only form probe invented an ordinary caller return") + } + compileLLVMToObject(t, llc, target.triple, "memory-form.ll", "memory-form.o", ir) + }) + } + } +} + +func TestX86RawNearReturnZeroCleanupLeafObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, encoding := range []string{"BYTE $0xc3", "BYTE $0xc2\nWORD $0"} { + t.Run(target.triple+"/"+strings.ReplaceAll(encoding, "\n", ";"), func(t *testing.T) { + source := "TEXT rawret(SB),4,$0-0\nMOVL $17,AX\n" + encoding + "\nRET\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawret": {Name: "rawret", Ret: I32}}, + }) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, target.triple, "raw-near-ret.ll", "raw-near-ret.o", module.String()) + }) + } + } +} diff --git a/amd64_raw_nop.go b/amd64_raw_nop.go new file mode 100644 index 000000000..bfb1aaf69 --- /dev/null +++ b/amd64_raw_nop.go @@ -0,0 +1,37 @@ +package plan9asm + +import "golang.org/x/arch/x86/x86asm" + +// The pinned decoder lacks the REX.W form of 0F 1F /0 (LLVM 22 NOOPQ). +// Only its operand width is inert. Keep every other prefix and use the decoder +// for the complete ModRM/SIB/displacement length and prefix validity. This +// narrowly scoped adapter never changes 90's XCHG/PAUSE aliases or another +// opcode, and its result is consumed only by the operand-free NOP handoff. +func decodedX86REXWNOP(code []byte) (x86asm.Inst, bool) { + if len(code) > 15 { + code = code[:15] + } + i := 0 + for i < len(code) { + switch code[i] { + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65, 0x66, 0x67, 0xf0, 0xf2, 0xf3: + i++ + default: + goto opcode + } + } + +opcode: + if i+4 > len(code) || code[i]&0xf8 != 0x48 || + code[i+1] != 0x0f || code[i+2] != 0x1f || code[i+3]&0x38 != 0 { + return x86asm.Inst{}, false + } + var adjusted [15]byte + copy(adjusted[:], code) + adjusted[i] &^= 8 + inst, err := x86asm.Decode(adjusted[:len(code)], 64) + if err != nil || inst.Op != x86asm.NOP || inst.Len <= i+3 { + return x86asm.Inst{}, false + } + return inst, true +} diff --git a/amd64_raw_nop_handoff_test.go b/amd64_raw_nop_handoff_test.go new file mode 100644 index 000000000..fe5531a35 --- /dev/null +++ b/amd64_raw_nop_handoff_test.go @@ -0,0 +1,207 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel SDM NOP (90 and 0F 1F /0) has no register or memory effect. +// Independently enumerate ModRM/SIB and prefix axes, including addresses +// which would need a caller, segment, stack or PC contract if dereferenced. +func x86RawNOPWitnesses(mode int) [][]byte { + forms := [][]byte{{0x90}, {0x66, 0x90}} + for mod := byte(0); mod < 4; mod++ { + for rm := byte(0); rm < 8; rm++ { + code := []byte{0x0f, 0x1f, mod<<6 | rm} + if mod != 3 && rm == 4 { + code = append(code, 0x24) + } + switch { + case mod == 1: + code = append(code, 0xff) + case mod == 2 || mod == 0 && rm == 5: + code = append(code, 0xff, 0xff, 0xff, 0x7f) + } + forms = append(forms, code) + } + } + for sib := 0; sib < 256; sib++ { + code := []byte{0x0f, 0x1f, 0x04, byte(sib)} + if sib&7 == 5 { + code = append(code, 0xff, 0xff, 0xff, 0x7f) + } + forms = append(forms, code) + } + for _, prefix := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65, 0x66} { + forms = append(forms, []byte{prefix, 0x0f, 0x1f, 0x00}) + } + forms = append(forms, []byte{0x66, 0x0f, 0x1f, 0x84, 0x00, 0, 0, 0, 0}) + if mode == 32 { + // The 16-bit address decoder must not manufacture BX/SI inputs. + forms = append(forms, []byte{0x67, 0x0f, 0x1f, 0x00}) + } else { + // Inert EIP-relative address and actual extended REX banks. + forms = append(forms, []byte{0x67, 0x0f, 0x1f, 0x05, 0xff, 0xff, 0xff, 0x7f}) + for rex := byte(0x40); rex < 0x50; rex++ { + forms = append(forms, []byte{rex, 0x0f, 0x1f, 0x04, 0x00}) + } + } + return forms +} + +func TestX86RawNOPPhysicalFieldsAreInert(t *testing.T) { + for _, mode := range []int{32, 64} { + arch := "amd64" + if mode == 32 { + arch = "386" + } + forms := x86RawNOPWitnesses(mode) + var complete []byte + for _, code := range forms { + complete = append(complete, code...) + } + x86HandoffOriginalBytes(t, arch, complete) + for _, code := range forms { + t.Run(fmt.Sprintf("mode%d/%x", mode, code), func(t *testing.T) { + physical, err := x86asm.Decode(code, mode) + // The independent literal object and runtime oracle cover the + // pinned decoder's missing REX.W NOP row; do not use the + // production adapter as its own acceptance oracle. + missingWidth := mode == 64 && code[0]&0xf8 == 0x48 + if !missingWidth && (err != nil || physical.Op != x86asm.NOP || physical.Len != len(code)) { + t.Fatalf("independent NOP encoding: %+v %v", physical, err) + } + instructions, err := decodeX86RawDirectiveGroup(code, mode, 0, "NOP witness", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(instructions) != 1 || instructions[0].Op != "NOP" || len(instructions[0].Args) != 0 || + instructions[0].x86UnboundAddressOverride || instructions[0].x86AddressBits != 0 { + t.Fatalf("inert address acquired an effect: %+v", instructions) + } + }) + } + } +} + +func TestX86RawNOPGoEncoderWitnesses(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for _, registration := range []string{ + "var ydivl = []ytab{\n\t{Zm_o, 2, argList{Yml}},\n}", + "{ANOPW, ydivl, Pe, opBytes{0x0f, 0x1f, 00}}", + "{ANOPL, ydivl, Px, opBytes{0x0f, 0x1f, 00}}", + } { + if !strings.Contains(string(table), registration) { + t.Fatal("Go NOP encoder changed:", registration) + } + } + for _, arch := range []string{"386", "amd64"} { + for _, spelling := range []string{"NOPL AX", "NOPW AX", "NOPL (AX)", "NOPW -1(AX)(CX*8)"} { + code := x86OrderingHintGoBytes(t, arch, "TEXT probe(SB),4,$0-0\n"+spelling+"\n") + mode := 64 + if arch == "386" { + mode = 32 + } + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, spelling, map[string]bool{}) + if err != nil || len(decoded) != 1 || decoded[0].Op != "NOP" { + t.Fatalf("actual %s %s bytes %x: %+v %v", arch, spelling, code, decoded, err) + } + } + } +} + +func TestX86RawNOPDoesNotEraseOtherEffectsOrBoundaries(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, code := range [][]byte{ + {0xf0, 0x90}, {0xf0, 0x0f, 0x1f, 0x00}, + {0xf0, 0x48, 0x0f, 0x1f, 0x00}, + {0x0f}, {0x0f, 0x1f}, {0x0f, 0x1f, 0x84, 0x00}, + {0x48, 0x0f, 0x1f, 0x84, 0x00}, + } { + if _, err := decodeX86RawDirectiveGroup(code, mode, 0, "invalid NOP", map[string]bool{}); err == nil { + t.Fatalf("mode %d accepted invalid/truncated bytes %x", mode, code) + } + } + for extension := byte(1); extension < 8; extension++ { + code := []byte{0x0f, 0x1f, extension << 3} + if _, err := decodeX86RawDirectiveGroup(code, mode, 0, "reserved NOP extension", map[string]bool{}); err == nil { + t.Fatalf("reserved /%d NOP accepted", extension) + } + if _, ok := decodedX86REXWNOP(append([]byte{0x48}, code...)); ok { + t.Fatalf("REX.W adapter accepted reserved /%d", extension) + } + } + for _, code := range [][]byte{{0xf3, 0x90}, {0x87, 0xc0}, {0x8b, 0x00}} { + instructions, err := decodeX86RawDirectiveGroup(code, mode, 0, "not NOP", map[string]bool{}) + if err != nil || len(instructions) != 1 || instructions[0].Op == "NOP" { + t.Fatalf("mode %d lost non-NOP effect %x: %+v %v", mode, code, instructions, err) + } + } + // A branch into a NOP's ModRM byte is still a mid-instruction edge. + interior := []byte{0x74, 1, 0x0f, 0x1f, 0x00, 0xc3} + if _, err := decodeX86RawDirectiveGroup(interior, mode, 0, "interior", map[string]bool{}); err == nil { + t.Fatal("branch into multi-byte NOP accepted") + } + boundary := []byte{0x74, 3, 0x0f, 0x1f, 0x00, 0x90, 0xc3} + if _, err := decodeX86RawDirectiveGroup(boundary, mode, 0, "boundary", map[string]bool{}); err != nil { + t.Fatal("valid branch/NOP boundaries:", err) + } + // The generic physical handoff remains strict for real memory. + physical, err := x86asm.Decode([]byte{0x8b, 0x00}, mode) + if err != nil { + t.Fatal(err) + } + if _, err := parseDecodedX86PhysicalInstruction(physical, "NOP"); err == nil { + t.Fatal("formatter erased a real memory read") + } + } +} + +func TestX86RawNOPThreeAPIsFiveTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86OrderingHintTargets { + mode := 64 + if target.arch == "386" { + mode = 32 + } + var source strings.Builder + source.WriteString("TEXT ·probe(SB),4,$0-4\n") + for _, code := range x86RawNOPWitnesses(mode) { + source.WriteString(x86RawDisplacementDirectives(code)) + } + source.WriteString("MOVL $73,ret+0(FP)\nRET\n") + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), "package ordering\nfunc probe() uint32\n", target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + if bytes.Contains([]byte(ir), []byte("inttoptr")) { + t.Fatal("inert NOP address became a native pointer") + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "nop.ll"), filepath.Join(dir, "nop.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", "-o", object, input)) + } + }) + } + } +} diff --git a/amd64_raw_nop_result_go127_test.go b/amd64_raw_nop_result_go127_test.go new file mode 100644 index 000000000..dfef90aa0 --- /dev/null +++ b/amd64_raw_nop_result_go127_test.go @@ -0,0 +1,12 @@ +//go:build go1.27 + +package plan9asm + +import "testing" + +// The discovering current-Go source combines PCALIGN with literal NOP padding. +// Go 1.20 rejects PCALIGN; the untagged root still checks every literal NOP and +// paired native-read contract on both the historical and current SDKs. +func TestX86RawNOPResultReadWithCurrentGoAlignment(t *testing.T) { + x86RawNOPResultReadComposition(t, true) +} diff --git a/amd64_raw_nop_result_runtime_unix_test.go b/amd64_raw_nop_result_runtime_unix_test.go new file mode 100644 index 000000000..27b93ae29 --- /dev/null +++ b/amd64_raw_nop_result_runtime_unix_test.go @@ -0,0 +1,72 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86RawNOPPairedNativeRead(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + literal := x86RawDisplacementDirectives([]byte{0x66, 0x0f, 0x1f, 0x84, 0, 0, 0, 0, 0}) + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + for _, vector := range []bool{false, true} { + for _, dead := range []bool{false, true} { + name, original := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count) + read, width, inputByte, outputByte := "MOVQ (DX),AX\n", 8, 0, 0 + if vector { + read, width, inputByte, outputByte = "MOVOU (DX),X1\nMOVQ X1,AX\n", 16, 1, 1 + } + body := literal + read + literal + discarded := 0 + if dead { + body += "MOVQ $0x87654321fedcba98,AX\n" + discarded = 1 + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-16\nMOVQ p+0(FP),DX\n%sMOVQ AX,ret+8(FP)\nRET\n", name, body) + fmt.Fprintf(&declarations, "func %s(p *[64]byte) uint64\n", name) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+strings.ReplaceAll(body, "(DX)", "(DI)")+"RET\n") + var encoded []string + for _, b := range code { + encoded = append(encoded, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(encoded, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\nextern uint64_t %s(const uint8_t*);\n", name, original) + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d,%d},\n", name, original, width, discarded, inputByte, outputByte) + count++ + } + } + if count != 4 { + t.Fatal("incomplete scalar/vector live/dead matrix", count) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + driver := fmt.Sprintf(x86NativeScalarReadRuntimeC, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "nop-native-read", text, driver, level) + if string(output) != "4 native scalar kernels: numerical 512, faults 48, tripwires 2; failures 0\n" { + t.Fatal("original NOP/read composition differs from LLVM", string(output)) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} diff --git a/amd64_raw_nop_result_test.go b/amd64_raw_nop_result_test.go new file mode 100644 index 000000000..7fa090869 --- /dev/null +++ b/amd64_raw_nop_result_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func TestX86RawNOPResultReadComposition(t *testing.T) { + x86RawNOPResultReadComposition(t, false) +} + +func x86RawNOPResultReadComposition(t *testing.T, align bool) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + mode, word, move := 64, 8, "MOVQ" + if target.arch == "386" { + mode, word, move = 32, 4, "MOVL" + } + var literals strings.Builder + if align { + literals.WriteString("PCALIGN $32\n") + } + for _, code := range x86RawNOPWitnesses(mode) { + literals.WriteString(x86RawDisplacementDirectives(code)) + } + body := fmt.Sprintf("%s p+0(FP),DX\n%sVMOVDQU (DX),Y1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+%d(FP)\nRET", + move, literals.String(), word) + _, file, sig, source := x86ResultInitializationFixture(t, target.arch, "func Value(p *[64]byte) uint32", body) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + normalized, err := decodeX86RawDirectives(file.Funcs[0], target.arch) + if err != nil { + t.Fatal(err) + } + if err := CheckX86ABI0ResultInitialization(normalized, sig, target.arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("inert raw bytes granted a standalone native read", err) + } + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, "func Value(p *[64]byte) uint32", target.arch, target.triple, api) + if err != nil { + t.Fatal("inert NOPs blocked the independently paired native read", api, err) + } + compileLLVMToObject(t, llc, target.triple, "nop-result.ll", "nop-result.o", ir) + compileLLVMToObject(t, llc, target.triple, "nop-result-opt.ll", "nop-result-opt.o", x86MulDivRoundOptimize(t, ir)) + } + }) + } +} + +func TestX86RawNOPResultReadRetainsRawBoundaries(t *testing.T) { + literal := x86RawDisplacementDirectives([]byte{0x66, 0x0f, 0x1f, 0x84, 0, 0, 0, 0, 0}) + _, file, sig, _ := x86ResultInitializationFixture(t, "amd64", "func Value(p *[64]byte) uint32", + "MOVQ p+0(FP),DX\n"+literal+"VMOVDQU (DX),Y1\nPEXTRD $0,X1,AX\nMOVL AX,ret0+8(FP)\nRET") + normalized, err := decodeX86RawDirectives(file.Funcs[0], "amd64") + if err != nil { + t.Fatal(err) + } + nop := -1 + for i, ins := range normalized.Instrs { + if ins.x86SourceRaw != nil { + if nop != -1 || ins.Op != "NOP" { + t.Fatal("fixture did not decode one inert raw instruction") + } + nop = i + } + } + if nop == -1 { + t.Fatal("missing actual raw NOP handoff") + } + for _, mutation := range []string{"original opcode", "original operands", "normalized opcode", "normalized operands", "encoded", "address size", "raw return", "raw TEXT", "source stack", "fixed local"} { + t.Run(mutation, func(t *testing.T) { + fn := normalized + fn.Instrs = append([]Instr(nil), fn.Instrs...) + ins := &fn.Instrs[nop] + origin := *ins.x86SourceRaw + ins.x86SourceRaw = &origin + switch mutation { + case "original opcode": + origin.op = "PAUSE" + case "original operands": + origin.args = []Operand{{Kind: OpReg, Reg: AX}} + case "normalized opcode": + ins.Op = "PAUSE" + case "normalized operands": + ins.Args = []Operand{{Kind: OpReg, Reg: AX}} + case "encoded": + ins.x86Encoded = true + case "address size": + ins.x86AddressBits = 32 + case "raw return": + ins.x86RawNearReturn = new(x86RawNearReturnForm) + case "raw TEXT": + fn.X86RawText = []byte{0x90} + case "source stack": + fn.x86SourceStack = new(x86SourceStackProof) + case "fixed local": + fn.x86FixedLocal = new(x86FixedLocalProof) + } + ctx := &amd64Ctx{fpScalarSource: fn, sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("inert padding granted another raw or entry contract", err) + } + }) + } +} diff --git a/amd64_raw_nop_runtime_test.go b/amd64_raw_nop_runtime_test.go new file mode 100644 index 000000000..d91a00488 --- /dev/null +++ b/amd64_raw_nop_runtime_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +// Reuse only the independent GP/FLAGS/MXCSR/canary driver. Replace every tested +// instruction with a disjoint part of the complete literal-NOP witness set. +// Noncanonical/null effective addresses must not fault or change native state. +func x86RawNOPRuntimeFixture(bits int) (declarations, assembly, driver, reference string) { + declarations, assembly, driver, reference = x86OrderingHintRuntimeFixture(bits) + var batches [4]strings.Builder + for i, code := range x86RawNOPWitnesses(bits) { + batches[i%len(batches)].WriteString(x86RawDisplacementDirectives(code)) + } + for i, op := range []string{"LFENCE", "MFENCE", "SFENCE", "PAUSE"} { + assembly = strings.ReplaceAll(assembly, "\n"+op+"\n", "\n"+batches[i].String()) + } + return +} + +// Portable LLVM execution is distinct from the required original-Go native +// roots. Rosetta rejects some valid register NOPs (for example 0F 1F C2), so +// it cannot certify this family's original machine execution. +func TestX86RawNOPLLVMRuntime(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + declarations, assembly, _, reference := x86RawNOPRuntimeFixture(64) + ir := x86NarrowOracleIR(t, assembly, declarations, execution.triple) + for _, level := range []string{"-O0", "-O2"} { + t.Run(level, func(t *testing.T) { + output := execution.compileAndRunLLVM(t, "nop", ir, reference, level) + if string(output) != "ordering observations 256\n" { + t.Fatalf("LLVM NOP GP/FLAGS/MXCSR/canary oracle: %q", output) + } + }) + } +} + +func TestX86RawNOPOriginal386StaticELFCompileOnly(t *testing.T) { + declarations, assembly, driver, _ := x86RawNOPRuntimeFixture(32) + original := x86OrderingHintBuildOriginal(t, t.TempDir(), "linux", "386", declarations, assembly, driver) + sealed, err := x87IA32StaticExecutable(original) + if err != nil { + t.Fatal(err) + } + t.Logf("actual original Go IA32 NOP ELF bytes=%d SHA256=%x; compile only", sealed.size, sealed.hash) +} diff --git a/amd64_raw_operand_handoff.go b/amd64_raw_operand_handoff.go new file mode 100644 index 000000000..9d019d6d8 --- /dev/null +++ b/amd64_raw_operand_handoff.go @@ -0,0 +1,205 @@ +package plan9asm + +import ( + "fmt" + "strings" + + "golang.org/x/arch/x86/x86asm" +) + +// parseDecodedX86PhysicalInstruction is the private decoder handoff, not a Go +// source grammar extension. GoSyntax orders operands and selects the audited +// mnemonic; the actual decoder Mem supplies the physical address. In +// particular, FS:..., default DS/ES, and RIP are not source register groups. +// Prefix, PC-relative layout, and address-size admission remain the caller's +// independent checks. No memory/caller/frame certificate is manufactured here. +func parseDecodedX86PhysicalInstruction(inst x86asm.Inst, syntax string) ([]Instr, error) { + physical := normalizeDecodedX86LEASegments(normalizeDecodedX86MemoryDisplacements(inst)) + const marker = "__plan9asm_decoded_memory_" + if strings.Contains(syntax, marker) { + return nil, fmt.Errorf("decoded syntax collides with private memory placeholder") + } + type memoryBinding struct { + name string + operand Operand + } + bindings := make(map[string]memoryBinding) + byName := make(map[string]Operand) + memoryCount := 0 + for _, arg := range physical.Args { + memory, ok := arg.(x86asm.Mem) + if !ok { + continue + } + operand, err := decodedX86PhysicalMemory(memory) + if err != nil { + return nil, err + } + name := fmt.Sprintf("%s%d", marker, memoryCount) + memoryCount++ + byName[name] = operand + // Use the same pinned formatter only to locate this complete operand + // token. Never interpret its text as a public memory/address role. + probe := physical + probe.Op = x86asm.NOP + probe.Prefix = x86asm.Prefixes{} + probe.Args = x86asm.Args{memory} + text := x86asm.GoSyntax(probe, 0, nil) + space := strings.IndexByte(text, ' ') + if space < 0 { + return nil, fmt.Errorf("decoder formatter omitted its physical memory operand") + } + text = text[space+1:] + spellings := []string{text, strings.TrimPrefix(normalizeDecodedX86SegmentMemorySyntax("NOP "+text), "NOP ")} + for _, spelling := range spellings { + if prior, exists := bindings[spelling]; exists && prior.operand.Mem != operand.Mem { + return nil, fmt.Errorf("ambiguous decoded physical memory %q", spelling) + } + bindings[spelling] = memoryBinding{name: name, operand: operand} + } + } + for _, prefix := range physical.Prefix { + if memoryCount == 0 || prefix&(x86asm.PrefixIgnored|x86asm.PrefixImplicit) != 0 { + continue + } + switch prefix & 0xff { + case x86asm.PrefixES, x86asm.PrefixCS, x86asm.PrefixSS, x86asm.PrefixDS, x86asm.PrefixFS, x86asm.PrefixGS: + bound := false + for _, argument := range physical.Args { + if memory, ok := argument.(x86asm.Mem); ok && memory.Segment.String() == prefix.String() { + bound = true + } + } + if !bound { + // A later operand-size prefix can hide this unused segment + // in GoSyntax. Absence from text is not an ignored witness. + return nil, fmt.Errorf("%w: raw %s has an unbound physical %s segment prefix", ErrProbeNeedsContext, inst.Op, prefix) + } + } + } + + parts := strings.Split(syntax, ";") + consumed := 0 + for i, part := range parts { + part = strings.TrimSpace(part) + // GoSyntax prints an architecturally ignored segment before the + // opcode (for example FS SCASB). Drop only that actual metadata-tagged + // prefix, not a used segment, an invalid prefix, or a textual guess. + for _, prefix := range physical.Prefix { + switch prefix & 0xff { + case x86asm.PrefixES, x86asm.PrefixCS, x86asm.PrefixSS, x86asm.PrefixDS, x86asm.PrefixFS, x86asm.PrefixGS: + if !strings.HasPrefix(part, prefix.String()+" ") { + continue + } + if prefix&x86asm.PrefixIgnored == 0 { + return nil, fmt.Errorf("%w: raw %s has an unbound physical %s segment prefix", ErrProbeNeedsContext, inst.Op, prefix) + } + part = strings.TrimPrefix(part, prefix.String()+" ") + } + } + space := strings.IndexByte(part, ' ') + if space >= 0 { + args := splitTopLevelCSV(part[space+1:]) + for j, arg := range args { + if binding, ok := bindings[strings.TrimSpace(arg)]; ok { + args[j] = binding.name + consumed++ + } + } + part = part[:space+1] + strings.Join(args, ", ") + } + parts[i] = part + } + if consumed != memoryCount { + return nil, fmt.Errorf("decoded physical memory handoff consumed %d operands, want %d", consumed, memoryCount) + } + instructions, err := parseDecodedX86Instruction(strings.Join(parts, "; ")) + if err != nil { + return nil, err + } + if len(instructions) != len(parts) { + return nil, fmt.Errorf("decoded instruction/prefix handoff changed instruction count") + } + restored := 0 + for i := range instructions { + for j, arg := range instructions[i].Args { + if arg.Kind != OpIdent { + continue + } + if operand, ok := byName[arg.Ident]; ok { + instructions[i].Args[j] = operand + restored++ + } + } + instructions[i].Raw = strings.TrimSpace(strings.Split(syntax, ";")[i]) + } + if restored != memoryCount { + return nil, fmt.Errorf("decoded physical memory handoff restored %d operands, want %d", restored, memoryCount) + } + return instructions, nil +} + +// The pinned decoder appends L to these environment-load mnemonics, whereas +// Go's complete ysvrs_mo rows use FLDENV/FRSTOR for the 32-bit layout. Bind the +// real opcode, full prefix metadata, and complete encoding before dropping +// that formatter suffix. The 14/94-byte operand-size override is not this +// 28/108-byte Go layout and remains unsupported. +func decodedX86X87EnvironmentLoad(inst x86asm.Inst, encoding []byte) (Op, bool, error) { + if inst.Op != x86asm.FLDENV && inst.Op != x86asm.FRSTOR { + return "", false, nil + } + witness, err := x86asm.Decode(encoding, inst.Mode) + witness = normalizeDecodedX86LEASegments(normalizeDecodedX86MemoryDisplacements(witness)) + memory, hasMemory := inst.Args[0].(x86asm.Mem) + witnessMemory, hasWitnessMemory := witness.Args[0].(x86asm.Mem) + if err != nil || witness.Len != len(encoding) || witness.Len != inst.Len || + witness.Op != inst.Op || witness.Opcode != inst.Opcode || witness.Prefix != inst.Prefix || + witness.DataSize != inst.DataSize || witness.AddrSize != inst.AddrSize || witness.MemBytes != inst.MemBytes || + !hasMemory || !hasWitnessMemory || memory != witnessMemory || + inst.Args[1] != nil || inst.Args[2] != nil || inst.Args[3] != nil { + return "", true, fmt.Errorf("%w: raw x87 environment load lacks its complete decoder encoding", ErrProbeNeedsContext) + } + if inst.DataSize != 32 { + return "", true, fmt.Errorf("raw %s uses a %d-bit environment layout not represented by Go %s", inst.Op, inst.DataSize, inst.Op) + } + return Op(inst.Op.String()), true, nil +} + +func decodedX86PhysicalMemory(memory x86asm.Mem) (Operand, error) { + register := func(reg x86asm.Reg) (Reg, error) { + if reg == 0 { + return "", nil + } + if reg == x86asm.RIP { + return Reg("IP"), nil // Consumed only by the separate RIP layout proof. + } + text := x86asm.GoSyntax(x86asm.Inst{Op: x86asm.NOP, Args: x86asm.Args{reg}}, 0, nil) + space := strings.IndexByte(text, ' ') + if space < 0 { + return "", fmt.Errorf("decoder formatter omitted its physical address register") + } + text = text[space+1:] + value, ok := parseReg(text) + if !ok { + return "", fmt.Errorf("unsupported decoded physical address register %s", reg) + } + return value, nil + } + base, err := register(memory.Base) + if err != nil { + return Operand{}, err + } + index, err := register(memory.Index) + if err != nil { + return Operand{}, err + } + segment := Reg("") + switch memory.Segment { + case 0: + case x86asm.ES, x86asm.CS, x86asm.SS, x86asm.DS, x86asm.FS, x86asm.GS: + segment = Reg(memory.Segment.String()) + default: + return Operand{}, fmt.Errorf("unsupported decoded physical segment %s", memory.Segment) + } + return Operand{Kind: OpMem, Mem: MemRef{Base: base, Index: index, Scale: int64(memory.Scale), Off: memory.Disp, Segment: segment}}, nil +} diff --git a/amd64_raw_operand_handoff_test.go b/amd64_raw_operand_handoff_test.go new file mode 100644 index 000000000..8a608f615 --- /dev/null +++ b/amd64_raw_operand_handoff_test.go @@ -0,0 +1,593 @@ +package plan9asm + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/xgo-dev/llvm" + "golang.org/x/arch/x86/x86asm" +) + +// This oracle assembles literal BYTE directives, not GoSyntax's synthetic +// memory spelling. Its complete contiguous text bytes independently anchor +// the private decoder input on both supported SDK test lanes. +func x86HandoffOriginalBytes(t *testing.T, arch string, code []byte) { + t.Helper() + source := "TEXT handoff(SB),4,$0-0\n" + x86RawDisplacementDirectives(code) + listing, err := x86ScalarLiteralGoListing(t, arch, source) + if err != nil { + t.Fatal(err) + } + var actual []byte + for _, line := range strings.Split(listing, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, err := strconv.ParseUint(fields[0][2:], 16, 32) + if err != nil || int(offset) != len(actual) { + t.Fatalf("noncontiguous stock Go bytes: %s", line) + } + // Both audited objfile.go printers use sixteen fixed three-byte + // hex columns, then two spaces and ASCII. Never parse the ASCII + // tail as another byte, even when it happens to be "ab" or "f.". + text := strings.TrimLeft(line, "\t ") + start := strings.IndexByte(text, ' ') + if start < 0 || len(text) < start+48 { + t.Fatalf("missing complete stock Go hex columns: %s", line) + } + gap := false + for i := 0; i < 16; i++ { + field := text[start+3*i : start+3*(i+1)] + if field == " " { + gap = true + continue + } + value, err := strconv.ParseUint(field[1:], 16, 8) + if gap || field[0] != ' ' || err != nil { + t.Fatalf("invalid stock Go hex column: %q", field) + } + actual = append(actual, byte(value)) + } + } + if !bytes.Equal(actual, code) { + t.Fatalf("stock Go raw text = %x, want complete %x", actual, code) + } +} + +func TestX86PrivateDecoderX87PhysicalMemoryHandoff(t *testing.T) { + forms := []struct { + op Op + code []byte + }{ + {"FSAVE", []byte{0xdd, 0x74, 0x8b, 0xf9}}, + {"FSTCW", []byte{0xd9, 0x7c, 0x8b, 0xf9}}, + {"FSTENV", []byte{0xd9, 0x74, 0x8b, 0xf9}}, + {"FSTSW", []byte{0xdd, 0x7c, 0x8b, 0xf9}}, + {"FLDCW", []byte{0xd9, 0x6c, 0x8b, 0xf9}}, + {"FLDENV", []byte{0xd9, 0x64, 0x8b, 0xf9}}, + {"FRSTOR", []byte{0xdd, 0x64, 0x8b, 0xf9}}, + } + segments := []struct { + prefix byte + reg Reg + }{{0x26, ES}, {0x2e, CS}, {0x36, SS}, {0x3e, DS}, {0x64, FS}, {0x65, GS}} + for _, arch := range []string{"386", "amd64"} { + var complete []byte + for _, form := range forms { + for _, segment := range segments { + complete = append(complete, segment.prefix) + complete = append(complete, form.code...) + } + } + complete = append(complete, 0xc3) + x86HandoffOriginalBytes(t, arch, complete) + for _, form := range forms { + for _, segment := range segments { + t.Run(arch+"/"+string(form.op)+"/"+string(segment.reg), func(t *testing.T) { + code := append([]byte{segment.prefix}, form.code...) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + want := Operand{Kind: OpMem, Mem: MemRef{Base: BX, Index: CX, Scale: 4, Off: -7, Segment: segment.reg}} + if arch == "amd64" && segment.reg != FS && segment.reg != GS { + // In mode64 the actual decoder ignores the legacy flat + // segment prefix; it supplies no physical override. + want.Mem.Segment = "" + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != form.op || len(decoded.Instrs[0].Args) != 1 || !reflect.DeepEqual(decoded.Instrs[0].Args[0], want) { + t.Fatalf("physical memory changed: %+v, want %s %+v", decoded.Instrs, form.op, want) + } + }) + } + } + } +} + +func TestX86PrivateDecoderImplicitPhysicalSegments(t *testing.T) { + forms := []struct { + name string + opcode byte + source bool + dest bool + }{ + {"MOVS", 0xa4, true, true}, {"CMPS", 0xa6, true, true}, + {"STOS", 0xaa, false, true}, {"LODS", 0xac, true, false}, + {"SCAS", 0xae, false, true}, {"INS", 0x6c, false, true}, + {"OUTS", 0x6e, true, false}, + } + for _, mode := range []int{32, 64} { + arch := "amd64" + if mode == 32 { + arch = "386" + } + var complete []byte + for _, form := range forms { + for _, segment := range []byte{0, 0x64, 0x65} { + for _, repeat := range []byte{0, 0xf2, 0xf3} { + code := []byte{0x67} + if repeat != 0 { + code = append(code, repeat) + } + if segment != 0 { + code = append(code, segment) + } + code = append(code, form.opcode) + complete = append(complete, code...) + } + } + } + complete = append(complete, 0xc3) + x86HandoffOriginalBytes(t, arch, complete) + for _, form := range forms { + for _, segment := range []byte{0, 0x64, 0x65} { + for _, repeat := range []byte{0, 0xf2, 0xf3} { + t.Run(fmt.Sprintf("mode%d/%s/seg%02x/rep%02x", mode, form.name, segment, repeat), func(t *testing.T) { + code := []byte{0x67} + if repeat != 0 { + code = append(code, repeat) + } + if segment != 0 { + code = append(code, segment) + } + code = append(code, form.opcode) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("complete physical decode: %+v %v", physical, err) + } + instructions, err := x86AddressOverrideDecode(mode, code) + if !form.source && segment != 0 { + // x/arch does not mark this unused FS/GS prefix as + // ignored. Do not guess or strip it: precise Context + // replaces the old accidental source-parser failure. + if !errors.Is(err, ErrProbeNeedsContext) || !strings.Contains(err.Error(), "unbound physical") { + t.Fatalf("unbound physical prefix: %v", err) + } + return + } + if err != nil { + t.Fatal(err) + } + var memories []MemRef + for _, ins := range instructions { + if ins.Op == OpTEXT || ins.Op == OpRET { + continue + } + if !ins.x86UnboundAddressOverride || ins.x86AddressBits != mode/2 { + t.Fatalf("lost physical address restriction: %+v", ins) + } + for _, arg := range ins.Args { + if arg.Kind == OpMem { + memories = append(memories, arg.Mem) + } + } + ctx, out := newAMD64CtxWithFuncForTest(t, Func{Instrs: []Instr{ins}}, FuncSig{Ret: Void}, nil) + ctx.goarch = arch + before := out.String() + if terminated, err := ctx.lowerInstr(0, 0, ins, nil, nil); terminated || !errors.Is(err, ErrProbeNeedsContext) || out.String() != before { + t.Fatalf("unbound physical memory emitted IR: %t %v", terminated, err) + } + } + wantSource := DS + if segment == 0x64 { + wantSource = FS + } else if segment == 0x65 { + wantSource = GS + } + seenSource, seenDest := false, false + for _, memory := range memories { + if memory.Base == SI && memory.Segment == wantSource { + seenSource = true + } else if memory.Base == DI && memory.Segment == ES { + seenDest = true + } else { + t.Fatalf("unexpected implicit physical memory: %+v", memory) + } + } + if seenSource != form.source || seenDest != form.dest { + t.Fatalf("lost source/destination segments: %+v", memories) + } + }) + } + } + } + } +} + +func TestX86PrivateDecoderRIPPoolAllPublicAPIs(t *testing.T) { + code := []byte{0x48, 0x8d, 0x05, 1, 0, 0, 0, 0xc3, 0x41} + x86HandoffOriginalBytes(t, "amd64", code) + file := mustParseX86RawReturn(t, x86StaticReturnSource("Bound", code)) + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // RIP is a mode64-only machine addressing form. + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, opt) + case "module": + module, callErr := TranslateModule(file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + default: + ctx := llvm.NewContext() + module, callErr := TranslateModuleInContext(ctx, file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + ctx.Dispose() + } + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "handoff.ll", "handoff.o", ir) + }) + } + } +} + +func TestX86PrivateDecoderX87FiveTargetObjects(t *testing.T) { + // Object-only controls for the already-supported FS/GS address spaces. + // They do not execute an unknown native pointer or prove caller bounds. + var code []byte + for _, opcode := range [][]byte{ + {0xdd, 0x30}, {0xd9, 0x38}, {0xd9, 0x30}, {0xdd, 0x38}, + {0xd9, 0x28}, {0xd9, 0x20}, {0xdd, 0x20}, + {0xdf, 0x20}, {0xdf, 0x30}, // packed BCD load/store + } { + for _, segment := range []byte{0x64, 0x65} { + code = append(code, segment) + code = append(code, opcode...) + } + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + // This is the same named RET domain as the old x87 positive. + // A raw RET plus unknown native memory remains Context. + source := x86StaticReturnSource("Bound", code) + + "FMOVX 0(AX)(FS),F0\nFMOVXP F0,0(AX)(FS)\n" + + "FMOVX 0(AX)(GS),F0\nFMOVXP F0,0(AX)(GS)\nRET\n" + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file := mustParseX86RawReturn(t, source) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, X87Mode: X87Software, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(api, func(t *testing.T) { + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, opt) + case "module": + module, callErr := TranslateModule(file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + default: + ctx := llvm.NewContext() + module, callErr := TranslateModuleInContext(ctx, file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + ctx.Dispose() + } + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "physical-x87.ll", "physical-x87.o", ir) + }) + } + }) + } +} + +func TestX86PrivateDecoderHandoffDoesNotWidenSourceOrLayout(t *testing.T) { + for _, source := range []string{"FSAVE FS:0(AX)", "MOVSB DS:0(SI),ES:0(DI)", "LEAQ 1(IP),AX"} { + t.Run("public/"+source, func(t *testing.T) { + if _, err := Parse(ArchAMD64, "TEXT raw(SB),4,$0-0\n"+source+"\nRET\n"); err == nil { + t.Fatal("private physical memory spelling became public source syntax") + } + }) + } + for _, opcode := range [][]byte{{0xdd, 0x30}, {0xd9, 0x30}, {0xd9, 0x20}, {0xdd, 0x20}} { + for _, mode := range []int{32, 64} { + t.Run(fmt.Sprintf("legacy-layout/%d/%x", mode, opcode), func(t *testing.T) { + code := append([]byte{0x66}, opcode...) + arch := "amd64" + if mode == 32 { + arch = "386" + } + x86HandoffOriginalBytes(t, arch, append(append([]byte(nil), code...), 0xc3)) + _, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err == nil || !strings.Contains(err.Error(), "16-bit environment layout") { + t.Fatalf("14/94-byte layout acquired the 28/108-byte alias: %v", err) + } + }) + } + } + code := []byte{0xd9, 0x20} + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + for name, mutate := range map[string]func(*x86asm.Inst){ + "prefix": func(i *x86asm.Inst) { i.Prefix[0] = x86asm.PrefixFS }, + "size": func(i *x86asm.Inst) { i.DataSize = 16 }, + "length": func(i *x86asm.Inst) { i.Len++ }, + "memory": func(i *x86asm.Inst) { i.Args[0] = x86asm.Mem{Base: x86asm.RBX} }, + "extra": func(i *x86asm.Inst) { i.Args[1] = x86asm.RAX }, + } { + t.Run("alias-witness/"+name, func(t *testing.T) { + changed := inst + mutate(&changed) + if _, recognized, err := decodedX86X87EnvironmentLoad(changed, code); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed decoded observation borrowed the environment alias: %v", err) + } + }) + } + for _, changed := range [][]byte{{0xd9}, {0xd9, 0x28}, {0xd9, 0x20, 0x90}} { + t.Run(fmt.Sprintf("alias-bytes/%x", changed), func(t *testing.T) { + if _, recognized, err := decodedX86X87EnvironmentLoad(inst, changed); !recognized || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("changed encoding borrowed the environment alias: %v", err) + } + }) + } + for _, code := range [][]byte{{0x67, 0x48, 0x8d, 0x05, 0, 0, 0, 0}, {0xf0, 0xd9, 0x20}} { + t.Run(fmt.Sprintf("physical-boundary/%x", code), func(t *testing.T) { + _, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err == nil { + t.Fatal("unbound EIP or invalid prefix acquired private memory admission") + } + }) + } +} + +func TestX86PrivateDecoderNamedX87MemoryConsumers(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT Bound(SB),4,$0-0\n") + var forms []string + for _, segment := range []string{"FS", "GS"} { + // Named FLDENV is the separately observed stock assembler defect + // below, not an accepted object control. Its complete raw opcode + // remains in the physical-memory and five-target object matrices. + for _, op := range []string{"FSAVE", "FSTCW", "FSTENV", "FSTSW", "FLDCW", "FRSTOR"} { + forms = append(forms, fmt.Sprintf("%s 0(AX)(%s)", op, segment)) + } + for _, suffix := range []string{"B", "X", "F", "D", "W", "L", "V"} { + forms = append(forms, fmt.Sprintf("FMOV%s 0(AX)(%s),F0", suffix, segment), + fmt.Sprintf("FMOV%sP F0,0(AX)(%s)", suffix, segment)) + } + } + source.WriteString(strings.Join(forms, "\n") + "\n") + source.WriteString("RET\n") + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + for _, form := range forms { + t.Run("original-Go/"+form, func(t *testing.T) { + x86LEAActualTargetGoObject(t, target.arch, target.triple, "TEXT Bound(SB),4,$0-0\n"+form+"\nRET\n") + }) + } + if t.Failed() { + return // Do not claim LLVM equivalence for a rejected Go input. + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + file := mustParseX86RawReturn(t, source.String()) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, X87Mode: X87Software, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(api, func(t *testing.T) { + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, opt) + case "module": + module, callErr := TranslateModule(file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + default: + ctx := llvm.NewContext() + module, callErr := TranslateModuleInContext(ctx, file, opt) + err = callErr + if err == nil { + ir = module.String() + module.Dispose() + } + ctx.Dispose() + } + if err != nil { + t.Fatal(err) + } + for _, addressSpace := range []string{"ptr addrspace(256)", "ptr addrspace(257)"} { + if !strings.Contains(ir, "getelementptr i8, "+addressSpace) || !strings.Contains(ir, "load i8, "+addressSpace) { + t.Fatalf("environment/BCD memory lost %s", addressSpace) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + compileLLVMToObject(t, llc, target.triple, "named-x87.ll", "named-x87.o", ir) + }) + } + }) + } +} + +func TestX86PrivateDecoderNamedFLDENVStockGoDefect(t *testing.T) { + // Do not turn a positionless backend defect into a generic source-rejection + // certificate. Bind the entire one-instruction fixture and exact process + // outcome separately; raw D9 /4 is independently accepted above. + for _, target := range x86RawReturnTargets { + for _, segment := range []string{"FS", "GS"} { + t.Run(target.triple+"/"+segment, func(t *testing.T) { + original := "TEXT Bound(SB),4,$0-0\nFLDENV 0(AX)(" + segment + ")\nRET\n" + dir := t.TempDir() + input, object := filepath.Join(dir, "fldenv.s"), filepath.Join(dir, "fldenv.o") + if err := os.WriteFile(input, []byte(original), 0600); err != nil { + t.Fatal(err) + } + stage, cancelStage := runtimeOracleStageContext(t) + defer cancelStage() + ctx, cancel := context.WithTimeout(stage, 30*time.Second) + defer cancel() + driver, driverHash := x86LEAActualSDKDriver(t) + goos := "linux" + if strings.Contains(target.triple, "apple") { + goos = "darwin" + } else if strings.Contains(target.triple, "windows") { + goos = "windows" + } + command := exec.CommandContext(ctx, driver, "tool", "asm", "-p", "sourceframe", "-o", object, input) + command.Env = append(os.Environ(), "GOENV=off", "GOWORK=off", "GOTOOLCHAIN=local", + "GOOS="+goos, "GOARCH="+target.arch, "CGO_ENABLED=0") + output, err := runtimeOracleCombinedOutput(ctx, command) + const diagnostic = "asm: asmand: bad address \nasm: assembly failed\n" + if failure := x86UnnamedAsmidxOutcome(output, err, ctx.Err(), diagnostic); failure != nil { + t.Fatalf("unknown named FLDENV backend observation: %v\n%s", failure, output) + } + if goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), false, target.arch) == nil { + t.Fatal("positionless backend defect became a source-rejection certificate") + } + for _, changed := range []string{"", diagnostic + diagnostic, diagnostic + "unexpected\n", "asm: assembly failed\n"} { + if x86UnnamedAsmidxOutcome([]byte(changed), err, nil, diagnostic) == nil { + t.Fatal("changed/missing/repeated backend diagnostic accepted") + } + } + for _, changed := range []error{nil, errors.New("tool launch"), errors.Join(err, errors.New("cleanup"))} { + if x86UnnamedAsmidxOutcome(output, changed, nil, diagnostic) == nil { + t.Fatal("missing/unknown/cleanup process failure accepted") + } + } + if x86UnnamedAsmidxOutcome(output, err, context.DeadlineExceeded, diagnostic) == nil { + t.Fatal("expired backend observation accepted") + } + if data, readErr := os.ReadFile(input); readErr != nil || string(data) != original { + t.Fatal("original named FLDENV source changed", readErr) + } + t.Logf("GO_BACKEND_DEFECT not-source-exclusion target=%s source=%q sourceSHA=%x driverSHA=%s diagnosticSHA=%x", + target.triple, original, sha256.Sum256([]byte(original)), driverHash, sha256.Sum256(output)) + }) + } + } +} + +func TestX86PrivateDecoderUnboundSegmentCannotEmit(t *testing.T) { + // These are exactly the eighteen old parser-error controls: mode32 + // STOSB/STOSL/INSB/INSL and mode64 plus STOSQ, each with FS/GS. They + // remain unbound machine Context, not new public syntax or object passes. + for _, target := range x86RawReturnTargets { + forms := [][]byte{{0xaa}, {0xab}, {0x6c}, {0x6d}} + mode := 32 + if target.arch == "amd64" { + mode = 64 + forms = append(forms, []byte{0x48, 0xab}) + } + for _, opcode := range forms { + for _, segment := range []byte{0x64, 0x65} { + code := append([]byte{0x67, segment}, opcode...) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatal("complete segment observation", physical, err) + } + memory, ok := physical.Args[0].(x86asm.Mem) + if !ok || memory.Segment != x86asm.ES || physical.Prefix[1]&x86asm.PrefixIgnored != 0 { + t.Fatal("unbound prefix borrowed an ignored/default-segment identity", physical) + } + for _, api := range []string{"text", "module", "owned-module"} { + t.Run(fmt.Sprintf("%s/%x/%s", target.triple, code, api), func(t *testing.T) { + file := mustParseX86RawReturn(t, x86StaticReturnSource("Bound", code)+"RET\n") + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + var callErr error + switch api { + case "text": + ir, err := Translate(file, opt) + callErr = err + if ir != "" { + t.Fatal("unbound segment emitted text") + } + case "module": + module, err := TranslateModule(file, opt) + callErr = err + if !module.IsNil() { + module.Dispose() + t.Fatal("unbound segment emitted a module") + } + default: + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + callErr = err + if !module.IsNil() { + module.Dispose() + ctx.Dispose() + t.Fatal("unbound segment emitted an owned module") + } + ctx.Dispose() + } + if !errors.Is(callErr, ErrProbeNeedsContext) || !strings.Contains(callErr.Error(), "unbound physical") { + t.Fatalf("unbound segment lost Context: %v", callErr) + } + }) + } + } + } + } +} diff --git a/amd64_raw_pack.go b/amd64_raw_pack.go new file mode 100644 index 000000000..70ed38492 --- /dev/null +++ b/amd64_raw_pack.go @@ -0,0 +1,38 @@ +package plan9asm + +import "fmt" + +// Go 1.27's four VPACK rows share the same three-operand grammar. The two +// dword-to-word operations also permit EVEX 32-bit memory broadcast. +var x86RawPackedNarrowOps = map[[2]int]struct { + op Op + broadcast bool + evexW0 bool +}{ + {1, 0x6b}: {op: "VPACKSSDW", broadcast: true, evexW0: true}, + {1, 0x63}: {op: "VPACKSSWB"}, + {2, 0x2b}: {op: "VPACKUSDW", broadcast: true, evexW0: true}, + {1, 0x67}: {op: "VPACKUSWB"}, +} + +func decodedX86VectorPackedSaturatingNarrowInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + spec, known := x86RawPackedNarrowOps[[2]int{p.mapNumber, p.opcode}] + if !known { + return Instr{}, 0, false, nil + } + // Intel's VEX pack rows ignore W. EVEX dword inputs require W0, + // while EVEX word inputs also ignore W. Go happens to emit W0 for all; + // that chosen encoding is not the complete legal physical grammar. + if p.evex && spec.evexW0 && p.w { + return Instr{}, 0, true, fmt.Errorf("packed saturating narrow: unsupported W bit") + } + bits := 16 + if spec.broadcast { + bits = 32 + } + return decodedX86BinaryVectorOperands(code, p, mode, spec.op, bits) +} diff --git a/amd64_raw_pack_test.go b/amd64_raw_pack_test.go new file mode 100644 index 000000000..19cd3c822 --- /dev/null +++ b/amd64_raw_pack_test.go @@ -0,0 +1,171 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawPackedSaturatingNarrowGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawpackedforms(SB),4,$0-0\n") + for _, op := range []string{"PACKSSLW", "PACKSSWB", "PACKUSDW", "PACKUSWB"} { + if arch == "amd64" && op != "PACKUSDW" { + fmt.Fprintf(&source, "%s M1, M2\n%s (AX), M2\n", op, op) + } + fmt.Fprintf(&source, "%s X1, X2\n%s (AX), X2\n", op, op) + } + if arch == "amd64" { + for _, op := range []string{"VPACKSSDW", "VPACKSSWB", "VPACKUSDW", "VPACKUSWB"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "%s %s1, %s2, %s3\n", op, width, width, width) + fmt.Fprintf(&source, "%s 32(AX), %s2, %s3\n", op, width, width) + fmt.Fprintf(&source, "%s %s20, %s21, K1, %s22\n", op, width, width, width) + } + fmt.Fprintf(&source, "%s Z20, Z21, K7, Z22\n", op) + fmt.Fprintf(&source, "%s.Z (AX), Z21, K7, Z22\n", op) + if op == "VPACKSSDW" || op == "VPACKUSDW" { + fmt.Fprintf(&source, "%s.BCST.Z 64(AX), Z21, K7, Z22\n", op) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d = %s %v, want %s %v", index, got.Op, got.Args, want[index].Op, want[index].Args) + } + } + if len(code) == 0 || code[len(code)-1] != 0xc3 { + t.Fatal("Go assembler did not end the fixture with RET") + } + var raw strings.Builder + raw.WriteString("TEXT rawpackedforms(SB),4,$0-0\n") + for _, value := range code[:len(code)-1] { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + raw.WriteString("RET\n") + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{ + Goarch: arch, + TargetTriple: triple, + Sigs: map[string]FuncSig{"rawpackedforms": {Name: "rawpackedforms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-packed-saturating-narrow.ll", "raw-packed-saturating-narrow.o", ir) + } + }) + } +} + +func TestX86RawPackedSaturatingNarrowRegisterAxes(t *testing.T) { + for key, spec := range x86RawPackedNarrowOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, regs := range [][3]int{{0, 1, 2}, {7, 15, 8}, {16, 20, 31}, {31, 30, 29}} { + for _, evex := range []bool{false, true} { + if !evex && (width == "Z" || regs[0] >= 16 || regs[1] >= 16 || regs[2] >= 16) { + continue + } + var code []byte + if evex { + vectorBits := byte(strings.Index("XYZ", width)) + code = encodeX86EVEXPackedMultiplyLow(false, vectorBits, 7, true, regs[0], regs[1], regs[2]) + code[1] = code[1]&^0x0f | byte(key[0]) + code[4] = byte(key[1]) + } else { + code = encodeX86VEXPackedMultiplyLow(width == "Y", regs[0], regs[1], regs[2]) + code[1] = code[1]&^0x1f | byte(key[0]) + code[3] = byte(key[1]) + } + got, length, matched, err := decodedX86VectorPackedSaturatingNarrowInstruction(code, 64) + if !matched || err != nil || length != len(code) { + t.Fatalf("decode %x = %+v length=%d matched=%v err=%v", code, got, length, matched, err) + } + wantOp := spec.op + wantArgs := []string{ + fmt.Sprintf("%s%d", width, regs[2]), + fmt.Sprintf("%s%d", width, regs[1]), + } + if evex { + wantOp += ".Z" + wantArgs = append(wantArgs, "K7") + } + wantArgs = append(wantArgs, fmt.Sprintf("%s%d", width, regs[0])) + if got.Op != wantOp || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %v", code, got, wantOp, wantArgs) + } + for index, arg := range got.Args { + if arg.String() != wantArgs[index] { + t.Fatalf("decode %x arg %d = %s, want %s", code, index, arg.String(), wantArgs[index]) + } + } + } + } + } + } +} + +func TestX86RawPackedSaturatingNarrowRejectsReservedForms(t *testing.T) { + base := encodeX86EVEXPackedMultiplyLow(false, 2, 1, false, 2, 3, 4) + base[1] = base[1]&^0x0f | 2 + base[4] = 0x2b + bytePackBroadcast := append([]byte(nil), base...) + bytePackBroadcast[1] = bytePackBroadcast[1]&^0x0f | 1 + bytePackBroadcast[3] |= 0x10 + bytePackBroadcast[4] = 0x63 + for _, test := range []struct { + name string + code []byte + }{ + {name: "missing ModRM", code: base[:5]}, + {name: "address override", code: append([]byte{0x67}, base...)}, + {name: "wrong pp", code: replaceX86Byte(base, 2, base[2]&^3)}, + {name: "wrong W", code: replaceX86Byte(base, 2, base[2]|0x80)}, + {name: "missing fixed bit", code: replaceX86Byte(base, 2, base[2]&^4)}, + {name: "reserved vector length", code: replaceX86Byte(base, 3, base[3]&^0x60|0x60)}, + {name: "zero without mask", code: replaceX86Byte(base, 3, base[3]&^7|0x80)}, + {name: "broadcast register", code: replaceX86Byte(base, 3, base[3]|0x10)}, + {name: "byte pack broadcast", code: bytePackBroadcast}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86VectorPackedSaturatingNarrowInstruction(test.code, 64); !matched || err == nil { + t.Fatalf("invalid %x matched=%v error=%v", test.code, matched, err) + } + }) + } +} + +func replaceX86Byte(source []byte, index int, value byte) []byte { + copyOfSource := append([]byte(nil), source...) + copyOfSource[index] = value + return copyOfSource +} diff --git a/amd64_raw_packed_arithmetic_literal_test.go b/amd64_raw_packed_arithmetic_literal_test.go new file mode 100644 index 000000000..3c56ee17b --- /dev/null +++ b/amd64_raw_packed_arithmetic_literal_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "sort" + "strings" + "testing" +) + +func x86RawPackedArithmeticLiteral(opcode byte, evex bool, width int, broadcast bool) []byte { + properties := decodedX86PackedIntegerArithmeticOps[opcode] + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + code := encodeRawScalarMove(evex, 1, int(opcode), 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | 1 + code[2] &^= 0x80 + if evex && properties.laneBytes == 8 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = properties.laneBytes + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*29+11)) + } + return code +} + +func x86PackedArithmeticOpcodes() []byte { + opcodes := make([]byte, 0, len(decodedX86PackedIntegerArithmeticOps)) + for opcode := range decodedX86PackedIntegerArithmeticOps { + opcodes = append(opcodes, opcode) + } + sort.Slice(opcodes, func(i, j int) bool { return opcodes[i] < opcodes[j] }) + return opcodes +} + +func TestDecodeX86RawPackedArithmeticRIPDataCompleteFamily(t *testing.T) { + for _, opcode := range x86PackedArithmeticOpcodes() { + properties := decodedX86PackedIntegerArithmeticOps[opcode] + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && properties.laneBytes >= 4 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", properties.op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedArithmeticLiteral(opcode, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := properties.op + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawPackedArithmeticRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawPackedArithmeticLiteral(0xd4, true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedArithmeticRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, opcode := range x86PackedArithmeticOpcodes() { + properties := decodedX86PackedIntegerArithmeticOps[opcode] + evex := index%2 == 0 + width := 32 + if evex { + width = 64 + } + broadcast := evex && properties.laneBytes >= 4 + code := x86RawPackedArithmeticLiteral(opcode, evex, width, broadcast) + name := fmt.Sprintf("packedArithmeticLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if broadcast { + dataWidth = properties.laneBytes + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "packed-arithmetic-literal.ll", "packed-arithmetic-literal.o", ir) + }) + } +} diff --git a/amd64_raw_packed_compare.go b/amd64_raw_packed_compare.go new file mode 100644 index 000000000..87df27e3f --- /dev/null +++ b/amd64_raw_packed_compare.go @@ -0,0 +1,82 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +func decodedX86PackedIntegerCompareInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + var op Op + var spec amd64PackedIntegerCompareSpec + for name, candidate := range amd64PackedIntegerCompareSpecs { + if p.mapNumber == candidate.mapNumber && p.opcode == candidate.opcode { + op, spec = name, candidate + break + } + } + if op == "" { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed integer compare: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.vectorLength > 2 { + return fail("invalid prefix or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + // Intel XED leaves VEX.W and EVEX byte/word W unconstrained. D/Q + // EVEX requires W0/W1. MASK_R reserves both destination extension bits. + if p.evex && (!p.fixed || p.zero || p.r != 0 || spec.laneBits >= 32 && p.w != (spec.laneBits == 64)) { + return fail("invalid EVEX fixed, zeroing, destination or width field") + } + if p.broadcast && (spec.laneBits < 32 || code[p.modRM]>>6 == 3) { + return fail("broadcast requires D/Q memory source") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + width := []string{"X", "Y", "Z"}[p.vectorLength] + byteWidth := 16 << p.vectorLength + scale := byteWidth + if p.broadcast { + scale = spec.laneBits / 8 + op += ".BCST" + } + var source Operand + var size int + var err error + if p.evex { + source, size, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, size, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + dst := Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8)) + args := []Operand{source, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))}} + if p.evex { + dst = Reg(fmt.Sprintf("K%d", code[p.modRM]>>3&7)) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + } + args = append(args, Operand{Kind: OpReg, Reg: dst}) + parts := make([]string, len(args)) + for i, arg := range args { + parts[i] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), x86Encoded: true}, p.modRM + size, true, nil +} diff --git a/amd64_raw_packed_compare_literal_test.go b/amd64_raw_packed_compare_literal_test.go new file mode 100644 index 000000000..0a13620ab --- /dev/null +++ b/amd64_raw_packed_compare_literal_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawPackedCompareLiteral(op string, evex bool, width int, broadcast bool) []byte { + spec := amd64PackedIntegerCompareSpecs[Op(op)] + code := encodeRawScalarMove(evex, 1, spec.opcode, 0, 0, 0, 0, false) + code[1] = code[1]&0xf0 | byte(spec.mapNumber) + code[2] &^= 0x80 + if evex && spec.laneBits == 64 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*17+5)) + } + return code +} + +func TestDecodeX86RawPackedCompareRIPDataCompleteFamily(t *testing.T) { + for _, op := range x86PackedCompareTestOps { + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && amd64PackedIntegerCompareSpecs[Op(op)].laneBits >= 32 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawPackedCompareLiteral(op, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(op) + if broadcast { + want += ".BCST" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawPackedCompareRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawPackedCompareLiteral("VPCMPEQD", true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedCompareRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, op := range x86PackedCompareTestOps { + spec := amd64PackedIntegerCompareSpecs[Op(op)] + evex := index%2 == 0 + width := 32 + if evex { + width = 64 + } + broadcast := evex && spec.laneBits >= 32 + code := x86RawPackedCompareLiteral(op, evex, width, broadcast) + name := fmt.Sprintf("packedCompareLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "packed-compare-literal.ll", "packed-compare-literal.o", ir) + }) + } +} diff --git a/amd64_raw_packed_compare_test.go b/amd64_raw_packed_compare_test.go new file mode 100644 index 000000000..765e44643 --- /dev/null +++ b/amd64_raw_packed_compare_test.go @@ -0,0 +1,506 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86PackedCompareTestOps = []string{"VPCMPEQB", "VPCMPEQW", "VPCMPEQD", "VPCMPEQQ", "VPCMPGTB", "VPCMPGTW", "VPCMPGTD", "VPCMPGTQ"} + +func TestX86PackedCompareGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`(?s)\{as: A(\w+), ytab: _yvpcmpeqb, prefix: Pavx, op: opBytes\{(.*?)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 8 || len(rows) != len(amd64PackedIntegerCompareSpecs) { + t.Fatalf("Go has %d family entries, grammar has %d", len(rows), len(amd64PackedIntegerCompareSpecs)) + } + for _, row := range rows { + spec, ok := amd64PackedIntegerCompareSpecs[Op(row[1])] + if !ok { + t.Fatalf("missing Go family member %s", row[1]) + } + width := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[row[1][len(row[1])-1]] + if spec.laneBits != width || spec.greater != strings.Contains(row[1], "GT") { + t.Fatalf("incorrect semantics for %s: %+v", row[1], spec) + } + opcodes := regexp.MustCompile(`0x[0-9A-F]+`).FindAllString(row[2], -1) + if len(opcodes) != 5 { + t.Fatalf("Go %s has %d encodings", row[1], len(opcodes)) + } + for _, opcode := range opcodes { + if opcode != fmt.Sprintf("0x%X", spec.opcode) { + t.Fatalf("Go opcode %s != spec %+v", opcode, spec) + } + } + if strings.Contains(row[2], "0F38") != (spec.mapNumber == 2) || strings.Contains(row[2], "evexBcst") != (width >= 32) { + t.Fatalf("Go map/broadcast does not match %+v: %s", spec, row[2]) + } + } + table := regexp.MustCompile(`(?s)var _yvpcmpeqb = \[\]ytab\{(.*?)\n\}`).FindStringSubmatch(string(data)) + if len(table) != 2 { + t.Fatal("missing Go operand table") + } + args := regexp.MustCompile(`argList\{([^}]+)\}`).FindAllStringSubmatch(table[1], -1) + want := []string{"Yxm, Yxr, Yxr", "Yym, Yyr, Yyr", "YxmEvex, YxrEvex, Yk", "YxmEvex, YxrEvex, Yknot0, Yk", "YymEvex, YyrEvex, Yk", "YymEvex, YyrEvex, Yknot0, Yk", "Yzm, Yzr, Yk", "Yzm, Yzr, Yknot0, Yk"} + if len(args) != len(want) { + t.Fatalf("Go operand rows %d != %d", len(args), len(want)) + } + for i, arg := range args { + if arg[1] != want[i] { + t.Fatalf("Go row %d=%s, want %s", i, arg[1], want[i]) + } + } +} + +func TestX86RawPackedCompareEncodingAxes(t *testing.T) { + count := 0 + for _, op := range x86PackedCompareTestOps { + // The test's encoding map is independent of the implementation spec. + opcode := map[string]byte{"VPCMPEQB": 0x74, "VPCMPEQW": 0x75, "VPCMPEQD": 0x76, "VPCMPEQQ": 0x29, "VPCMPGTB": 0x64, "VPCMPGTW": 0x65, "VPCMPGTD": 0x66, "VPCMPGTQ": 0x37}[op] + mapNumber := byte(1) + if strings.HasSuffix(op, "Q") { + mapNumber = 2 + } + for _, evex := range []bool{false, true} { + limit, destLimit, widths := 16, 16, 2 + if evex { + limit, destLimit, widths = 32, 8, 3 + } + for width := 0; width < widths; width++ { + for src1 := 0; src1 < limit; src1++ { + for src2 := 0; src2 < limit; src2++ { + for dst := 0; dst < destLimit; dst++ { + code := encodeRawScalarMove(evex, 1, int(opcode), src2, src1, dst, 0, false) + code[1] = code[1]&0xe0 | mapNumber + if evex { + // Preserve R' (K destination extension) when replacing map. + code[1] |= 0x10 + code[2] &^= 0x80 + if mapNumber == 2 { + code[2] |= 0x80 + } + code[3] |= byte(width)<<5 | byte(dst) + } else { + code[2] |= byte(width) << 2 + } + variants := 1 + if !evex { + variants = 2 + } + for variant := 0; variant < variants; variant++ { + if variant != 0 { + code[2] |= 0x80 + } + got, size, matched, err := decodedX86PackedIntegerCompareInstruction(code, 64) + prefix := []string{"X", "Y", "Z"}[width] + args := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src2))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src1))}} + dest := Reg(fmt.Sprintf("%s%d", prefix, dst)) + if evex { + dest = Reg(fmt.Sprintf("K%d", dst)) + if dst != 0 { + args = append(args, Operand{Kind: OpReg, Reg: dest}) + } + } + args = append(args, Operand{Kind: OpReg, Reg: dest}) + if err != nil || !matched || size != len(code) || got.Op != Op(op) || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x: %+v %d %v %v; want %s %+v", code, got, size, matched, err, op, args) + } + _, _, matched, err = decodedX86PackedIntegerCompareInstruction(code, 32) + if !matched || (err != nil) != (src1 >= 8 || src2 >= 8 || !evex && dst >= 8) { + t.Fatalf("386 decode %x: matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + } + if count != 327680 { + t.Fatalf("covered %d encodings, want 327680", count) + } +} + +func TestX86RawPackedCompareInvalidAndMemoryForms(t *testing.T) { + valid := []byte{0x62, 0xf1, 0x75, 0x0b, 0x76, 0xe0} // VPCMPEQD X0,X1,K3,K4. + for name, mutate := range map[string]func([]byte){ + "fixed": func(b []byte) { b[2] &^= 4 }, "length": func(b []byte) { b[3] |= 0x60 }, + "zero": func(b []byte) { b[3] |= 0x80 }, "broadcast-register": func(b []byte) { b[3] |= 0x10 }, + "width": func(b []byte) { b[2] |= 0x80 }, "mask-dest-r": func(b []byte) { b[1] &^= 0x80 }, + "mask-dest-rprime": func(b []byte) { b[1] &^= 0x10 }, "prefix": func(b []byte) { b[2] &^= 3 }, + } { + b := append([]byte(nil), valid...) + mutate(b) + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted %s %x", name, b) + } + } + for _, b := range [][]byte{valid[:5], {0x62, 0xf1, 0x75, 0x0b, 0x76, 0x04}, {0x62, 0xf1, 0x75, 0x0b, 0x76, 0x40}, append([]byte{0x67}, valid...), {0x62, 0xf1, 0x75, 0x1b, 0x74, 0x00}, {0xc5, 0xf9, 0x74}, {0xc4, 0xe1, 0x79, 0x76, 0x05, 0, 0, 0, 0}} { + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted invalid %x", b) + } + } + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(valid, 16); !matched || err == nil { + t.Fatal("accepted mode16") + } + for _, b := range [][]byte{nil, {0x62}, {0x90}, {0xc5, 0xf9, 0x73, 0xc0}} { + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); matched || err != nil { + t.Fatalf("claimed unrelated %x", b) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, broadcast := range []bool{false, true} { + b := []byte{segment.prefix, 0x62, 0x91, 0x75, 0x4b, 0x76, 0x64, 0x88, 0xfe} + offset := int64(-128) + op := Op("VPCMPEQD") + if broadcast { + b[4] |= 0x10 + offset = -8 + op += ".BCST" + } + got, size, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64) + want := MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: offset} + if err != nil || !matched || size != len(b) || got.Op != op || !reflect.DeepEqual(got.Args[0].Mem, want) { + t.Fatalf("decode %x=%+v: %v", b, got, err) + } + } + } + // Byte/word EVEX.W is ignored, whereas dword and qword W is fixed. + for _, opcode := range []byte{0x74, 0x75, 0x64, 0x65} { + b := append([]byte(nil), valid...) + b[4] = opcode + b[2] |= 0x80 + if _, _, matched, err := decodedX86PackedIntegerCompareInstruction(b, 64); !matched || err != nil { + t.Fatalf("ignored W: %x %v", b, err) + } + } +} + +func TestX86RawPackedCompareGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 31 + } + var source strings.Builder + source.WriteString("TEXT compare(SB),4,$0-0\n") + for _, op := range x86PackedCompareTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, reg := range []int{0, last} { + inputs := []string{fmt.Sprintf("%s%d", width, last-reg), "(AX)", "-7(BX)(CX*4)", "64(SI)", "8192(DI)"} + for _, input := range inputs { + fmt.Fprintf(&source, "%s %s,%s%d,K0\n", op, input, width, reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s,%s%d,K7,K7\n", op, input, width, reg) + } + } + if strings.HasSuffix(op, "D") || strings.HasSuffix(op, "Q") { + for _, input := range []string{"(AX)", "-8(BX)(CX*4)", "64(SI)", "8192(DI)"} { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,K7\n", op, input, width, reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,K1,K0\n", op, input, width, reg) + } + } + } + } + if width != "Z" { + for _, input := range []string{width + "0", "(AX)", "-7(BX)(CX*4)", "8192(DI)"} { + fmt.Fprintf(&source, "%s %s,%s6,%s7\n", op, input, width, width) + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + got, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(got.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(got.Instrs), len(want)) + } + for i, instruction := range got.Instrs { + if instruction.Op != want[i].Op || !reflect.DeepEqual(instruction.Args, want[i].Args) { + t.Fatalf("instruction %d: %+v, want %+v", i, instruction, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "compare", code) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"compare": {Name: "compare", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "compare.ll", "compare.o", ir) + } + t.Logf("%s: %d actual Go encodings", arch, len(want)-1) + }) + } +} + +func TestX86PackedCompareGo386HighRegisters(t *testing.T) { + for _, op := range x86PackedCompareTestOps { + for _, width := range []string{"X", "Y"} { + for _, regs := range [][3]int{{8, 0, 1}, {0, 8, 1}, {0, 1, 8}, {15, 14, 13}} { + operands := fmt.Sprintf("%s%d,%s%d,%s%d", width, regs[0], width, regs[1], width, regs[2]) + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + t.Run(op+"/"+operands+"/"+triple, func(t *testing.T) { + source := "TEXT ·compare(SB),4,$0-0\n" + op + " " + operands + "\nRET\n" + // Preserve actual Go frontend acceptance, without treating + // those names as physical mode32 register identities. + x86LEAActualTargetGoObject(t, "386", triple, source) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86PackedIntegerCompareAPI(t, source, "func compare()\n", "386", triple, api) + if !errors.Is(err, ErrProbeNeedsContext) || ir != "" { + t.Fatalf("frontend high bank borrowed physical permission: IR=%d err=%v", len(ir), err) + } + }) + } + }) + } + } + } + } +} + +func TestX86PackedCompareGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86PackedCompareTestOps { + lines := []string{op + " X0,Y1,Y2", op + " Z0,Z1,Z2", op + " X16,X1,X2", op + " X0,X1,(AX)", op + ".Z X0,X1,K1", op + ".SAE X0,X1,K1", op + " X0,X1,K0,K2", op + ".BCST X0,X1,K2"} + if strings.HasSuffix(op, "B") || strings.HasSuffix(op, "W") { + lines = append(lines, op+".BCST (AX),X1,K2") + } + if arch == "386" { + lines = append(lines, op+" Z8,Z0,K1", op+" X0,X1,K1,K2") + } + for _, line := range lines { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(path, []byte("TEXT bad(SB),4,$0-0\n"+line+"\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + if out, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("Go %s accepted %s: %s", arch, line, out) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86PackedSignedGreaterThanRejected(t, arch, triple, line) + } + } + } +} + +func TestX86PackedCompareMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + cross := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !cross { + t.Skip("execution requires amd64 or Rosetta; five-target object tests remain required") + } + var source, decl strings.Builder + checks := map[string]*strings.Builder{"views": {}, "memory": {}} + sigs := map[string]FuncSig{} + index := 0 + for _, op := range x86PackedCompareTestOps { + bits := map[byte]int{'B': 8, 'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + greater := 0 + if strings.Contains(op, "GT") { + greater = 1 + } + for w, width := range []string{"X", "Y", "Z"} { + for mode := 0; mode < 5; mode++ { // VEX reg/mem; EVEX reg/mem/broadcast. + if mode < 2 && w == 2 || mode == 4 && bits < 32 { + continue + } + for alias := 0; alias < 2; alias++ { + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("compare%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\nVMOVUPS (AX),Z0\n", name) + input := "(BX)" + if mode == 0 || mode == 2 { + source.WriteString("VMOVUPS (BX),Z1\n") + input = width + "1" + } + dst := width + "0" + if alias == 1 { + dst = width + "1" + } + if mode < 2 { + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s %s,%s0,%s", op, input, width, dst), raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\n", alias, alias, alias) + } else { + dst = "K7" + if alias == 1 { + dst = "K1" + } + suffix := "" + if mode == 4 { + suffix = ".BCST" + } + source.WriteString("KMOVQ mask+24(FP),K1\n") + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s%s %s,%s0,K1,%s", op, suffix, input, width, dst), raw) + fmt.Fprintf(&source, "KMOVQ %s,(DI)\nVMOVUPS Z0,16(DI)\n", dst) + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&decl, "extern void %s(const void*,const void*,void*,uint64_t);\n", name) + kind := "views" + if mode >= 2 { + kind = "memory" + } + fmt.Fprintf(checks[kind], "if(check(%s,%d,%d,%d,%d)) {fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, 16< +#include +#include +` + x86TestGuardPagesC + decl.String() + ` +static uint64_t lane(const uint8_t *p,int bytes) {uint64_t v=0;memcpy(&v,p,bytes);return v;} +static int cmp(const uint8_t *a,const uint8_t *b,int bytes,int gt) { + uint64_t x=lane(a,bytes),y=lane(b,bytes),sign=UINT64_C(1)<<(bytes*8-1); + return gt ? (x^sign)>(y^sign) : x==y; +} +static int check(void(*fn)(const void*,const void*,void*,uint64_t),int width,int bytes,int gt,int mode) { + for(int n=0;n<64;n++) { + uint8_t ab[65],bb[65],out[128],want[112]={0};uint8_t *a=ab+1,*b=bb+1; + for(int i=0;i<64;i++) {a[i]=(uint8_t)(i*37+n*19);b[i]=(n%3==0)?a[i]:(uint8_t)(i*13-n*7);} + uint64_t mask=(n==0)?0:(n==1)?UINT64_C(1)<<63:(n==2)?UINT64_MAX:UINT64_C(0x5a5a01239876cdef)>>(n%32),expected=0; + int lanes=width/bytes; + if(mode<2) { + for(int i=0;i>i)&1) expected|=(uint64_t)cmp(a+i*bytes,b+(mode==4?0:i*bytes),bytes,gt)<=3 && !active)?NULL:b,out,mask); + if(lane(out,8)!=expected || memcmp(out+16,a,64)) {fprintf(stderr,"mask compare mismatch mode=%d width=%d bytes=%d n=%d\n",mode,width,bytes,n);return 1;} + } + } + if(mode>=3) { + // Only the enabled source lanes are accessible. This detects an eager + // full-width read even when the mask is nonzero, not just NULL/zero mask. + uint8_t a[64],out[128];memset(a,0,64); + int lanes=width/bytes; + for(int enabled=1;enabled<=lanes;enabled++) { + int readable=(mode==4)?bytes:enabled*bytes; + uint8_t *b=guard+page-readable;memset(b,0,(size_t)readable); + uint64_t mask=enabled==64?UINT64_MAX:(UINT64_C(1)< 16 { + sourcePrefix = "Y" + } + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for destination := 0; destination < 32; destination++ { + for source := 0; source < 32; source++ { + code := encodeX86EVEXPackedExtend(spec.opcode, byte(vectorBits), masking.mask, masking.zeroing, destination, source) + got, length, ok, err := decodedX86PackedExtendInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", sourcePrefix, source)} + if masking.mask != 0 { + wantArgs = append(wantArgs, fmt.Sprintf("K%d", masking.mask)) + } + wantArgs = append(wantArgs, fmt.Sprintf("%s%d", destinationPrefix, destination)) + if got.Op != spec.op+Op(masking.suffix) || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op+Op(masking.suffix), strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op+Op(masking.suffix), strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + } + if count != 110592 { + t.Fatalf("covered %d EVEX packed-extend register encodings, want 110592", count) + } +} + +func TestDecodedX86PackedExtendCompressedMemoryAndInvalidForms(t *testing.T) { + for _, spec := range rawPackedExtendTestSpecs { + for vectorBits, destinationPrefix := range []string{"X", "Y", "Z"} { + destinationBytes := 16 << vectorBits + sourceBytes := destinationBytes * spec.inputBits / spec.outputBits + code := encodeX86EVEXPackedExtend(spec.opcode, byte(vectorBits), 2, true, 21, 0) + code[5] = 0x68 + code = append(code, 0x7f) + got, length, ok, err := decodedX86PackedExtendInstruction(code, 64) + wantSource := fmt.Sprintf("%d(AX)", sourceBytes*127) + wantDestination := fmt.Sprintf("%s21", destinationPrefix) + if err != nil || !ok || length != len(code) || got.Op != spec.op+".Z" || len(got.Args) != 3 || got.Args[0].String() != wantSource || got.Args[1].String() != "K2" || got.Args[2].String() != wantDestination { + t.Fatalf("decode %x = %+v, length=%d ok=%v err=%v; want %s.Z %s, K2, %s", code, got, length, ok, err, spec.op, wantSource, wantDestination) + } + } + } + + valid := encodeX86VEXPackedExtend(0x32, false, 0, 0) + invalidVVVV := append([]byte(nil), valid...) + invalidVVVV[2] &^= 0x08 + for name, code := range map[string][]byte{ + "truncated": {0xc4, 0xe2}, + "wrong map": {0xc4, 0xe1, 0x79, 0x32, 0xc0}, + "wrong pp": {0xc4, 0xe2, 0x78, 0x32, 0xc0}, + "unrelated opcode": {0xc4, 0xe2, 0x79, 0x2f, 0xc0}, + } { + if instruction, _, matched, decodeErr := decodedX86PackedExtendInstruction(code, 64); matched || decodeErr != nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, code, instruction, matched, decodeErr) + } + } + // This family's VEX.W is ignored. It must not be confused with EVEX + // dword-to-qword W0, or rejected merely because Go emits canonical W0. + if ins, _, matched, err := decodedX86PackedExtendInstruction([]byte{0xc4, 0xe2, 0xf9, 0x32, 0xc0}, 64); !matched || err != nil || ins.Op != "VPMOVZXBQ" { + t.Fatalf("legal VEX.WIG form rejected: %+v/%v", ins, err) + } + if _, _, matched, decodeErr := decodedX86PackedExtendInstruction(invalidVVVV, 64); !matched || decodeErr == nil { + t.Fatalf("non-reserved VEX.vvvv encoding %x returned ok=%v err=%v", invalidVVVV, matched, decodeErr) + } + invalidBroadcast := encodeX86EVEXPackedExtend(0x32, 2, 1, false, 0, 0) + invalidBroadcast[3] |= 0x10 + invalidBroadcast[5] = 0x40 + invalidBroadcast = append(invalidBroadcast, 1) + if _, _, matched, decodeErr := decodedX86PackedExtendInstruction(invalidBroadcast, 64); !matched || decodeErr == nil { + t.Fatalf("EVEX broadcast returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_packed_float_move.go b/amd64_raw_packed_float_move.go new file mode 100644 index 000000000..8a2fed36d --- /dev/null +++ b/amd64_raw_packed_float_move.go @@ -0,0 +1,101 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's packed move grammars share width, direction and memory axes. The pinned +// generic decoder can misread VEX moves as legacy instructions or lose their +// vector width, so decode the complete float and VEX integer families here. +func decodedX86PackedMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedMoveInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedMoveInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + var op Op + store := false + switch p.opcode { + case 0x28, 0x29: + if p.pp > 1 { + return Instr{}, 0, false, nil + } + op = Op("VMOVA" + []string{"PS", "PD"}[p.pp]) + store = p.opcode == 0x29 + case 0x10, 0x11: + if p.pp > 1 { + return Instr{}, 0, false, nil + } + op = Op("VMOVU" + []string{"PS", "PD"}[p.pp]) + store = p.opcode == 0x11 + case 0x6f, 0x7f: + if p.evex || (p.pp != 1 && p.pp != 2) { + return Instr{}, 0, false, nil + } + op = Op([]string{"", "VMOVDQA", "VMOVDQU"}[p.pp]) + store = p.opcode == 0x7f + default: + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed float move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("vvvv and V' must be reserved") + } + if p.evex && (!p.fixed || p.w != (p.pp == 1)) { + return fail("invalid EVEX fixed or width bit") + } + if p.vectorLength > 2 || p.broadcast { + return fail("reserved vector length or broadcast bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + registerForm := code[p.modRM]>>6 == 3 + if p.zero && (p.mask == 0 || store && !registerForm) { + return fail("zeroing requires a nonzero mask and register destination") + } + width := []string{"X", "Y", "Z"}[p.vectorLength] + regNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && ((!p.evex || width == "Z") && regNumber >= 8) { + return fail("extended register unavailable in 32-bit mode") + } + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, regNumber))} + var rm Operand + var consumed int + var err error + if p.evex { + rm, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<= 0x78 && p.opcode <= 0x7b { + names := [...]Op{"VCVTTPS2UQQ", "VCVTPS2UQQ", "VCVTTPS2QQ", "VCVTPS2QQ"} + name := names[p.opcode-0x78] + if p.w { + name = Op(strings.Replace(string(name), "PS", "PD", 1)) + } + return name + } + switch { + case p.opcode == 0x5b && p.pp == 1 && !p.w: + return "VCVTPS2DQ" + case p.opcode == 0x5b && p.pp == 2 && !p.w: + return "VCVTTPS2DQ" + case p.opcode == 0xe6 && p.pp == 3 && p.w: + return "VCVTPD2DQ" + case p.opcode == 0xe6 && p.pp == 1 && p.w: + return "VCVTTPD2DQ" + case (p.opcode == 0x78 || p.opcode == 0x79) && p.pp == 0: + name := "VCVT" + if p.opcode == 0x78 { + name += "T" + } + if p.w { + return Op(name + "PD2UDQ") + } + return Op(name + "PS2UDQ") + case p.opcode == 0x7a && p.pp == 2 && !p.w: + return "VCVTUDQ2PD" + case p.opcode == 0x7a && p.pp == 3 && !p.w: + return "VCVTUDQ2PS" + case p.opcode == 0xe6 && p.pp == 2 && p.w: + return "VCVTQQ2PD" + case p.opcode == 0x7a && p.pp == 2 && p.w: + return "VCVTUQQ2PD" + case p.opcode == 0x5b && p.pp == 0 && p.w: + return "VCVTQQ2PS" + case p.opcode == 0x7a && p.pp == 3 && p.w: + return "VCVTUQQ2PS" + } + return "" +} + +func decodedX86PackedNumericConversionInstruction(code []byte, mode int) (Instr, int, bool, error) { + return decodedX86PackedNumericConversionInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86PackedNumericConversionInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched { + return Instr{}, 0, false, nil + } + name := x86RawPackedNumericName(p) + if name == "" { + return Instr{}, 0, false, nil + } + fail := func(reason string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed numeric conversion: %s", reason) + } + if mode != 32 && mode != 64 || p.addressOverride || p.upper != 0 || + p.evex && (!p.fixed || p.zero && p.mask == 0) || len(code) <= p.modRM { + return fail("invalid physical mode, prefix, reserved field, or truncated ModRM") + } + register := code[p.modRM]>>6 == 3 + embedded := p.evex && p.broadcast && register + vl := p.vectorLength + if embedded { + vl = 2 + } + if vl > 2 || !p.evex && vl > 1 { + return fail("reserved vector length") + } + spec, _ := x86PackedConversionSpecification(name) + if spec.sourceBits > spec.destinationBits && vl < 2 { + name += [...]Op{"X", "Y"}[vl] + spec, _ = x86PackedConversionSpecification(name) + } + shape := spec.shapes[0] + if len(spec.shapes) > 1 { + shape = spec.shapes[vl] + } + if embedded && spec.explicit == "" { + return fail("embedded rounding/SAE absent from this conversion") + } + width := shape.lanes * spec.sourceBits / 8 + if p.broadcast && !register { + width = spec.sourceBits / 8 + } + prefix := map[int]string{16: "X", 32: "Y", 64: "Z"} + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = memory.evex(code[p.modRM:], mode, p.b, p.x, p.segment, prefix[shape.sourceBytes], width) + } else { + source, consumed, err = memory.vex(code[p.modRM:], mode, p.b, p.x, p.segment, prefix[shape.sourceBytes]) + } + if err != nil { + return Instr{}, 0, true, err + } + if embedded { + if spec.explicit == "sae" { + name += ".SAE" // LL is ignored for SAE-only register encodings. + } else { + name += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } else if p.broadcast { + name += ".BCST" + } + if p.zero { + name += ".Z" + } + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix[shape.destinationBytes], int(code[p.modRM]>>3&7)+p.r*8))}) + ins := Instr{Op: name, Args: args, x86Encoded: true, x86PackedNumericEVEX: p.evex} + arch := "amd64" + if mode == 32 { + arch = "386" + } + shapeInput := ins + if memory.physicalPC && source.Kind == OpMem && source.Mem.Base == "RIP" { + // The physical decoder has already checked the complete disp32. + // Check the conversion's memory class, not a Go source RIP spelling. + // Return the original physical operand; only a boundary observer may + // request this mode and it discards all decoded operands. + shapeInput.Args = append([]Operand(nil), ins.Args...) + shapeInput.Args[0] = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + } + _, _, err = x86PackedConversionProperties(shapeInput, arch) + if err != nil { + return Instr{}, 0, true, err + } + printed := make([]string, len(args)) + for index, operand := range args { + printed[index] = operand.String() + } + ins.Raw = string(name) + " " + strings.Join(printed, ", ") + return ins, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_precision_convert.go b/amd64_raw_packed_precision_convert.go new file mode 100644 index 000000000..6603dbbe5 --- /dev/null +++ b/amd64_raw_packed_precision_convert.go @@ -0,0 +1,128 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Opcode 5A with no prefix or 66 converts packed singles to doubles or +// doubles to singles. Go 1.27 exposes the VEX.128/.256 and EVEX.128/.256/.512 +// forms under VCVTPS2PD and VCVTPD2PSX/Y (or unsuffixed Z). Scalar F2/F3 5A +// belongs to the separate scalar-precision grammar. +func decodedX86PackedPrecisionConvertInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || p.opcode != 0x5a || (p.pp != 0 && p.pp != 1) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed precision conversion: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("reserved vvvv/V' field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.evex { + if !p.fixed || p.w != (p.pp == 1) || p.zero && p.mask == 0 { + return fail("invalid EVEX fixed, width, mask or zeroing field") + } + } + + registerSource := code[p.modRM]>>6 == 3 + if mode == 32 { + if !p.evex && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended VEX register in 32-bit mode") + } + if p.evex && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + } + embeddedControl := p.evex && p.broadcast && registerSource + broadcast := p.evex && p.broadcast && !registerSource + if p.evex && !embeddedControl && p.vectorLength > 2 { + return fail("reserved EVEX vector length") + } + + vectorLength := p.vectorLength + if embeddedControl { + vectorLength = 2 // SAE or rounding implies the 512-bit register form. + } + var op Op + var sourcePrefix, destinationPrefix string + var memoryScale int + if p.pp == 0 { + op = "VCVTPS2PD" + destinationPrefix = [...]string{"X", "Y", "Z"}[vectorLength] + sourcePrefix = "X" + if vectorLength == 2 { + sourcePrefix = "Y" + } + memoryScale = 8 << vectorLength + } else { + op = "VCVTPD2PS" + if vectorLength < 2 { + op += [...]Op{"X", "Y"}[vectorLength] + } + sourcePrefix = [...]string{"X", "Y", "Z"}[vectorLength] + destinationPrefix = "X" + if vectorLength == 2 { + destinationPrefix = "Y" + } + memoryScale = 16 << vectorLength + } + if mode == 32 { + if sourcePrefix == "Z" && registerSource && (p.b != 0 || p.x != 0) { + return fail("Z8-Z31 are not Go assembler source registers in 32-bit mode") + } + if destinationPrefix == "Z" && p.r != 0 { + return fail("Z8-Z31 are not Go assembler destination registers in 32-bit mode") + } + } + if embeddedControl { + if p.pp == 0 { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } else if broadcast { + op += ".BCST" + if p.pp == 0 { + memoryScale = 4 + } else { + memoryScale = 8 + } + } + if p.zero { + op += ".Z" + } + + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix, memoryScale) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, sourcePrefix) + } + if err != nil { + return Instr{}, 0, true, err + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", destinationPrefix, int(code[p.modRM]>>3&7)+p.r*8))} + args := []Operand{source} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_precision_convert_test.go b/amd64_raw_packed_precision_convert_test.go new file mode 100644 index 000000000..ebf22488c --- /dev/null +++ b/amd64_raw_packed_precision_convert_test.go @@ -0,0 +1,316 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" +) + +var x86RawPackedPrecisionForms = []struct { + op Op + pp byte + length byte + evex bool + src Reg + dst Reg +}{ + {"VCVTPS2PD", 0, 0, false, "X1", "X0"}, + {"VCVTPS2PD", 0, 1, false, "X1", "Y0"}, + {"VCVTPS2PD", 0, 0, true, "X1", "X0"}, + {"VCVTPS2PD", 0, 1, true, "X1", "Y0"}, + {"VCVTPS2PD", 0, 2, true, "Y1", "Z0"}, + {"VCVTPD2PSX", 1, 0, false, "X1", "X0"}, + {"VCVTPD2PSY", 1, 1, false, "Y1", "X0"}, + {"VCVTPD2PSX", 1, 0, true, "X1", "X0"}, + {"VCVTPD2PSY", 1, 1, true, "Y1", "X0"}, + {"VCVTPD2PS", 1, 2, true, "Z1", "Y0"}, +} + +func rawPackedPrecisionBytes(pp, length byte, evex, memory bool) []byte { + modRM := byte(0xc1) + if memory { + modRM = 0x08 + } + if evex { + w := byte(0) + if pp == 1 { + w = 0x80 + } + return []byte{0x62, 0xf1, 0x7c | pp | w, 0x08 | length<<5, 0x5a, modRM} + } + return []byte{0xc5, 0xf8 | pp | length<<2, 0x5a, modRM} +} + +func TestX86RawPackedPrecisionConvertArrowRegression(t *testing.T) { + // Arrow emits this VEX.256 VCVTPS2PD as raw bytes. x/arch v0.14 + // incorrectly starts with a legacy POP rather than decoding the vector op. + code := []byte{0xc5, 0xfc, 0x5a, 0x04, 0xba} + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != "VCVTPS2PD" { + t.Fatalf("raw %#x decoded as %+v", code, decoded.Instrs) + } +} + +func TestX86RawPackedPrecisionConvertCompleteGo127BaseForms(t *testing.T) { + if len(x86RawPackedPrecisionForms) != 10 { + t.Fatal("Go 1.27 packed 5A table requires ten VEX/EVEX width forms") + } + for _, form := range x86RawPackedPrecisionForms { + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s/LL=%d/EVEX=%t/memory=%t", form.op, form.length, form.evex, memory) + t.Run(name, func(t *testing.T) { + code := rawPackedPrecisionBytes(form.pp, form.length, form.evex, memory) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != form.op { + t.Fatalf("raw %#x decoded as %+v; want %s", code, decoded.Instrs, form.op) + } + args := decoded.Instrs[0].Args + if len(args) != 2 { + t.Fatalf("raw %#x operands = %+v", code, args) + } + if memory { + if args[0].Kind != OpMem || args[0].Mem.Base != AX { + t.Fatalf("raw %#x memory source = %+v", code, args[0]) + } + } else if args[0].Kind != OpReg || args[0].Reg != form.src { + t.Fatalf("raw %#x source = %+v; want %s", code, args[0], form.src) + } + wantDestination := form.dst + if memory { + wantDestination = Reg(string(form.dst[:1]) + "1") + } + if args[1].Reg != wantDestination { + t.Fatalf("raw %#x destination = %s; want %s", code, args[1].Reg, wantDestination) + } + }) + } + } +} + +func TestX86RawPackedPrecisionConvertEncodingAxes(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + op Op + args []string + }{ + {"VEX-W-ignored", []byte{0xc4, 0xe1, 0xfc, 0x5a, 0xc1}, "VCVTPS2PD", []string{"X1", "Y0"}}, + {"masked", []byte{0x62, 0xf1, 0x7c, 0x09, 0x5a, 0xc1}, "VCVTPS2PD", []string{"X1", "K1", "X0"}}, + {"broadcast-zero", []byte{0x62, 0xf1, 0x7c, 0x99, 0x5a, 0x00}, "VCVTPS2PD.BCST.Z", []string{"0(AX)", "K1", "X0"}}, + {"single-to-double-SAE", []byte{0x62, 0xf1, 0x7c, 0x18, 0x5a, 0xc1}, "VCVTPS2PD.SAE", []string{"Y1", "Z0"}}, + {"double-to-single-round-down", []byte{0x62, 0xf1, 0xfd, 0x39, 0x5a, 0xd8}, "VCVTPD2PS.RD_SAE", []string{"Z0", "K1", "Y3"}}, + {"extended-registers", []byte{0x62, 0xa1, 0x7c, 0x08, 0x5a, 0xc1}, "VCVTPS2PD", []string{"X17", "X16"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86PackedPrecisionConvertInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) || got.Op != tc.op { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s", tc.code, got, length, recognized, err, tc.op) + } + if len(got.Args) != len(tc.args) { + t.Fatalf("decode %#x operands = %+v; want %v", tc.code, got.Args, tc.args) + } + for index, arg := range got.Args { + if arg.String() != tc.args[index] { + t.Fatalf("decode %#x operand %d = %s; want %s", tc.code, index, arg.String(), tc.args[index]) + } + } + }) + } + for _, tc := range []struct { + name string + code []byte + off int64 + }{ + {"single-Z-disp8", []byte{0x62, 0xf1, 0x7c, 0x48, 0x5a, 0x40, 0x02}, 64}, + {"double-Y-disp8", []byte{0x62, 0xf1, 0xfd, 0x28, 0x5a, 0x40, 0x02}, 64}, + {"double-broadcast-disp8", []byte{0x62, 0xf1, 0xfd, 0x38, 0x5a, 0x40, 0x02}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + got, _, recognized, err := decodedX86PackedPrecisionConvertInstruction(tc.code, 64) + if !recognized || err != nil || got.Args[0].Kind != OpMem || got.Args[0].Mem.Off != tc.off { + t.Fatalf("decode %#x = %+v, %v, %v; want offset %d", tc.code, got, recognized, err, tc.off) + } + }) + } + for _, code := range [][]byte{ + {0x62, 0xa1, 0x7c, 0x08, 0x5a, 0xc1}, // X17 -> X16. + {0x62, 0xa1, 0xfd, 0x28, 0x5a, 0xc1}, // Y17 -> X16. + } { + if got, _, recognized, err := decodedX86PackedPrecisionConvertInstruction(code, 32); !recognized || err != nil || len(got.Args) != 2 { + t.Fatalf("386 EVEX high X/Y register %#x = %+v, %v, %v", code, got, recognized, err) + } + } + requireX86GoAssemblerResult(t, "386", "TEXT high(SB),4,$0-0\n\tVCVTPS2PD X17, K1, X16\n\tVCVTPD2PSY Y17, K1, X16\n\tRET\n", true) +} + +func TestX86RawPackedPrecisionConvertRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"VEX-vvvv", []byte{0xc5, 0xe8, 0x5a, 0xc1}, 64}, + {"EVEX-vvvv", []byte{0x62, 0xf1, 0x6c, 0x08, 0x5a, 0xc1}, 64}, + {"EVEX-zero-without-mask", []byte{0x62, 0xf1, 0x7c, 0x88, 0x5a, 0xc1}, 64}, + {"EVEX-LL3-without-control", []byte{0x62, 0xf1, 0x7c, 0x68, 0x5a, 0xc1}, 64}, + {"EVEX-W-mismatch", []byte{0x62, 0xf1, 0xfc, 0x08, 0x5a, 0xc1}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xf8, 0x5a, 0xc1}, 64}, + {"missing-ModRM", []byte{0xc5, 0xf8, 0x5a}, 64}, + {"extended-386", []byte{0xc4, 0x61, 0x78, 0x5a, 0xc1}, 32}, + {"Z-destination-386", []byte{0x62, 0xe1, 0x7c, 0x48, 0x5a, 0xc1}, 32}, + {"Z-source-386", []byte{0x62, 0xb1, 0xfd, 0x48, 0x5a, 0xc1}, 32}, + {"extended-memory-386", []byte{0x62, 0xb1, 0x7c, 0x08, 0x5a, 0x01}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86PackedPrecisionConvertInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } +} + +func TestX86RawPackedPrecisionConvertCompilesEveryBaseForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawprecisionconversion(SB),4,$0-0\n") + encodings := make([][]byte, 0, len(x86RawPackedPrecisionForms)+5) + for _, form := range x86RawPackedPrecisionForms { + encodings = append(encodings, rawPackedPrecisionBytes(form.pp, form.length, form.evex, false)) + } + encodings = append(encodings, + []byte{0x62, 0xf1, 0x7c, 0x99, 0x5a, 0x00}, // PS broadcast, mask, zero. + []byte{0x62, 0xf1, 0x7c, 0x18, 0x5a, 0xc1}, // PS suppress exceptions. + []byte{0x62, 0xf1, 0xfd, 0x39, 0x5a, 0xd8}, // PD embedded rounding. + []byte{0x62, 0xf1, 0xfd, 0x38, 0x5a, 0x40, 0x02}, // PD broadcast disp8. + []byte{0x62, 0xa1, 0x7c, 0x08, 0x5a, 0xc1}, // EVEX high X in 386/amd64. + ) + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawprecisionconversion": {Name: "rawprecisionconversion", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-precision-conversion.ll", "raw-precision-conversion.o", ir) + }) + } +} + +func TestX86RawPackedPrecisionConvertRuntimeSemantics(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("runtime execution test only runs on amd64 host") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + const namedSource = ` +TEXT packedprecision(SB),4,$0-24 + MOVQ out+0(FP), AX + MOVQ singles+8(FP), BX + MOVQ doubles+16(FP), CX + VCVTPS2PD (BX), Y1 + VMOVUPD Y1, (AX) + VCVTPD2PSY (CX), X2 + VMOVUPS X2, 32(AX) + VZEROUPPER + RET +` + rawSource := strings.NewReplacer( + "VCVTPS2PD (BX), Y1", "BYTE $0xc5; BYTE $0xfc; BYTE $0x5a; BYTE $0x0b", + "VCVTPD2PSY (CX), X2", "BYTE $0xc5; BYTE $0xfd; BYTE $0x5a; BYTE $0x11", + ).Replace(namedSource) + requireX86GoAssemblerResult(t, "amd64", namedSource, true) + requireX86GoAssemblerResult(t, "amd64", rawSource, true) + + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + runPrefix = []string{"/usr/bin/arch", "-x86_64"} + } + options := Options{ + TargetTriple: triple, + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "packedprecision": { + Name: "packedprecision", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }, + }, + } + const mainC = ` +#include +#include +extern void packedprecision(uint8_t *, const float *, const double *); +int main(void) { + const float singles[4] = {1.5f, -2.25f, 3.0f, 4.0f}; + const double doubles[4] = {1.5, -2.25, 3.0, 4.0}; + uint8_t out[48]; + double widened[4]; + float narrowed[4]; + memset(out, 0xcc, sizeof(out)); + packedprecision(out, singles, doubles); + memcpy(widened, out, sizeof(widened)); + memcpy(narrowed, out + 32, sizeof(narrowed)); + for (int i = 0; i < 4; i++) { + if (widened[i] != (double)singles[i]) return 10 + i; + if (narrowed[i] != (float)doubles[i]) return 20 + i; + } + return 0; +} +` + for _, tc := range []struct{ name, source string }{ + {"named", namedSource}, + {"raw", rawSource}, + } { + t.Run(tc.name, func(t *testing.T) { + file, err := Parse(ArchAMD64, tc.source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, options) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "packed_precision_"+tc.name, triple, ir, mainC, runPrefix) + }) + } +} diff --git a/amd64_raw_packed_sign.go b/amd64_raw_packed_sign.go new file mode 100644 index 000000000..3f5ff84fd --- /dev/null +++ b/amd64_raw_packed_sign.go @@ -0,0 +1,44 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's _yvaddsubpd table gives all three VPSIGN instructions exactly +// the same VEX.128/VEX.256 register-and-memory operand grammar. +var x86RawPackedSignOps = [...]Op{"VPSIGNB", "VPSIGNW", "VPSIGND"} + +func decodedX86RawPackedSignInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.opcode < 0x08 || p.opcode > 0x0a { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed sign: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.pp != 1 || p.w || p.addressOverride { + return fail("Go's packed sign table only permits VEX.66.0F38.W0 without address override") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + width := [...]string{"X", "Y"}[p.vectorLength] + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (destinationNumber >= 8 || p.upper >= 8) { + return fail("extended vector register unavailable in 32-bit mode") + } + source, consumed, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + data := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))} + op := x86RawPackedSignOps[p.opcode-0x08] + args := []Operand{source, data, destination} + printed := []string{source.String(), data.String(), destination.String()} + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_sign_test.go b/amd64_raw_packed_sign_test.go new file mode 100644 index 000000000..8aa726316 --- /dev/null +++ b/amd64_raw_packed_sign_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawPackedSignGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawpackedsign(SB),4,$0-0\n") + for _, op := range []string{"VPSIGNB", "VPSIGNW", "VPSIGND"} { + for _, width := range []string{"X", "Y"} { + fmt.Fprintf(&source, "%s %s2, %s3, %s4\n", op, width, width, width) + fmt.Fprintf(&source, "%s 24(BX), %s3, %s4\n", op, width, width) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s12, %s13, %s14\n", op, width, width, width) + fmt.Fprintf(&source, "%s -32(R8), %s13, %s14\n", op, width, width) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } + }) + } +} + +func TestX86RawPackedSignRejectsFormsOutsideGoTable(t *testing.T) { + const source = "TEXT rawpackedsign(SB),4,$0-0\nVPSIGNB Y2, Y3, Y4\nRET\n" + assembled := assembleX87ControlBytes(t, "amd64", source) + if len(assembled) < 6 || assembled[0] != 0xc4 { + t.Fatalf("Go did not emit a VEX packed sign: %x", assembled) + } + valid := assembled[:len(assembled)-1] + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"wrong prefix", func(code []byte) []byte { code[2] &^= 3; return code }}, + {"wrong width", func(code []byte) []byte { code[2] |= 0x80; return code }}, + {"address override", func(code []byte) []byte { return append([]byte{0x67}, code...) }}, + {"missing ModRM", func(code []byte) []byte { return code[:4] }}, + {"high register on 386", func(code []byte) []byte { code[1] &^= 0x80; return code }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), valid...)) + mode := 64 + if test.name == "high register on 386" { + mode = 32 + } + if _, _, recognized, err := decodedX86RawPackedSignInstruction(code, mode); !recognized || err == nil { + t.Fatalf("invalid %x: recognized=%v err=%v", code, recognized, err) + } + }) + } +} + +func TestTranslateX86RawPackedSignLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, form := range []string{ + "VPSIGNB Y2, Y3, Y4", + "VPSIGNW X2, X3, X4", + "VPSIGND 16(BX), Y3, Y4", + } { + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(form+"/"+target.triple, func(t *testing.T) { + source := "TEXT rawpackedsign(SB),4,$0-0\n" + form + "\nRET\n" + code := assembleX87ControlBytes(t, target.goarch, source) + var raw strings.Builder + raw.WriteString("TEXT rawpackedsign(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(form, "(") { + file = x86RawUnprovedReturnFormProbeFile(t, target.goarch, "rawpackedsign", code) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawpackedsign": {Name: "rawpackedsign", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-sign.ll", "raw-packed-sign.o", ir) + }) + } + } +} diff --git a/amd64_raw_packed_testmask.go b/amd64_raw_packed_testmask.go new file mode 100644 index 000000000..e665384c1 --- /dev/null +++ b/amd64_raw_packed_testmask.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27's _yvpshufbitqmb forms are the product of test polarity (66/F3), +// B/W versus D/Q opcode, and EVEX.W. Each row has X/Y/Z widths and an +// optional K write-mask; only D/Q permits scalar-memory broadcast. +var x86RawPackedTestMaskOps = [2][2][2]Op{ + { + {"VPTESTMB", "VPTESTMW"}, + {"VPTESTMD", "VPTESTMQ"}, + }, + { + {"VPTESTNMB", "VPTESTNMW"}, + {"VPTESTNMD", "VPTESTNMQ"}, + }, +} + +func decodedX86RawPackedTestMaskInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || !p.evex || p.mapNumber != 2 || + (p.pp != 1 && p.pp != 2) || (p.opcode != 0x26 && p.opcode != 0x27) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("packed test mask: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.vectorLength > 2 || p.r != 0 || p.zero { + return fail("invalid reserved EVEX encoding bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + registerSource := modRM>>6 == 3 + if p.broadcast && (registerSource || p.opcode != 0x27) { + return fail("broadcast requires a D/Q scalar-memory source") + } + if mode == 32 && !registerSource && (p.b != 0 || p.x != 0) { + return fail("extended memory address in 32-bit mode") + } + + polarity := p.pp - 1 + widthGroup := p.opcode - 0x26 + w := 0 + if p.w { + w = 1 + } + op := x86RawPackedTestMaskOps[polarity][widthGroup][w] + laneBits := amd64PackedTestMaskSpecs[string(op)].laneBits + vectorPrefix := [...]string{"X", "Y", "Z"}[p.vectorLength] + accessBytes := 16 << p.vectorLength + if p.broadcast { + accessBytes = laneBits / 8 + op += ".BCST" + } + first, consumed, err := decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, vectorPrefix, accessBytes) + if err != nil { + return Instr{}, 0, true, err + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, p.upper))} + args := []Operand{first, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + destination := modRM >> 3 & 7 + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", destination))}) + printed := make([]string, len(args)) + for index, arg := range args { + printed[index] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: true, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_packed_testmask_test.go b/amd64_raw_packed_testmask_test.go new file mode 100644 index 000000000..6d26bb3a1 --- /dev/null +++ b/amd64_raw_packed_testmask_test.go @@ -0,0 +1,195 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, first, second, destination, writeMask int, memory, broadcast bool) []byte { + if memory { + first = 0 + } + p0 := byte(1<<7 | (1-first/16)<<6 | (1-first/8&1)<<5 | 1<<4 | 2) + p1 := byte(4 | (polarity + 1) | (^second&15)<<3) + if w != 0 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | (1-second/16)<<3 | writeMask) + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | destination<<3 | first&7) + if memory { + modRM = byte(0x40 | destination<<3) + } + code := []byte{0x62, p0, p1, p2, byte(0x26 + widthGroup), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestTranslateRawPackedTestMaskGorseRegression(t *testing.T) { + const source = `TEXT rawPackedTestMask(SB), $0-0 + LONG $0x4875f262 + WORD $0xc026 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawPackedTestMask": {Name: "rawPackedTestMask", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "icmp ne") { + t.Fatalf("raw VPTESTMB omitted per-lane mask test:\n%s", ir) + } +} + +func TestDecodedX86RawPackedTestMaskCompleteGoRows(t *testing.T) { + if len(amd64PackedTestMaskSpecs) != 8 { + t.Fatalf("Go 1.27 packed test-mask family has %d members, want eight", len(amd64PackedTestMaskSpecs)) + } + seen := make(map[Op]bool) + for polarity, groups := range x86RawPackedTestMaskOps { + for widthGroup, widths := range groups { + for w, op := range widths { + spec, ok := amd64PackedTestMaskSpecs[string(op)] + if !ok || seen[op] || spec.testZero != (polarity != 0) || + spec.laneBits != (8<<(widthGroup*2+w)) { + t.Fatalf("packed test-mask row %d/%d/%d = %s, spec %+v", polarity, widthGroup, w, op, spec) + } + seen[op] = true + for vectorBits, prefix := range [...]string{"X", "Y", "Z"} { + for _, memory := range []bool{false, true} { + for _, writeMask := range []int{0, 3} { + for _, broadcast := range []bool{false, true} { + if broadcast && (!memory || widthGroup == 0) { + continue + } + code := encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, 20, 21, 2, writeMask, memory, broadcast) + got, length, recognized, err := decodedX86RawPackedTestMaskInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x = %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := op + if broadcast { + wantOp += ".BCST" + } + if got.Op != wantOp || got.Args[1].String() != fmt.Sprintf("%s21", prefix) || + got.Args[len(got.Args)-1].String() != "K2" { + t.Fatalf("decode %x = %+v, want %s %s21, K2", code, got, wantOp, prefix) + } + if writeMask != 0 && (len(got.Args) != 4 || got.Args[2].String() != "K3") || + writeMask == 0 && len(got.Args) != 3 { + t.Fatalf("decode %x = %+v, lost write mask", code, got) + } + first := got.Args[0] + if memory { + wantBytes := 16 << vectorBits + if broadcast { + wantBytes = spec.laneBits / 8 + } + if first.Kind != OpMem || first.Mem.Base != AX || first.Mem.Off != int64(wantBytes) { + t.Fatalf("decode %x = %+v, want compressed displacement %d", code, got, wantBytes) + } + } else if first.String() != fmt.Sprintf("%s20", prefix) { + t.Fatalf("decode %x = %+v, want %s20", code, got, prefix) + } + } + } + } + } + } + } + } + if len(seen) != len(amd64PackedTestMaskSpecs) { + t.Fatalf("decoded %d distinct packed test-mask ops, want %d", len(seen), len(amd64PackedTestMaskSpecs)) + } +} + +func TestDecodedX86RawPackedTestMaskRejectsReservedBits(t *testing.T) { + base := encodeX86RawPackedTestMask(0, 0, 0, 2, 1, 2, 3, 0, false, false) + for _, test := range []struct { + name string + code []byte + }{ + {"fixed bit", []byte{base[0], base[1], base[2] &^ 4, base[3], base[4], base[5]}}, + {"K register extension", []byte{base[0], base[1] &^ 0x80, base[2], base[3], base[4], base[5]}}, + {"reserved length", []byte{base[0], base[1], base[2], base[3] | 0x20, base[4], base[5]}}, + {"zeroing", []byte{base[0], base[1], base[2], base[3] | 0x80, base[4], base[5]}}, + {"register broadcast", []byte{base[0], base[1], base[2], base[3] | 0x10, base[4], base[5]}}, + {"missing ModRM", base[:5]}, + {"address override", append([]byte{0x67}, base...)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawPackedTestMaskInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + byteBroadcast := encodeX86RawPackedTestMask(0, 0, 0, 2, 0, 1, 2, 0, true, true) + if _, _, recognized, err := decodedX86RawPackedTestMaskInstruction(byteBroadcast, 64); !recognized || err == nil { + t.Fatalf("accepted byte broadcast %x", byteBroadcast) + } +} + +func TestTranslateRawPackedTestMaskLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedTestMaskFamily(SB), $0-0\n") + for polarity, groups := range x86RawPackedTestMaskOps { + for widthGroup, widths := range groups { + for w := range widths { + for vectorBits := 0; vectorBits < 3; vectorBits++ { + for _, code := range [][]byte{ + encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, 1, 2, 3, 0, false, false), + encodeX86RawPackedTestMask(polarity, widthGroup, w, vectorBits, 0, 2, 3, 4, true, widthGroup != 0), + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawPackedTestMaskFamily": {Name: "rawPackedTestMaskFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-test-mask.ll", "raw-packed-test-mask.o", ir) + }) + } +} diff --git a/amd64_raw_packed_widen_conversion.go b/amd64_raw_packed_widen_conversion.go new file mode 100644 index 000000000..4ae731b95 --- /dev/null +++ b/amd64_raw_packed_widen_conversion.go @@ -0,0 +1,27 @@ +package plan9asm + +// The eight packed float-to-(u)int64 conversions share Go 1.27's +// _yvcvtps2qq and _yvcvtpd2qq grammars. Their source width is determined +// by the float lane size and destination width, with common mask, zeroing, +// broadcast, and embedded rounding/SAE rules. +var x86RawPackedWidenConversionByOpcode = map[int]Op{ + 0x7b: "VCVTPS2QQ", + 0x79: "VCVTPS2UQQ", + 0x7a: "VCVTTPS2QQ", + 0x78: "VCVTTPS2UQQ", +} + +var x86RawPackedDoubleConversionByOpcode = map[int]Op{ + 0x7b: "VCVTPD2QQ", + 0x79: "VCVTPD2UQQ", + 0x7a: "VCVTTPD2QQ", + 0x78: "VCVTTPD2UQQ", +} + +func decodedX86PackedWidenConversionInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 1 || p.pp != 1 || p.opcode < 0x78 || p.opcode > 0x7b { + return Instr{}, 0, false, nil + } + return decodedX86PackedNumericConversionInstruction(code, mode) +} diff --git a/amd64_raw_packed_widen_conversion_test.go b/amd64_raw_packed_widen_conversion_test.go new file mode 100644 index 000000000..d1a588393 --- /dev/null +++ b/amd64_raw_packed_widen_conversion_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +var x86RawPackedWidenConversionOps = []Op{ + "VCVTPS2QQ", "VCVTPS2UQQ", "VCVTTPS2QQ", "VCVTTPS2UQQ", + "VCVTPD2QQ", "VCVTPD2UQQ", "VCVTTPD2QQ", "VCVTTPD2UQQ", +} + +func x86RawPackedWidenConversionGoForms() string { + var source strings.Builder + source.WriteString("TEXT rawPackedWidenConversion(SB),4,$0-0\n") + for _, op := range x86RawPackedWidenConversionOps { + registers := []struct{ source, destination string }{ + {"X0", "X1"}, {"X0", "Y1"}, {"Y0", "Z1"}, + } + if strings.Contains(string(op), "PD2") { + registers = []struct{ source, destination string }{ + {"X0", "X1"}, {"Y0", "Y1"}, {"Z0", "Z1"}, + } + } + for _, registers := range registers { + fmt.Fprintf(&source, "\t%s %s, %s\n", op, registers.source, registers.destination) + fmt.Fprintf(&source, "\t%s 0(AX), %s\n", op, registers.destination) + fmt.Fprintf(&source, "\t%s %s, K1, %s\n", op, registers.source, registers.destination) + fmt.Fprintf(&source, "\t%s.Z 0(AX), K1, %s\n", op, registers.destination) + fmt.Fprintf(&source, "\t%s.BCST 0(AX), %s\n", op, registers.destination) + } + roundingSource := "Y0" + if strings.Contains(string(op), "PD2") { + roundingSource = "Z0" + } + if strings.HasPrefix(string(op), "VCVTT") { + fmt.Fprintf(&source, "\t%s.SAE %s, Z1\n", op, roundingSource) + } else { + fmt.Fprintf(&source, "\t%s.RN_SAE %s, Z1\n", op, roundingSource) + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestDecodeX86RawPackedWidenConversionCompleteGoForms(t *testing.T) { + source := x86RawPackedWidenConversionGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } +} + +func TestDecodeX86RawPackedWidenConversionSourceLocalRIPData(t *testing.T) { + for _, double := range []bool{false, true} { + ops := x86RawPackedWidenConversionByOpcode + if double { + ops = x86RawPackedDoubleConversionByOpcode + } + for opcode, op := range ops { + for vectorLength := 0; vectorLength < 3; vectorLength++ { + for _, broadcast := range []bool{false, true} { + p2 := byte(0x08 | vectorLength<<5) + width := 8 << vectorLength + p1 := byte(0x7d) + if double { + p1 |= 0x80 + width *= 2 + } + if broadcast { + p2 |= 0x10 + width = 4 + if double { + width = 8 + } + } + code := []byte{0x62, 0xf1, p1, p2, byte(opcode), 0x05, 1, 0, 0, 0, 0xc3} + for value := 0; value < width; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "packed widening conversion RIP", map[string]bool{}) + if err != nil { + t.Fatalf("%s width %d broadcast=%t: %v", op, width, broadcast, err) + } + if len(decoded) != 2 || len(decoded[0].x86RIPLiteralData) != width || + decoded[0].Args[0].Kind != OpSym { + t.Fatalf("%s width %d broadcast=%t: %#v", op, width, broadcast, decoded) + } + } + } + } + } +} + +func TestTranslateX86RawPackedWidenConversionLLVM22Objects(t *testing.T) { + source := x86RawPackedWidenConversionGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + file := x86RawUnprovedReturnFormProbeFile(t, "amd64", "rawPackedWidenConversion", code) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawPackedWidenConversion": {Name: "rawPackedWidenConversion", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-packed-widen.ll", "raw-packed-widen.o", ir) + }) + } +} diff --git a/amd64_raw_parallel_bits.go b/amd64_raw_parallel_bits.go new file mode 100644 index 000000000..8add3f110 --- /dev/null +++ b/amd64_raw_parallel_bits.go @@ -0,0 +1,49 @@ +package plan9asm + +import "fmt" + +// PDEP/PEXT share Go's _yandnl register/memory-mask grammar. Intel XED's +// datafiles/hswbmi/hsw-bmi-vex-isa.xed.txt confirms W is ignored in not64 +// mode, while L must be zero. This is BMI2, not an arbitrary AVX fallback. +func decodedX86ParallelBitsInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || p.opcode != 0xf5 || (p.pp != 2 && p.pp != 3) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("parallel bit operation: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.vectorLength != 0 { + return fail("requires VEX.L0 encoding") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended register in 32-bit mode") + } + bits := 32 + if p.w && mode == 64 { + bits = 64 + } + var op Op + for name, spec := range amd64ParallelBitSpecs { + if spec.bits == bits && spec.prefix == p.pp { + op = Op(name) + break + } + } + mask, size, err := decodedX86VEXRMSource(code[p.modRM:], mode, p.b, p.x, p.segment) + if err != nil { + return Instr{}, 0, true, err + } + src, _ := decodedX86GeneralRegister(p.upper) + dst, _ := decodedX86GeneralRegister(int(code[p.modRM]>>3&7) + p.r*8) + return Instr{Op: op, Args: []Operand{mask, {Kind: OpReg, Reg: src}, {Kind: OpReg, Reg: dst}}, Raw: fmt.Sprintf("%s %s, %s, %s", op, mask.String(), src, dst)}, p.modRM + size, true, nil +} diff --git a/amd64_raw_parallel_bits_test.go b/amd64_raw_parallel_bits_test.go new file mode 100644 index 000000000..af6e53aec --- /dev/null +++ b/amd64_raw_parallel_bits_test.go @@ -0,0 +1,285 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86RawParallelBitsGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + regs := []string{"AX", "SP", "DI"} + if arch == "amd64" { + regs = append(regs, "R8", "R15") + } + var source strings.Builder + source.WriteString("TEXT parallelbits(SB),4,$0-0\n") + // Go _yandnl: one register/memory mask, source GP, destination GP + // row, independently encoded by each of PDEPL/Q and PEXTL/Q. + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + for _, dst := range regs { + for _, mask := range []string{dst, "(BX)", "-7(BP)(CX*4)", "8192(SI)"} { + fmt.Fprintf(&source, "%s %s,%s,%s\n", op, mask, dst, dst) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if arch == "386" && strings.HasSuffix(string(want[i].Op), "Q") { + want[i].Op = Op(strings.TrimSuffix(string(want[i].Op), "Q") + "L") + } + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawStackObservingFormProbeFile(t, arch, "parallelbits", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"parallelbits": {Name: "parallelbits", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-parallel-bits.ll", "raw-parallel-bits.o", ir) + } + t.Logf("checked %d Go encodings", len(want)-1) + }) + } +} + +func TestX86RawParallelBitsEncodingAxes(t *testing.T) { + count := 0 + for _, deposit := range []bool{false, true} { + for _, wide := range []bool{false, true} { + pp, op := 2, "PEXT" + if deposit { + pp, op = 3, "PDEP" + } + for mask := 0; mask < 16; mask++ { + for src := 0; src < 16; src++ { + for dst := 0; dst < 16; dst++ { + code := encodeRawScalarMove(false, pp, 0xf5, mask, src, dst, 0, false) + code[1] = code[1]&0xe0 | 2 + if wide { + code[2] |= 0x80 + } + for _, mode := range []int{32, 64} { + got, n, matched, err := decodedX86ParallelBitsInstruction(code, mode) + if mode == 32 && (mask >= 8 || src >= 8 || dst >= 8) { + if !matched || err == nil { + t.Fatalf("386 accepted %x", code) + } + continue + } + width := "L" + if wide && mode == 64 { + width = "Q" + } + mr, _ := decodedX86GeneralRegister(mask) + sr, _ := decodedX86GeneralRegister(src) + dr, _ := decodedX86GeneralRegister(dst) + want := []Operand{{Kind: OpReg, Reg: mr}, {Kind: OpReg, Reg: sr}, {Kind: OpReg, Reg: dr}} + if err != nil || !matched || n != len(code) || got.Op != Op(op+width) || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x mode=%d: %+v %v, want %s %+v", code, mode, got, err, op+width, want) + } + } + count++ + } + } + } + } + } + if count != 16384 { + t.Fatalf("covered %d encodings", count) + } +} + +func TestX86RawParallelBitsInvalidAndSegmentForms(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x7a, 0xf5}, {0xc4, 0xe2, 0x7a, 0xf5, 0x04}, {0xc4, 0xe2, 0x7a, 0xf5, 0x45}, + {0xc4, 0xe2, 0x7e, 0xf5, 0xc0}, // Reserved L=1. + {0x62, 0xf2, 0x7e, 0x08, 0xf5, 0xc0}, // Not a Go EVEX family. + {0x67, 0xc4, 0xe2, 0x7a, 0xf5, 0x00}, + } { + if _, _, matched, err := decodedX86ParallelBitsInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x", code) + } + } + if _, _, matched, err := decodedX86ParallelBitsInstruction([]byte{0xc4, 0xe2, 0x7a, 0xf5, 0xc0}, 16); !matched || err == nil { + t.Fatal("accepted invalid mode") + } + for _, code := range [][]byte{nil, {0xc4}, {0xc4, 0xe2, 0x78, 0xf5, 0xc0}, {0xc4, 0xe1, 0x7a, 0xf5, 0xc0}} { + if _, _, matched, _ := decodedX86ParallelBitsInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + for _, seg := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + code := []byte{seg.prefix, 0xc4, 0x02, 0xb2, 0xf5, 0x64, 0x88, 0xf9} + got, n, matched, err := decodedX86ParallelBitsInstruction(code, 64) + want := []Operand{{Kind: OpMem, Mem: MemRef{Segment: seg.reg, Base: "R8", Index: "R9", Scale: 4, Off: -7}}, {Kind: OpReg, Reg: "R9"}, {Kind: OpReg, Reg: "R12"}} + if err != nil || !matched || n != len(code) || got.Op != "PEXTQ" || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v, want %+v", code, got, err, want) + } + } +} + +func TestX86ParallelBitsGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + for _, args := range []string{" AX,BX", " X0,AX,BX", " AX,X0,BX", " AX,BX,X0", " AX,(BX),CX", " AX,BX,(CX)", " $1,BX,CX", ".Z AX,BX,CX", ".BCST (AX),BX,CX"} { + source := "TEXT invalid(SB),4,$0-0\n" + op + args + "\nRET\n" + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + if output, err := cmd.CombinedOutput(); err == nil { + t.Fatalf("Go %s accepted %s%s: %s", arch, op, args, output) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"invalid": {Name: "invalid", Ret: Void}}}); err == nil { + t.Fatalf("translator accepted %s%s on %s", op, args, arch) + } + } + } + } +} + +func TestX86ParallelBitsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; separate five-target object tests are required") + } + var source, decl, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range []string{"PDEPL", "PDEPQ", "PEXTL", "PEXTQ"} { + for _, memory := range []bool{false, true} { + for dst := 8; dst <= 10; dst++ { + name := fmt.Sprintf("parallelbits%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ value+0(FP),R8\nMOVQ mask+8(FP),SI\nMOVQ (SI),R9\nMOVQ out+16(FP),DI\nMOVQ $-1,R10\nMOVL $2147483647,CX\nADDL $1,CX\n", name) + mask := "R9" + if memory { + mask = "(SI)" + } + code := assembleX87ControlBytes(t, "amd64", fmt.Sprintf("TEXT probe(SB),4,$0-0\n%s %s,R8,R%d\nRET\n", op, mask, dst)) + for _, b := range code[:len(code)-1] { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + fmt.Fprintf(&source, "MOVQ R%d,(DI)\nSETCS 8(DI)\nSETEQ 9(DI)\nSETPS 10(DI)\nSETMI 11(DI)\nSETOS 12(DI)\nRET\n", dst) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{I64, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I64, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}}}} + fmt.Fprintf(&decl, "extern void %s(uint64_t,const void *,void *);\n", name) + bits, deposit := 32, 0 + if strings.HasSuffix(op, "Q") { + bits = 64 + } + if strings.HasPrefix(op, "PDEP") { + deposit = 1 + } + fmt.Fprintf(&checks, "if(check(%s,%d,%d)) return %d;\n", name, bits, deposit, index) + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple = "x86_64-apple-macosx" + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := `#include +#include +#include +` + decl.String() + ` +static uint64_t reference(uint64_t value,uint64_t mask,int bits,int deposit) { + if(bits==32) {value=(uint32_t)value;mask=(uint32_t)mask;} + uint64_t result=0,cursor=1; + while(mask) {uint64_t low=mask&(-mask);if(deposit ? (value&cursor)!=0 : (value&low)!=0) result|=deposit?low:cursor;mask&=mask-1;cursor<<=1;} + return result; +} +static int check(void (*fn)(uint64_t,const void *,void *),int bits,int deposit) { + uint64_t state=0x9e3779b97f4a7c15ULL; + for(int i=0;i<256;i++) { + state^=state<<13;state^=state>>7;state^=state<<17;uint64_t value=state; + state^=state<<13;state^=state>>7;state^=state<<17;uint64_t mask=state; + if(i==0)mask=0;if(i==1)mask=~(uint64_t)0;if(i>=2&&i<66)mask=(uint64_t)1<<(i-2); + uint8_t storage[9],out[16]={0};memcpy(storage+1,&mask,8); + fn(value,storage+1,out);uint64_t actual;memcpy(&actual,out,8); + const uint8_t flags[5]={0,0,1,1,1}; + if(actual!=reference(value,mask,bits,deposit)||memcmp(out+8,flags,5)) {fprintf(stderr,"parallel bits mismatch bits=%d deposit=%d case=%d\n",bits,deposit,i);return 1;} + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "parallel_bits", triple, ir, mainC, prefix) +} + +func TestX86ParallelBits386IgnoresW(t *testing.T) { + // Intel XED hsw-bmi-vex-isa.xed.txt has unconstrained W in not64 + // mode: both Go-accepted L and Q spellings access only a 32-bit mask. + for _, op := range []string{"PDEPQ", "PEXTQ"} { + source := "TEXT width386(SB),4,$0-0\n" + op + " (AX),BX,CX\nRET\n" + assembleX87ControlBytes(t, "386", source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: map[string]FuncSig{"width386": {Name: "width386", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(ir, "lshr i64") || !strings.Contains(ir, "lshr i32") || !strings.Contains(ir, "load i32, ptr") { + t.Fatalf("386 %s failed to normalize width", op) + } + } +} diff --git a/amd64_raw_permute128_test.go b/amd64_raw_permute128_test.go new file mode 100644 index 000000000..d6f18c0d5 --- /dev/null +++ b/amd64_raw_permute128_test.go @@ -0,0 +1,111 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVPERM2X128(opcode, immediate byte, first, second, destination int) []byte { + p0 := byte((1-destination/8)<<7 | 1<<6 | (1-first/8)<<5 | 3) + p1 := byte((^second&15)<<3 | 5) + modRM := byte(0xc0 | destination&7<<3 | first&7) + return []byte{0xc4, p0, p1, opcode, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedPsiphonVPERM2I128(t *testing.T) { + code := []byte{0xc4, 0xe3, 0x6d, 0x46, 0xc3, 0x20} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "Psiphon VPERM2I128 sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + const want = "VPERM2I128 $32, Y3, Y2, Y0" + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, want+" ") { + t.Fatalf("decoded %x as %#v, want %q", code, decoded, want) + } +} + +func TestDecodedX86RawVPERM2X128CompleteRegisterFamily(t *testing.T) { + count := 0 + for _, operation := range []struct { + opcode byte + name string + }{ + {opcode: 0x06, name: "VPERM2F128"}, + {opcode: 0x46, name: "VPERM2I128"}, + } { + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(first*16 + destination) + code := encodeX86RawVPERM2X128(operation.opcode, immediate, first, second, destination) + got, length, ok, err := decodedX86Permute128Instruction(code, 64) + want := fmt.Sprintf("%s $%d, Y%d, Y%d, Y%d", operation.name, immediate, first, second, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VPERM2F128/VPERM2I128 register encodings, want 8192", count) + } +} + +func TestDecodedX86RawVPERM2X128MemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + { + name: "VPERM2F128 extended SIB", + code: []byte{0x65, 0xc4, 0x03, 0x15, 0x06, 0x64, 0x8b, 0x20, 0xff}, + want: "VPERM2F128 $255, 32(R11)(R9*4)(GS), Y13, Y12", + }, + { + name: "VPERM2I128 extended SIB", + code: []byte{0x64, 0xc4, 0x03, 0x15, 0x46, 0x64, 0x8b, 0x20, 0xfe}, + want: "VPERM2I128 $254, 32(R11)(R9*4)(FS), Y13, Y12", + }, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86Permute128Instruction(test.code, 64) + if err != nil || !ok || length != len(test.code) || got.Raw != test.want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", test.code, got, length, ok, err, test.want) + } + }) + } + + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 operand limit", mode: 32, encoding: encodeX86RawVPERM2X128(0x46, 1, 0, 1, 2), wantOK: true}, + {name: "wrong pp", mode: 64, encoding: []byte{0xc4, 0xe3, 0x6c, 0x46, 0xc3, 4}, wantOK: true}, + {name: "W", mode: 64, encoding: []byte{0xc4, 0xe3, 0xed, 0x46, 0xc3, 4}, wantOK: true}, + {name: "128-bit vector length", mode: 64, encoding: []byte{0xc4, 0xe3, 0x69, 0x46, 0xc3, 4}, wantOK: true}, + {name: "truncated immediate", mode: 64, encoding: []byte{0xc4, 0xe3, 0x6d, 0x46, 0xc3}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0xc4, 0xe3, 0x6d, 0x46, 0xc3, 4}, wantOK: true}, + {name: "legacy prefix", mode: 64, encoding: []byte{0x66, 0xc4, 0xe3, 0x6d, 0x46, 0xc3, 4}}, + {name: "wrong opcode", mode: 64, encoding: []byte{0xc4, 0xe3, 0x6d, 0x47, 0xc3, 4}}, + {name: "wrong map", mode: 64, encoding: []byte{0xc4, 0xe2, 0x6d, 0x46, 0xc3, 4}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86Permute128Instruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x returned ok=%v err=%v, want ok=%v", test.encoding, ok, err, test.wantOK) + } + if test.wantOK && err == nil { + t.Fatalf("invalid encoding %x was accepted", test.encoding) + } + if !test.wantOK && err != nil { + t.Fatalf("unrelated encoding %x returned err=%v", test.encoding, err) + } + }) + } +} diff --git a/amd64_raw_permute_inlane_test.go b/amd64_raw_permute_inlane_test.go new file mode 100644 index 000000000..14b7f86d7 --- /dev/null +++ b/amd64_raw_permute_inlane_test.go @@ -0,0 +1,292 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86RawVEXInLaneFloatingPermute(double, immediate bool, vectorBits byte, first, second, destination int, imm byte) []byte { + mapNumber := byte(2) + opcode := byte(0x0c) + if immediate { + mapNumber = 3 + opcode = 0x04 + } + if double { + opcode++ + } + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | mapNumber + p1 := byte(^second&15)<<3 | vectorBits<<2 | 1 + if immediate { + p1 = 0x78 | vectorBits<<2 | 1 + } + code := []byte{0xc4, p0, p1, opcode, byte(0xc0 | destination&7<<3 | first&7)} + if immediate { + code = append(code, imm) + } + return code +} + +func encodeX86RawEVEXInLaneFloatingPermute(double, immediate bool, vectorBits byte, mask int, zeroing, broadcast bool, first, second, destination int, imm byte) []byte { + mapNumber := byte(2) + opcode := byte(0x0c) + if immediate { + mapNumber = 3 + opcode = 0x04 + } + if double { + opcode++ + } + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | mapNumber + p1 := byte(^second&15)<<3 | 5 + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if immediate { + p1 = 0x7d + p2 |= 0x08 + } + if double { + p1 |= 0x80 + } + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + code := []byte{0x62, p0, p1, p2, opcode, byte(0xc0 | destination&7<<3 | first&7)} + if immediate { + code = append(code, imm) + } + return code +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVPERMILPS(t *testing.T) { + // vpermilps $180, %ymm2, %ymm3; + // vblendps $72, %ymm3, %ymm1, %ymm3 + code := []byte{0xc4, 0xe3, 0x7d, 0x04, 0xda, 0xb4, 0xc4, 0xe3, 0x75, 0x0c, 0xdb, 0x48} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VPERMILPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VPERMILPS $180, Y2, Y3 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawInLaneFloatingPermuteCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, double := range []bool{false, true} { + op := "VPERMILPS" + if double { + op = "VPERMILPD" + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for first := 0; first < 16; first++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXInLaneFloatingPermute(double, true, vectorBits, first, 0, destination, 0xa5) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + want := fmt.Sprintf("%s $165, %s%d, %s%d", op, vectorName, first, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXInLaneFloatingPermute(double, false, vectorBits, first, second, destination, 0) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + want := fmt.Sprintf("%s %s%d, %s%d, %s%d", op, vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{{}, {mask: 3}, {mask: 7, zeroing: true, suffix: ".Z"}} { + for first := 0; first < 32; first++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXInLaneFloatingPermute(double, true, vectorBits, masking.mask, masking.zeroing, false, first, 0, destination, 0x5a) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + args := []string{"$90", fmt.Sprintf("%s%d", vectorName, first)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := op + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXInLaneFloatingPermute(double, false, vectorBits, masking.mask, masking.zeroing, false, first, second, destination, 0) + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", vectorName, first), fmt.Sprintf("%s%d", vectorName, second)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := op + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 625664 { + t.Fatalf("covered %d in-lane floating-permute register encodings, want 625664", count) + } +} + +func TestDecodedX86RawInLaneFloatingPermuteMemoryAndInvalidForms(t *testing.T) { + vexMemory := []byte{0xc4, 0x03, 0x7d, 0x05, 0x7c, 0x8b, 0x02, 0xfe} + got, length, ok, err := decodedX86InLaneFloatingPermuteInstruction(vexMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 2} + if err != nil || !ok || length != len(vexMemory) || got.Op != "VPERMILPD" || got.Args[0].Imm != 254 || got.Args[2].String() != "Y15" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexImmediateMemory := []byte{0x62, 0xf3, 0xfd, 0xdb, 0x05, 0x50, 0x02, 0x07} + got, length, ok, err = decodedX86InLaneFloatingPermuteInstruction(evexImmediateMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 16} + if err != nil || !ok || length != len(evexImmediateMemory) || got.Op != "VPERMILPD.BCST.Z" || got.Args[0].Imm != 7 || got.Args[2].String() != "K3" || got.Args[3].String() != "Z2" || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexImmediateMemory, got, length, ok, err) + } + evexVariableMemory := []byte{0x62, 0xf2, 0x75, 0x38, 0x0c, 0x50, 0x02} + got, length, ok, err = decodedX86InLaneFloatingPermuteInstruction(evexVariableMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 8} + if err != nil || !ok || length != len(evexVariableMemory) || got.Op != "VPERMILPS.BCST" || got.Args[1].String() != "Y1" || got.Args[2].String() != "Y2" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexVariableMemory, got, length, ok, err) + } + + validVEXImmediate := encodeX86RawVEXInLaneFloatingPermute(false, true, 0, 7, 0, 7, 1) + validEVEXVariable := encodeX86RawEVEXInLaneFloatingPermute(false, false, 2, 0, false, false, 7, 6, 5, 0) + validEVEXImmediate := encodeX86RawEVEXInLaneFloatingPermute(false, true, 2, 0, false, false, 7, 0, 5, 1) + for name, invalid := range map[string][]byte{ + "vex missing immediate": validVEXImmediate[:len(validVEXImmediate)-1], + "vex W": {validVEXImmediate[0], validVEXImmediate[1], validVEXImmediate[2] | 0x80, validVEXImmediate[3], validVEXImmediate[4], validVEXImmediate[5]}, + "vex prefix": {validVEXImmediate[0], validVEXImmediate[1], validVEXImmediate[2] &^ 0x01, validVEXImmediate[3], validVEXImmediate[4], validVEXImmediate[5]}, + "vex immediate vvvv": {validVEXImmediate[0], validVEXImmediate[1], validVEXImmediate[2] &^ 0x08, validVEXImmediate[3], validVEXImmediate[4], validVEXImmediate[5]}, + "evex fixed bit": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2] &^ 0x04, validEVEXVariable[3], validEVEXVariable[4], validEVEXVariable[5]}, + "evex wrong W": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2] | 0x80, validEVEXVariable[3], validEVEXVariable[4], validEVEXVariable[5]}, + "evex immediate vvvvv": {validEVEXImmediate[0], validEVEXImmediate[1], validEVEXImmediate[2] &^ 0x08, validEVEXImmediate[3] &^ 0x08, validEVEXImmediate[4], validEVEXImmediate[5], validEVEXImmediate[6]}, + "evex reserved length": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2], validEVEXVariable[3] | 0x60, validEVEXVariable[4], validEVEXVariable[5]}, + "zero without mask": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2], validEVEXVariable[3] | 0x80, validEVEXVariable[4], validEVEXVariable[5]}, + "register broadcast": {validEVEXVariable[0], validEVEXVariable[1], validEVEXVariable[2], validEVEXVariable[3] | 0x10, validEVEXVariable[4], validEVEXVariable[5]}, + "address override": append([]byte{0x67}, validVEXImmediate...), + } { + if instruction, _, matched, decodeErr := decodedX86InLaneFloatingPermuteInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + masked386 := encodeX86RawEVEXInLaneFloatingPermute(false, false, 0, 1, false, false, 0, 1, 2, 0) + if instruction, _, matched, decodeErr := decodedX86InLaneFloatingPermuteInstruction(masked386, 32); !matched || decodeErr == nil { + t.Fatalf("386 masked encoding %x decoded as %+v, ok=%v, err=%v", masked386, instruction, matched, decodeErr) + } + highZ386 := encodeX86RawEVEXInLaneFloatingPermute(false, false, 2, 0, false, false, 8, 1, 2, 0) + if instruction, _, matched, decodeErr := decodedX86InLaneFloatingPermuteInstruction(highZ386, 32); !matched || decodeErr == nil { + t.Fatalf("386 high-Z encoding %x decoded as %+v, ok=%v, err=%v", highZ386, instruction, matched, decodeErr) + } +} + +func TestX86InLaneFloatingPermuteRawGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + VPERMILPS $7, X0, X1 + VPERMILPD X0, X1, X2 + VPERMILPS $7, X20, X21 + VPERMILPD Z0, Z6, Z7 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "VPERMILPS $7, Z8, Z0", + "VPERMILPD Z0, Z1, Z8", + "VPERMILPS X0, X1, K1, X2", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawInLaneFloatingPermuteTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_in_lane_floating_permute(SB),NOSPLIT,$0-0\n") + for _, double := range []bool{false, true} { + var encodings [][]byte + if target.goarch == "amd64" { + encodings = append(encodings, + encodeX86RawVEXInLaneFloatingPermute(double, true, 1, 15, 0, 14, 0xa5), + encodeX86RawVEXInLaneFloatingPermute(double, false, 1, 15, 14, 13, 0), + encodeX86RawEVEXInLaneFloatingPermute(double, true, 2, 7, true, false, 31, 0, 30, 0x5a), + encodeX86RawEVEXInLaneFloatingPermute(double, false, 2, 3, false, false, 31, 30, 29, 0), + ) + } else { + encodings = append(encodings, + encodeX86RawVEXInLaneFloatingPermute(double, true, 1, 7, 0, 6, 0xa5), + encodeX86RawVEXInLaneFloatingPermute(double, false, 1, 7, 6, 5, 0), + encodeX86RawEVEXInLaneFloatingPermute(double, true, 0, 0, false, false, 20, 0, 21, 0x5a), + encodeX86RawEVEXInLaneFloatingPermute(double, false, 2, 0, false, false, 5, 6, 7, 0), + ) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "raw_in_lane_floating_permute": {Name: "raw_in_lane_floating_permute", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-in-lane-floating-permute-"+target.name+".ll", "raw-in-lane-floating-permute-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_portio.go b/amd64_raw_portio.go new file mode 100644 index 000000000..2428c7984 --- /dev/null +++ b/amd64_raw_portio.go @@ -0,0 +1,54 @@ +package plan9asm + +import "fmt" + +// decodedX86ScalarPortIOInstruction covers Go's complete yin table: input +// and output, byte/word/long, with either an immediate port or implicit DX. +// x/arch's GoSyntax prints the architectural accumulator and DX operands, +// whereas Go's assembler source syntax omits both. Its old decoder also calls +// the byte encodings INL/OUTL. Decode these small fixed forms from the opcode. +func decodedX86ScalarPortIOInstruction(code []byte) (Instr, int, bool, error) { + if len(code) == 0 { + return Instr{}, 0, false, nil + } + index := 0 + word := false + if code[index] == 0x66 { + word = true + index++ + if len(code) == index { + return Instr{}, 0, false, nil + } + } + opcode := code[index] + if opcode < 0xe4 || (opcode > 0xe7 && opcode < 0xec) || opcode > 0xef { + return Instr{}, 0, false, nil + } + input := opcode == 0xe4 || opcode == 0xe5 || opcode == 0xec || opcode == 0xed + byteWidth := opcode&1 == 0 + name := "OUT" + if input { + name = "IN" + } + width := "L" + if byteWidth { + width = "B" + } else if word { + width = "W" + } + index++ + var args []Operand + if opcode >= 0xe4 && opcode <= 0xe7 { + if len(code) == index { + return Instr{}, 0, true, fmt.Errorf("missing immediate port byte") + } + args = []Operand{{Kind: OpImm, Imm: int64(code[index])}} + index++ + } + op := Op(name + width) + raw := string(op) + if len(args) != 0 { + raw = fmt.Sprintf("%s $%d", op, args[0].Imm) + } + return Instr{Op: op, Args: args, Raw: raw}, index, true, nil +} diff --git a/amd64_raw_portio_test.go b/amd64_raw_portio_test.go new file mode 100644 index 000000000..a35a5d359 --- /dev/null +++ b/amd64_raw_portio_test.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawScalarPortIOCompleteEncodingFamily(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + port int64 + }{ + {"in byte immediate", []byte{0xe4, 0x7f}, "INB", 0x7f}, + {"in word immediate", []byte{0x66, 0xe5, 0x7f}, "INW", 0x7f}, + {"in long immediate", []byte{0xe5, 0x7f}, "INL", 0x7f}, + {"in byte DX", []byte{0xec}, "INB", -1}, + {"in word DX", []byte{0x66, 0xed}, "INW", -1}, + {"in long DX", []byte{0xed}, "INL", -1}, + {"out byte immediate", []byte{0xe6, 0x7f}, "OUTB", 0x7f}, + {"out word immediate", []byte{0x66, 0xe7, 0x7f}, "OUTW", 0x7f}, + {"out long immediate", []byte{0xe7, 0x7f}, "OUTL", 0x7f}, + {"out byte DX", []byte{0xee}, "OUTB", -1}, + {"out word DX", []byte{0x66, 0xef}, "OUTW", -1}, + {"out long DX", []byte{0xef}, "OUTL", -1}, + } + for _, goarch := range []string{"386", "amd64"} { + for _, test := range tests { + t.Run(goarch+"/"+test.name, func(t *testing.T) { + fn := Func{} + for _, value := range test.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 { + t.Fatalf("decoded %#x as %#v, want one instruction", test.code, got.Instrs) + } + ins := got.Instrs[0] + wantArgs := 0 + if test.port >= 0 { + wantArgs = 1 + } + if ins.Op != test.op || len(ins.Args) != wantArgs { + t.Fatalf("decoded %#x as %#v, want %s with port %d", test.code, ins, test.op, test.port) + } + if test.port >= 0 && (ins.Args[0].Kind != OpImm || ins.Args[0].Imm != test.port) { + t.Fatalf("decoded %#x port as %#v, want $%d", test.code, ins.Args[0], test.port) + } + }) + } + } +} + +func TestX86RawScalarPortIOImmediateByteRange(t *testing.T) { + for _, opcode := range []byte{0xe4, 0xe5, 0xe6, 0xe7} { + for port := 0; port < 256; port++ { + instruction, length, ok, err := decodedX86ScalarPortIOInstruction([]byte{opcode, byte(port)}) + if !ok || err != nil || length != 2 || len(instruction.Args) != 1 || instruction.Args[0].Imm != int64(port) { + t.Fatalf("opcode %#x port %d: instruction=%#v length=%d recognized=%t error=%v", opcode, port, instruction, length, ok, err) + } + } + _, _, ok, err := decodedX86ScalarPortIOInstruction([]byte{opcode}) + if !ok || err == nil { + t.Fatalf("opcode %#x with missing port: recognized=%t error=%v", opcode, ok, err) + } + } +} + +func TestX86RawScalarPortIOCompilesEveryForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawportio(SB),NOSPLIT,$0-0\n") + for _, code := range [][]byte{ + {0xe4, 0x7f}, {0x66, 0xe5, 0x7f}, {0xe5, 0x7f}, + {0xec}, {0x66, 0xed}, {0xed}, + {0xe6, 0x7f}, {0x66, 0xe7, 0x7f}, {0xe7, 0x7f}, + {0xee}, {0x66, 0xef}, {0xef}, + } { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawportio": {Name: "rawportio", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-portio.ll", "raw-portio.o", ir) + }) + } +} diff --git a/amd64_raw_prefetch_hint_test.go b/amd64_raw_prefetch_hint_test.go new file mode 100644 index 000000000..275642faf --- /dev/null +++ b/amd64_raw_prefetch_hint_test.go @@ -0,0 +1,79 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestX86RawExtendedPrefetchHints(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {goarch: "386", triple: "i386-unknown-linux-gnu"}, + } { + t.Run(target.goarch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawExtendedPrefetch(SB),$0-0\n") + codes := [][]byte{ + {0x0f, 0x18, 0x20}, // /4, base register + {0x0f, 0x18, 0x30}, // /6, base register + {0x0f, 0x18, 0x38}, // /7, base register + {0x65, 0x0f, 0x18, 0x74, 0x8b, 0xf9}, // /6, GS and SIB with disp8 + } + if target.goarch == "amd64" { + codes = append(codes, []byte{0x4d, 0x0f, 0x18, 0x7c, 0x8c, 0x10}) // /7, extended base/index + } + for _, code := range codes { + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawExtendedPrefetch": {Name: "rawExtendedPrefetch", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-extended-prefetch.ll", "raw-extended-prefetch.o", ir) + }) + } +} + +func TestX86RawExtendedPrefetchHintsRejectInvalidForms(t *testing.T) { + for _, code := range [][]byte{ + {0x0f, 0x18, 0x28}, // /5 is reserved. + {0x0f, 0x18, 0xf0}, // /6 requires memory. + {0x67, 0x0f, 0x18, 0x30}, // Address-size override is not modeled. + {0x0f, 0x18, 0x35, 0, 0, 0, 0}, // RIP-relative address has no proven source layout. + } { + var source strings.Builder + source.WriteString("TEXT badRawPrefetch(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"badRawPrefetch": {Name: "badRawPrefetch", Ret: Void}}, + }); err == nil { + t.Fatalf("accepted unsupported raw prefetch form %x", code) + } + } +} diff --git a/amd64_raw_prefix.go b/amd64_raw_prefix.go new file mode 100644 index 000000000..69f4c743c --- /dev/null +++ b/amd64_raw_prefix.go @@ -0,0 +1,60 @@ +package plan9asm + +import ( + "fmt" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel specifies that the r/m field is ignored for each unprefixed register +// fence encoding. The pinned decoder recognizes only the canonical r/m=0, +// and the cache-line recognizer otherwise mistakes SFENCE /7 for CLFLUSH. +// Do not strip or reinterpret any prefix here: those remain independently +// decoded and validated, including LOCK and the distinct 66 memory forms. +func decodedX86OrderingFenceInstruction(code []byte, mode int) (Instr, int, bool) { + if mode != 32 && mode != 64 || len(code) < 3 || code[0] != 0x0f || code[1] != 0xae || code[2]>>6 != 3 { + return Instr{}, 0, false + } + op := Op("") + switch code[2] >> 3 & 7 { + case 5: + op = "LFENCE" + case 6: + op = "MFENCE" + case 7: + op = "SFENCE" + default: + return Instr{}, 0, false + } + return Instr{Op: op}, 3, true +} + +// x/arch keeps invalidity in metadata that GoSyntax can erase or print as a +// standalone prefix with the real instruction in its operands. Reject before +// any width/alias rewrite. PrefixIgnored alone is not machine invalidity. +func decodedX86PrefixError(inst x86asm.Inst) error { + for _, prefix := range inst.Prefix { + if prefix&x86asm.PrefixInvalid != 0 { + return fmt.Errorf("raw %s has a machine-invalid prefix 0x%02x", inst.Op, byte(prefix)) + } + } + for _, prefix := range inst.Prefix { + if prefix&0xff == x86asm.PrefixLOCK && prefix&x86asm.PrefixIgnored == 0 { + // This is valid native syntax, not invalid source. The existing + // standalone LOCK lowerer cannot bind a printed whole RMW or its + // atomic effect. Do not silently discard the following opcode. + return fmt.Errorf("%w: raw LOCK %s requires its complete atomic prefix/operand effect", ErrProbeNeedsContext, inst.Op) + } + if prefix&(x86asm.PrefixIgnored|x86asm.PrefixImplicit) != 0 { + continue + } + switch prefix { + case x86asm.PrefixXACQUIRE, x86asm.PrefixXRELEASE, x86asm.PrefixBND: + // GoSyntax uses only the low prefix byte for REP/REPNE. That + // textual repeater is not the original transactional or bounds + // effect and can leak onto a later string operation. + return fmt.Errorf("%w: raw %s %s requires its complete native prefix effect", ErrProbeNeedsContext, prefix, inst.Op) + } + } + return nil +} diff --git a/amd64_raw_prefix_class_test.go b/amd64_raw_prefix_class_test.go new file mode 100644 index 000000000..a0824a60f --- /dev/null +++ b/amd64_raw_prefix_class_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86OrderingHintsPhysicalFenceRMIsIgnored(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + op Op + base byte + }{ + {"LFENCE", 0xe8}, + {"MFENCE", 0xf0}, + {"SFENCE", 0xf8}, + } { + for rm := byte(0); rm < 8; rm++ { + t.Run(fmt.Sprintf("%d/%s/rm=%d", mode, row.op, rm), func(t *testing.T) { + code := []byte{0x0f, 0xae, row.base | rm} + x86PrefixOriginalBytes(t, mode, code) + got, err := decodeX86RawDirectiveGroup(code, mode, 0, "physical fence", nil) + if err != nil || len(got) != 1 || got[0].Op != row.op || len(got[0].Args) != 0 { + t.Fatalf("physical ignored-r/m fence changed: %x: %+v/%v", code, got, err) + } + }) + } + } + } +} + +// The complete Intel LOCK list is independent of the translator. The original +// named Go object binds width, destination and opcode; no RMW is executed here. +// Valid raw LOCK remains Context until the complete atomic effect is bound. +func TestX86RawValidLockCompleteRMWClass(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + mode := 32 + widths := "BWL" + if arch == "amd64" { + mode = 64 + widths += "Q" + } + var forms []string + for _, stem := range []string{"ADD", "ADC", "AND", "OR", "SBB", "SUB", "XOR", "CMPXCHG", "XADD", "XCHG"} { + for _, width := range widths { + reg := "CX" + if width == 'B' { + reg = "CL" + } + forms = append(forms, fmt.Sprintf("%s%c %s,(AX)", stem, width, reg)) + } + } + for _, stem := range []string{"INC", "DEC", "NEG", "NOT"} { + for _, width := range widths { + forms = append(forms, fmt.Sprintf("%s%c (AX)", stem, width)) + } + } + for _, stem := range []string{"BTC", "BTR", "BTS"} { + for _, width := range strings.TrimPrefix(widths, "B") { + forms = append(forms, fmt.Sprintf("%s%c CX,(AX)", stem, width)) + } + } + forms = append(forms, "CMPXCHG8B (AX)") + if arch == "amd64" { + forms = append(forms, "CMPXCHG16B (AX)") + } + want := 49 + if arch == "amd64" { + want = 67 + } + if len(forms) != want { + t.Fatal("complete native-width LOCK inventory changed", arch, len(forms)) + } + for _, form := range forms { + t.Run(arch+"/"+form, func(t *testing.T) { + source := "TEXT locked(SB),4,$0-0\nLOCK\n" + form + "\nRET\n" + actual := x86OrderingHintGoBytes(t, arch, source) + if len(actual) < 3 || actual[0] != 0xf0 || actual[len(actual)-1] != 0xc3 { + t.Fatalf("original named LOCK object lacks complete prefix/body/RET: %x", actual) + } + code := actual[:len(actual)-1] + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatalf("actual complete LOCK instruction %+v: %v", physical, err) + } + for _, prefix := range physical.Prefix { + if prefix&x86asm.PrefixInvalid != 0 { + t.Fatal("accepted memory RMW has invalid machine LOCK", physical) + } + } + got, err := decodeX86RawDirectiveGroup(code, mode, 0, source, nil) + if !errors.Is(err, ErrProbeNeedsContext) || len(got) != 0 { + t.Fatalf("complete valid LOCK silently lost atomic/body effects: %+v/%v", got, err) + } + // The simple named (AX) form ends in its ModRM byte. Change only + // mod to register addressing, retaining the original opcode/width. + // BYTE acceptance binds invalid machine bytes, not ISA validity. + bad := append([]byte(nil), code...) + bad[len(bad)-1] |= 0xc0 + x86PrefixOriginalBytes(t, mode, bad) + if got, err := decodeX86RawDirectiveGroup(bad, mode, 0, "invalid register-only LOCK "+form, nil); err == nil { + t.Fatalf("complete LOCK class admitted register-only RMW: %x/%+v", bad, got) + } + }) + } + } +} + +func TestX86RawVisibleEffectPrefixesNeedWholeContract(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, code := range [][]byte{ + {0xf3, 0x89, 0x08}, // XRELEASE MOVL CX,(AX), no explicit LOCK. + {0xf3, 0x87, 0x08}, // XRELEASE on implicitly locked XCHG. + {0xf2, 0x87, 0x08}, // XACQUIRE on implicitly locked XCHG. + {0xf2, 0xc3}, // BND RET, not an ordinary near return. + } { + t.Run(fmt.Sprintf("%d/%x", mode, code), func(t *testing.T) { + x86PrefixOriginalBytes(t, mode, code) + physical, err := x86asm.Decode(code, mode) + if err != nil || physical.Len != len(code) { + t.Fatal("complete original prefixed object", physical, err) + } + got, err := decodeX86RawDirectiveGroup(code, mode, 0, "whole effect prefix", nil) + t.Logf("bytes=%x actualPrefix=%#v GoSyntax=%q parsed=%+v error=%v", code, + physical.Prefix, x86asm.GoSyntax(physical, 0, nil), got, err) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("visible native effect became an ordinary opcode/string repeat without a contract", err) + } + }) + } + } +} + +// These BYTE objects observe conversion, not a new string/caller contract. +// Visible prefixes must either retain the complete body or fail closed. +func TestX86RawVisibleStringPrefixesRetainBody(t *testing.T) { + for _, mode := range []int{32, 64} { + for _, row := range []struct { + name string + code []byte + body Op + }{ + {"REP MOVSB", []byte{0xf3, 0xa4}, "MOVSB"}, + {"REPE CMPSB", []byte{0xf3, 0xa6}, "CMPSB"}, + {"REPNE CMPSB", []byte{0xf2, 0xa6}, "CMPSB"}, + {"duplicate REP MOVSB", []byte{0xf3, 0xf3, 0xa4}, "MOVSB"}, + {"XRELEASE MOV", []byte{0xf3, 0x89, 0x08}, "MOVL"}, + {"XACQUIRE LOCK ADD", []byte{0xf2, 0xf0, 0x01, 0x08}, "ADDL"}, + {"XRELEASE LOCK ADD", []byte{0xf3, 0xf0, 0x01, 0x08}, "ADDL"}, + } { + t.Run(fmt.Sprintf("%d/%s", mode, row.name), func(t *testing.T) { + x86PrefixOriginalBytes(t, mode, row.code) + physical, err := x86asm.Decode(row.code, mode) + if err != nil || physical.Len != len(row.code) { + t.Fatal("complete physical visible prefix", physical, err) + } + got, err := decodeX86RawDirectiveGroup(row.code, mode, 0, row.name, nil) + t.Logf("actual bytes=%x syntax=%q whole=%+v error=%v", row.code, + x86asm.GoSyntax(physical, 0, nil), got, err) + if err != nil { + return + } + body := false + for _, instruction := range got { + body = body || instruction.Op == row.body + } + if !body { + t.Fatal("visible prefix conversion discarded its actual instruction body", got) + } + }) + } + } +} diff --git a/amd64_raw_qq_to_float.go b/amd64_raw_qq_to_float.go new file mode 100644 index 000000000..38b3869f2 --- /dev/null +++ b/amd64_raw_qq_to_float.go @@ -0,0 +1,32 @@ +package plan9asm + +// x86RawQQToFloatForm maps opcode and mandatory prefix to the Go 1.27 +// QWORD-to-PS/PD optabs. PS narrows the Y/Z destination by one vector width. +func x86RawQQToFloatForm(p x86RawVectorEncoding) (base Op, narrow bool, ok bool) { + switch { + case p.opcode == 0x5b && p.pp == 0: + return "VCVTQQ2PS", true, true + case p.opcode == 0x7a && p.pp == 3: + return "VCVTUQQ2PS", true, true + case p.opcode == 0xe6 && p.pp == 2: + return "VCVTQQ2PD", false, true + case p.opcode == 0x7a && p.pp == 2: + return "VCVTUQQ2PD", false, true + default: + return "", false, false + } +} + +// decodedX86RawQQToFloatInstruction handles all four EVEX QWORD-to-PS/PD +// conversion families, including every X/Y/Z, K-mask, broadcast, and +// maximum-width embedded-rounding form. +func decodedX86RawQQToFloatInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || !p.w || p.mapNumber != 1 { + return Instr{}, 0, false, nil + } + if _, _, known := x86RawQQToFloatForm(p); !known { + return Instr{}, 0, false, nil + } + return decodedX86PackedNumericConversionInstruction(code, mode) +} diff --git a/amd64_raw_qq_to_float_test.go b/amd64_raw_qq_to_float_test.go new file mode 100644 index 000000000..6b6e4452e --- /dev/null +++ b/amd64_raw_qq_to_float_test.go @@ -0,0 +1,283 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func x86RawQQToPSEncoding(unsigned bool, vectorBits, mask int, zeroing, broadcast, memory bool) []byte { + p1, opcode := byte(0xfc), byte(0x5b) + if unsigned { + p1, opcode = 0xff, 0x7a + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc1) // source register 1, destination register 0. + if memory { + modRM = 0x40 // disp8(AX), destination register 0. + } + code := []byte{0x62, 0xf1, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func x86RawQQToPDEncoding(unsigned bool, vectorBits, mask int, zeroing, broadcast, memory bool) []byte { + code := x86RawQQToPSEncoding(unsigned, vectorBits, mask, zeroing, broadcast, memory) + code[2] = 0xfe // EVEX.W1, F3 mandatory prefix. + if !unsigned { + code[4] = 0xe6 + } + return code +} + +func TestDecodeX86RawQQToPSRampAVX512Regression(t *testing.T) { + // Go's assembler decodes these bytes as VCVTQQ2PS Z5, Y16. + code := []byte{0x62, 0xe1, 0xfc, 0x48, 0x5b, 0xc5, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "ramp AVX512 QQ to PS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VCVTQQ2PS" || + decoded[0].Args[0].Reg != "Z5" || decoded[0].Args[1].Reg != "Y16" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawQQToPDRampAVX512Regression(t *testing.T) { + // Go's assembler decodes these bytes as VCVTQQ2PD Z4, Z12. + code := []byte{0x62, 0x71, 0xfe, 0x48, 0xe6, 0xe4, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "ramp AVX512 QQ to PD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VCVTQQ2PD" || + decoded[0].Args[0].Reg != "Z4" || decoded[0].Args[1].Reg != "Z12" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawQQToPSCompleteGoForms(t *testing.T) { + for _, unsigned := range []bool{false, true} { + base := "VCVTQQ2PS" + if unsigned { + base = "VCVTUQQ2PS" + } + for width := 0; width < 3; width++ { + baseOp := base + [...]string{"X", "Y", ""}[width] + source := [...]string{"X1", "Y1", "Z1"}[width] + destination := [...]string{"X0", "X0", "Y0"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + op string + }{ + {name: "register", op: baseOp}, + {name: "masked register", mask: 3, op: baseOp}, + {name: "zero register", mask: 7, zeroing: true, op: baseOp + ".Z"}, + {name: "memory", memory: true, op: baseOp}, + {name: "broadcast memory", memory: true, broadcast: true, op: baseOp + ".BCST"}, + {name: "broadcast zero memory", mask: 7, zeroing: true, memory: true, broadcast: true, op: baseOp + ".BCST.Z"}, + } { + name := fmt.Sprintf("%s/width%d/%s", base, width, form.name) + t.Run(name, func(t *testing.T) { + code := x86RawQQToPSEncoding(unsigned, width, form.mask, form.zeroing, form.broadcast, form.memory) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != Op(form.op) || got.Args[len(got.Args)-1].String() != destination { + t.Fatalf("decode %x = %+v, want %s ... %s", code, got, form.op, destination) + } + if !form.memory && got.Args[0].String() != source { + t.Fatalf("decode %x source = %s, want %s", code, got.Args[0], source) + } + if len(got.Args) != 2+map[bool]int{true: 1}[form.mask != 0] { + t.Fatalf("decode %x operands = %+v", code, got.Args) + } + }) + } + } + for rounding := 0; rounding < 4; rounding++ { + code := x86RawQQToPSEncoding(unsigned, rounding, 4, true, true, false) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + want := base + [...]string{".RN_SAE.Z", ".RD_SAE.Z", ".RU_SAE.Z", ".RZ_SAE.Z"}[rounding] + if err != nil || !ok || length != len(code) || string(got.Op) != want || + got.Args[0].String() != "Z1" || got.Args[2].String() != "Y0" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s Z1, K4, Y0", code, got, length, ok, err, want) + } + } + } +} + +func TestDecodeX86RawQQToPDCompleteGoForms(t *testing.T) { + for _, unsigned := range []bool{false, true} { + base := "VCVTQQ2PD" + if unsigned { + base = "VCVTUQQ2PD" + } + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + suffix string + }{ + {name: "register"}, + {name: "masked zero register", mask: 5, zeroing: true, suffix: ".Z"}, + {name: "memory", memory: true}, + {name: "broadcast memory", memory: true, broadcast: true, suffix: ".BCST"}, + {name: "broadcast zero memory", memory: true, broadcast: true, mask: 7, zeroing: true, suffix: ".BCST.Z"}, + } { + name := fmt.Sprintf("%s/width%d/%s", base, width, form.name) + t.Run(name, func(t *testing.T) { + code := x86RawQQToPDEncoding(unsigned, width, form.mask, form.zeroing, form.broadcast, form.memory) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != Op(base+form.suffix) || + got.Args[len(got.Args)-1].String() != vector+"0" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if !form.memory && got.Args[0].String() != vector+"1" { + t.Fatalf("decode %x source = %s, want %s1", code, got.Args[0], vector) + } + }) + } + } + for rounding := 0; rounding < 4; rounding++ { + code := x86RawQQToPDEncoding(unsigned, rounding, 4, true, true, false) + got, length, ok, err := decodedX86RawQQToFloatInstruction(code, 64) + want := base + [...]string{".RN_SAE.Z", ".RD_SAE.Z", ".RU_SAE.Z", ".RZ_SAE.Z"}[rounding] + if err != nil || !ok || length != len(code) || string(got.Op) != want || + got.Args[0].String() != "Z1" || got.Args[2].String() != "Z0" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %s Z1, K4, Z0", code, got, length, ok, err, want) + } + } + } +} + +func TestDecodeX86RawQQToFloatRIPLiteral(t *testing.T) { + for _, kind := range []struct { + name string + encode func(bool, int, int, bool, bool, bool) []byte + }{ + {name: "PS", encode: x86RawQQToPSEncoding}, + {name: "PD", encode: x86RawQQToPDEncoding}, + } { + for _, unsigned := range []bool{false, true} { + for width := 0; width < 3; width++ { + for _, broadcast := range []bool{false, true} { + code := kind.encode(unsigned, width, 1, false, broadcast, false) + code[5] = 0x05 // RIP-relative source, destination 0. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + if broadcast { + literalWidth = 8 + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/unsigned%t/width%d/broadcast%t", kind.name, unsigned, width, broadcast) + t.Run(name, func(t *testing.T) { + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !decoded[0].x86RIPLiteral || + len(decoded[0].x86RIPLiteralData) != literalWidth || + broadcast != strings.Contains(string(decoded[0].Op), ".BCST") { + t.Fatalf("decoded %x as %+v", code, decoded) + } + }) + } + } + } + } +} + +func TestTranslateX86RawQQToFloatLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawQQToFloat(SB),$0-0\n") + appendBytes := func(code []byte) { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + for _, encode := range []func(bool, int, int, bool, bool, bool) []byte{ + x86RawQQToPSEncoding, + x86RawQQToPDEncoding, + } { + for _, unsigned := range []bool{false, true} { + for width := 0; width < 3; width++ { + appendBytes(encode(unsigned, width, 0, false, false, false)) + appendBytes(encode(unsigned, width, 0, false, false, true)) + appendBytes(encode(unsigned, width, 0, false, true, true)) + if target.goarch == "amd64" { + appendBytes(encode(unsigned, width, 7, true, false, false)) + appendBytes(encode(unsigned, width, 7, true, true, true)) + } + } + if target.goarch == "amd64" { + for rounding := 0; rounding < 4; rounding++ { + appendBytes(encode(unsigned, rounding, 4, true, true, false)) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawQQToFloat": {Name: "rawQQToFloat", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-qq-to-float.ll", "raw-qq-to-float.o", ir) + }) + } +} + +func TestDecodeX86RawQQToPSRejectsInvalidForms(t *testing.T) { + zeroWithoutMask := x86RawQQToPSEncoding(false, 0, 0, true, false, false) + reservedLength := x86RawQQToPSEncoding(false, 3, 0, false, false, false) + reservedVVVV := x86RawQQToPSEncoding(false, 0, 0, false, false, false) + reservedVVVV[2] &^= 0x08 + for _, code := range [][]byte{zeroWithoutMask, reservedLength, reservedVVVV} { + if _, _, ok, err := decodedX86RawQQToFloatInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid %x returned ok=%v, err=%v", code, ok, err) + } + } +} diff --git a/amd64_raw_qword_permute.go b/amd64_raw_qword_permute.go new file mode 100644 index 000000000..fa0bf1291 --- /dev/null +++ b/amd64_raw_qword_permute.go @@ -0,0 +1,118 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go's _yvpermq table gives VPERMQ and VPERMPD the same operand grammar. +// The 0F3A encodings use imm8 control; the EVEX-only 0F38 encodings use a +// vector control operand. The table below keeps the opcode split explicit. +var x86RawQwordPermutes = map[[2]int]Op{ + {3, 0x00}: "VPERMQ", + {3, 0x01}: "VPERMPD", + {2, 0x36}: "VPERMQ", + {2, 0x16}: "VPERMPD", +} + +func decodedX86RawQwordPermuteInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.pp != 1 || !p.w { + return Instr{}, 0, false, nil + } + op, known := x86RawQwordPermutes[[2]int{p.mapNumber, p.opcode}] + if !known { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("qword permute: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength < 1 || p.vectorLength > 2 || !p.evex && p.vectorLength != 1 { + return fail("only Y and Z vector lengths are in Go's qword permute table") + } + if p.evex && !p.fixed { + return fail("invalid EVEX fixed bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + immediate := p.mapNumber == 3 + if !immediate && !p.evex { + return fail("variable-control form requires EVEX") + } + if immediate && p.upper != 0 { + return fail("immediate form reserves vvvv and V'") + } + registerSource := code[p.modRM]>>6 == 3 + if p.broadcast && (registerSource || !p.evex) { + return fail("broadcast requires EVEX memory source") + } + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (!p.evex && destinationNumber >= 8 || + width == "Z" && (destinationNumber >= 8 || !immediate && p.upper >= 8)) { + return fail("destination or control register unavailable in 32-bit mode") + } + + scale := 16 << p.vectorLength + if p.broadcast { + scale = 8 + } + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width, scale) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + } + if err != nil { + return Instr{}, 0, true, err + } + if mode == 32 && width == "Z" && source.Kind == OpReg { + index, _ := amd64VectorRegisterIndex(source.Reg, 64) + if index >= 8 { + return fail("Z source register unavailable in 32-bit mode") + } + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))} + args := make([]Operand, 0, 4) + if immediate { + if len(code) <= p.modRM+consumed { + return fail("missing immediate byte") + } + args = append(args, Operand{Kind: OpImm, Imm: int64(code[p.modRM+consumed])}) + } else { + args = append(args, source) + source = Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, p.upper))} + } + args = append(args, source) + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + length := p.modRM + consumed + if immediate { + length++ + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, length, true, nil +} diff --git a/amd64_raw_qword_permute_test.go b/amd64_raw_qword_permute_test.go new file mode 100644 index 000000000..ad4c45e20 --- /dev/null +++ b/amd64_raw_qword_permute_test.go @@ -0,0 +1,149 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawQwordPermuteGoEncoderForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawqwordpermute(SB),4,$0-0\n") + for _, op := range []string{"VPERMQ", "VPERMPD"} { + fmt.Fprintf(&source, "%s $216, Y2, Y2\n", op) + fmt.Fprintf(&source, "%s $1, 64(BX), Y3\n", op) + fmt.Fprintf(&source, "%s $2, Y20, Y21\n", op) + if arch == "amd64" { + fmt.Fprintf(&source, "%s $3, Z20, Z21\n", op) + } else { + fmt.Fprintf(&source, "%s $3, Z0, Z1\n", op) + } + fmt.Fprintf(&source, "%s.BCST $4, 8(BX), Y21\n", op) + fmt.Fprintf(&source, "%s Y0, Y1, Y2\n", op) + fmt.Fprintf(&source, "%s (BX), Y1, Y2\n", op) + fmt.Fprintf(&source, "%s Y20, Y21, Y22\n", op) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST 16(BX), Z21, Z22\n", op) + } else { + fmt.Fprintf(&source, "%s.BCST 16(BX), Z1, Z2\n", op) + } + if arch == "amd64" { + fmt.Fprintf(&source, "%s.Z $5, -32(R8), K2, Z21\n", op) + fmt.Fprintf(&source, "%s.Z Z20, Z21, K3, Z22\n", op) + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } + }) + } +} + +func TestX86RawQwordPermuteRejectsMalformedForms(t *testing.T) { + const source = "TEXT rawqwordpermute(SB),4,$0-0\nVPERMQ $1, Y2, Y3\nRET\n" + assembled := assembleX87ControlBytes(t, "amd64", source) + if len(assembled) < 7 || assembled[0] != 0xc4 { + t.Fatalf("Go did not emit a VEX permute: %x", assembled) + } + valid := assembled[:len(assembled)-1] + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing immediate", func(code []byte) []byte { return code[:len(code)-1] }}, + {"reserved vvvv", func(code []byte) []byte { code[2] &^= 0x08; return code }}, + {"wrong vector length", func(code []byte) []byte { code[2] &^= 0x04; return code }}, + {"address override", func(code []byte) []byte { return append([]byte{0x67}, code...) }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), valid...)) + if _, _, recognized, err := decodedX86RawQwordPermuteInstruction(code, 64); !recognized || err == nil { + t.Fatalf("invalid %x: recognized=%v err=%v", code, recognized, err) + } + }) + } + const evexSource = "TEXT rawqwordpermute(SB),4,$0-0\nVPERMPD.Z Z1, Z2, K1, Z3\nRET\n" + evex := assembleX87ControlBytes(t, "amd64", evexSource) + if len(evex) < 7 || evex[0] != 0x62 { + t.Fatalf("Go did not emit an EVEX permute: %x", evex) + } + for _, test := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing mask", func(code []byte) []byte { code[3] &^= 7; return code }}, + {"invalid vector length", func(code []byte) []byte { code[3] &^= 0x60; return code }}, + {"missing fixed bit", func(code []byte) []byte { code[2] &^= 4; return code }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.mutate(append([]byte(nil), evex[:len(evex)-1]...)) + if _, _, recognized, err := decodedX86RawQwordPermuteInstruction(code, 64); !recognized || err == nil { + t.Fatalf("invalid %x: recognized=%v err=%v", code, recognized, err) + } + }) + } +} + +func TestTranslateX86RawQwordPermuteLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, form := range []string{ + "VPERMQ $216, Y2, Y3", + "VPERMPD Z1, Z2, Z3", + } { + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(form+"/"+target.triple, func(t *testing.T) { + source := "TEXT rawqwordpermute(SB),4,$0-0\n" + form + "\nRET\n" + code := assembleX87ControlBytes(t, target.goarch, source) + var raw strings.Builder + raw.WriteString("TEXT rawqwordpermute(SB),4,$0-0\n") + for _, value := range code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", value) + } + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawqwordpermute": {Name: "rawqwordpermute", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-qword-permute.ll", "raw-qword-permute.o", ir) + }) + } + } +} diff --git a/amd64_raw_rdpid_test.go b/amd64_raw_rdpid_test.go new file mode 100644 index 000000000..dafd8d26f --- /dev/null +++ b/amd64_raw_rdpid_test.go @@ -0,0 +1,61 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodeX86RawRDPIDCompleteRegisterFamily(t *testing.T) { + registers := [...]string{ + "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI", + "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15", + } + for _, mode := range []int{32, 64} { + count := 8 + if mode == 64 { + count = len(registers) + } + for register := 0; register < count; register++ { + code := []byte{0xf3} + if register >= 8 { + code = append(code, 0x41) + } + code = append(code, 0x0f, 0xc7, byte(0xf8|register&7)) + decoded, err := decodeX86RawDirectiveGroup(code, mode, 0, "RDPID raw bytes", nil) + if err != nil { + t.Fatalf("mode %d, %x: %v", mode, code, err) + } + if len(decoded) != 1 || decoded[0].Op != "RDPID" || len(decoded[0].Args) != 1 || + decoded[0].Args[0].Reg != Reg(registers[register]) { + t.Fatalf("mode %d, %x decoded as %+v", mode, code, decoded) + } + } + } +} + +func TestTranslateX86RawRDPIDReportedByteArena(t *testing.T) { + const source = `TEXT rawRDPID(SB),$0-0 + BYTE $0xF3; BYTE $0x0F; BYTE $0xC7; BYTE $0xF9 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawRDPID": {Name: "rawRDPID", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.x86.rdpid()") { + t.Fatalf("raw RDPID did not call the RDPID intrinsic:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-rdpid.ll", "raw-rdpid.o", ir) +} diff --git a/amd64_raw_return_fp_width_contract_test.go b/amd64_raw_return_fp_width_contract_test.go new file mode 100644 index 000000000..1ade2f726 --- /dev/null +++ b/amd64_raw_return_fp_width_contract_test.go @@ -0,0 +1,228 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86RawReturnScalarFrame(word int, typ LLVMType) FuncSig { + return FuncSig{Name: "Raw", Args: []LLVMType{typ}, Ret: typ, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: int64(word), Type: typ, Index: 0, Field: -1}}, + }} +} + +// A byte/word write to a wider FP slot must preserve the untouched bytes. +// The bounded raw-return contract does not establish that partial-width +// lowering, even when the access is inside the declared scalar's bounds. +// These original returning sources are never executed or rewritten as passes. +func TestX86RawReturnPartialFPAccessNeedsContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + partialMoves := []string{"MOVB", "MOVW"} + if word == 8 { + partialMoves = append(partialMoves, "MOVL") + } + for _, partial := range partialMoves { + for _, access := range []string{ + partial + " $17,x+0(FP)\n" + move + " x+0(FP),AX", + partial + " x+0(FP),AX", + fmt.Sprintf("%s $17,ret+%d(FP)", partial, word), + fmt.Sprintf("%s ret+%d(FP),AX", partial, word), + } { + for _, encoding := range []string{"BYTE $0xc3", "BYTE $0xc2\nWORD $0"} { + t.Run(target.triple+"/"+access+"/"+encoding, func(t *testing.T) { + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s\n%s\nRET\n", 2*word, access, encoding) + requireX86GoAssemblerResult(t, target.arch, source, true) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": x86RawReturnScalarFrame(word, typ)}} + x86RawReturnPointerAPIs(t, source, opt, true) + }) + } + } + } + } +} + +func TestX86RawReturnActualGoDeclarationRejectsPartialFPStore(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, declaration := "MOVQ", 8, "func Raw(x uint64) uint64" + if target.arch == "386" { + move, word, declaration = "MOVL", 4, "func Raw(x uint32) uint32" + } + source := fmt.Sprintf("TEXT ·Raw(SB),4,$0-%d\nMOVB $17,x+0(FP)\n%s x+0(FP),AX\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 2*word, move, move, word) + requireX86GoAssemblerResult(t, target.arch, source, true) + pkg := mustGoPackage(t, "test/rawret", "package rawret\n"+declaration+"\n") + options := GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }} + ctx := llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), options) + if translation != nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s actual declaration authorized an unproved partial FP store: %v", target.triple, err) + } + } +} + +func TestX86RawReturnFrameSlotMustMatchSignature(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, test := range []struct { + name string + mutate func(*FuncSig) + }{ + {"overlap-parameter", func(sig *FuncSig) { + sig.Args = append(sig.Args, I32) + sig.Frame.Params = append(sig.Frame.Params, FrameSlot{Offset: int64(word / 2), Type: I32, Index: 1, Field: -1}) + }}, + {"overlap-result", func(sig *FuncSig) { sig.Frame.Results[0].Offset = int64(word / 2) }}, + {"parameter-type", func(sig *FuncSig) { sig.Args[0] = I8 }}, + {"parameter-scalar-field", func(sig *FuncSig) { sig.Frame.Params[0].Field = 0 }}, + {"parameter-scalar-path", func(sig *FuncSig) { sig.Frame.Params[0].Fields = []int{0} }}, + {"parameter-field-type", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ i8, ptr }") + sig.Frame.Params[0].Field = 0 + }}, + {"parameter-field-index", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Params[0].Field = 2 + }}, + {"parameter-negative-path", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Params[0].Fields = []int{-1} + }}, + {"parameter-nested-path", func(sig *FuncSig) { + sig.Args[0] = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Params[0].Fields = []int{1, 0} + }}, + {"result-type", func(sig *FuncSig) { sig.Ret = I8 }}, + {"result-scalar-field", func(sig *FuncSig) { sig.Frame.Results[0].Field = 0 }}, + {"result-scalar-path", func(sig *FuncSig) { sig.Frame.Results[0].Fields = []int{0} }}, + {"result-tuple-type", func(sig *FuncSig) { sig.Ret = "{ i8, ptr }" }}, + {"result-tuple-field", func(sig *FuncSig) { + sig.Ret = LLVMType("{ " + string(typ) + ", ptr }") + sig.Frame.Results[0].Field = 1 + }}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+0(FP),AX\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 2*word, move, move, word) + requireX86GoAssemblerResult(t, target.arch, source, true) + sig := x86RawReturnScalarFrame(word, typ) + test.mutate(&sig) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": sig}}, true) + }) + } + } +} + +func TestX86RawReturnBoundAggregateScalarSlotObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, explicitPath := range []bool{false, true} { + for _, tupleResult := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/path%t/tuple%t", target.triple, explicitPath, tupleResult), func(t *testing.T) { + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+%d(FP),AX\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 3*word, move, word, move, 2*word) + requireX86GoAssemblerResult(t, target.arch, source, true) + sig := FuncSig{Name: "Raw", Args: []LLVMType{LLVMType("{ ptr, " + string(typ) + " }")}, Ret: typ, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: int64(word), Type: typ, Index: 0, Field: 1}}, + Results: []FrameSlot{{Offset: int64(2 * word), Type: typ, Index: 0, Field: -1}}, + }} + if explicitPath { + sig.Frame.Params[0].Field = -1 + sig.Frame.Params[0].Fields = []int{1} + } + if tupleResult { + sig.Ret = LLVMType("{ ptr, " + string(typ) + " }") + sig.Frame.Results = []FrameSlot{ + {Offset: int64(2 * word), Type: Ptr, Index: 0, Field: -1}, + {Offset: int64(3 * word), Type: typ, Index: 1, Field: -1}, + } + source = fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+%d(FP),AX\n%s $0,ret0+%d(FP)\n%s AX,ret1+%d(FP)\nBYTE $0xc3\nRET\n", 4*word, move, word, move, 2*word, move, 3*word) + requireX86GoAssemblerResult(t, target.arch, source, true) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Raw": sig}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "bound-field.ll", "bound-field.o", ir) + }) + } + } + } +} + +// Object evidence only: this independent UD2 harness cannot return to a caller +// and is never executed. It exposes existing generic partial FP lowering; a +// compiled object does not certify preservation of bytes outside the write. +func TestX86PartialFPStoreCompileOnlyEvidence(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + word, typ := 8, I64 + if target.arch == "386" { + word, typ = 4, I32 + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\nMOVB $17,x+0(FP)\nUD2\nRET\n", 2*word) + dir := t.TempDir() + asm, object := filepath.Join(dir, "partial.s"), filepath.Join(dir, "partial.o") + if err := os.WriteFile(asm, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-p", "example.com/partial", "-o", object, asm) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+target.arch) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("Go assembler failed: %v\n%s", err, out) + } + out, err := exec.Command("go", "tool", "objdump", "-s", "Raw", object).CombinedOutput() + if err != nil || !strings.Contains(string(out), "MOVB $0x11") { + t.Fatalf("Go object did not retain the one-byte write: %v\n%s", err, out) + } + t.Logf("actual Go object, not executed:\n%s", out) + ir, err := Translate(mustParseX86RawReturn(t, source), Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": x86RawReturnScalarFrame(word, typ)}, + }) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(ir, "\n") { + if strings.Contains(line, "17") || strings.Contains(line, "fp_arg") || strings.Contains(line, "zext i8") || + strings.Contains(line, "store "+string(typ)+" %") || strings.Contains(line, "classic_frame") { + t.Log("generic lowering observation (not a semantic pass):", line) + } + } + compileLLVMToObject(t, llc, target.triple, "partial-compile-only.ll", "partial-compile-only.o", ir) + }) + } +} diff --git a/amd64_raw_return_pointer_contract_test.go b/amd64_raw_return_pointer_contract_test.go new file mode 100644 index 000000000..ffe4b0785 --- /dev/null +++ b/amd64_raw_return_pointer_contract_test.go @@ -0,0 +1,300 @@ +package plan9asm + +import ( + "errors" + "fmt" + "sort" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// These are compile-only contract counterexamples, not programs that mutate a +// live return address. Go acceptance proves source grammar, never alias safety. +func x86RawReturnPointerAPIs(t *testing.T, source string, opt Options, wantContext bool) { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, api := range []string{"text", "module", "owned-module", "annotated"} { + var callErr error + switch api { + case "text": + _, callErr = Translate(file, opt) + case "module": + module, err := TranslateModule(file, opt) + if err == nil { + module.Dispose() + } + callErr = err + default: + ctx := llvm.NewContext() + options := opt + options.AnnotateSource = api == "annotated" + module, err := TranslateModuleInContext(ctx, file, options) + if err == nil { + module.Dispose() + } + ctx.Dispose() + callErr = err + } + if wantContext && !errors.Is(callErr, ErrProbeNeedsContext) { + t.Errorf("%s did not reject an unproved native continuation: %v", api, callErr) + } + if !wantContext && callErr != nil { + t.Errorf("%s rejected a bounded FP/static leaf: %v", api, callErr) + } + } +} + +func TestX86RawReturnPointerMemoryNeedsContinuationContract(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + instructions := []string{ + "MOVB $0,(AX)", "MOVW $0,1(AX)", "MOVL $0,4(AX)", + "MOVB (AX),AL", "ADDL $1,(AX)", "XCHGL AX,(AX)", + "MOVOU X0,(AX)", "SETEQ (AX)", "BTSL $1,(AX)", + "VMASKMOVPS X0,X1,(AX)", "VMASKMOVPD X0,X1,(AX)", + "VPMASKMOVD X0,X1,(AX)", "VPMASKMOVQ X0,X1,(AX)", + "VMOVDQU32 X0,K1,(AX)", + move + " AX,BX\nMOVB $0,(BX)", + move + " $saved<>(SB),AX\nMOVB $0,(AX)", + "MOVB $0,saved<>(SB)(AX*1)", + "MOVB $0,0(AX)(CX*2)", + } + if target.arch == "amd64" { + instructions = append(instructions, "MOVQ $0,8(AX)") + } + for _, instruction := range instructions { + t.Run(target.triple+"/"+instruction, func(t *testing.T) { + source := fmt.Sprintf("GLOBL saved<>(SB),8,$32\nTEXT Raw(SB),4,$0-%d\n%s p+0(FP),AX\n%s\nBYTE $0xc3\nRET\n", word, move, instruction) + // Go permits indexed static-base addressing in 386, but rejects + // this spelling in amd64. Keep the latter as a defensive parsed + // source negative, not a positive Go format witness. + goAccepted := target.arch == "386" || instruction != "MOVB $0,saved<>(SB)(AX*1)" + requireX86GoAssemblerResult(t, target.arch, source, goAccepted) + sig := FuncSig{Name: "Raw", Args: []LLVMType{Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}}} + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": sig}}, true) + }) + } + } +} + +func TestX86RawReturnImplicitMemoryNeedsContinuationContract(t *testing.T) { + instructions := []string{"XLAT", "MASKMOVQ M0,M1", "MASKMOVOU X0,X1", "MASKMOVDQU X0,X1", "VMASKMOVDQU X0,X1"} + for op := range x86StringSpecs { + instructions = append(instructions, string(op)) + } + for _, op := range []string{"INSB", "INSW", "INSL", "OUTSB", "OUTSW", "OUTSL"} { + instructions = append(instructions, op) + } + sort.Strings(instructions) + for _, target := range x86RawReturnTargets { + for _, instruction := range instructions { + if target.arch == "386" && strings.HasSuffix(instruction, "Q") && !strings.HasPrefix(instruction, "MASK") { + continue // Go's 32-bit string table has no Q-width forms. + } + t.Run(target.triple+"/"+instruction, func(t *testing.T) { + source := "TEXT Raw(SB),4,$0-0\n" + instruction + "\nBYTE $0xc3\nRET\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Ret: Void}}}, true) + }) + } + } +} + +func TestX86RawReturnFPRequiresBoundTypedSlot(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, frame := range []FrameLayout{ + {}, {Params: []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}}}, + } { + offset := 0 + if len(frame.Params) != 0 { + offset = 100 + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+%d(FP),AX\nBYTE $0xc3\nRET\n", word, move, offset) + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ, Frame: frame}}}, true) + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s AX,ret+%d(FP)\nBYTE $0xc3\nRET\n", 2*word, move, word) + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ, + Frame: FrameLayout{Results: []FrameSlot{{Offset: int64(word), Type: typ, Index: 1, Field: -1}}}}}}, true) + } +} + +func TestX86RawReturnMemoryRangeAndUnknownShapeNeedContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, test := range []struct { + name, instruction string + slot FrameSlot + }{ + {"fp-too-narrow", move + " x+0(FP),AX", FrameSlot{Type: I8, Index: 0, Field: -1}}, + {"fp-unrecognized-type", move + " x+0(FP),AX", FrameSlot{Type: "i128", Index: 0, Field: -1}}, + {"fp-unbound-index", move + " x+0(FP),AX", FrameSlot{Type: typ, Index: 1, Field: -1}}, + {"static-before-region", "MOVL AX,saved<>-1(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + {"static-after-region", "MOVL AX,saved<>+32(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + {"static-cross-end", "MOVL AX,saved<>+30(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + {"static-no-object-contract", "MOVL AX,unknown(SB)", FrameSlot{Type: typ, Index: 0, Field: -1}}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source := fmt.Sprintf("GLOBL saved<>(SB),8,$32\nTEXT Raw(SB),4,$0-%d\n%s\nBYTE $0xc3\nRET\n", word, test.instruction) + requireX86GoAssemblerResult(t, target.arch, source, true) + x86RawReturnPointerAPIs(t, source, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ, + Frame: FrameLayout{Params: []FrameSlot{test.slot}}}}}, true) + }) + } + } +} + +func TestX86RawReturnBoundFPAndStaticObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + for _, encoding := range []string{"BYTE $0xc3", "BYTE $0xc2\nWORD $0"} { + for _, pointer := range []bool{false, true} { + valueType := typ + if pointer { + valueType = Ptr + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+0(FP),AX\n%s AX,ret+%d(FP)\n%s\nRET\n", 2*word, move, move, word, encoding) + requireX86GoAssemblerResult(t, target.arch, source, true) + sig := FuncSig{Name: "Raw", Args: []LLVMType{valueType}, Ret: valueType, + Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: valueType, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: int64(word), Type: valueType, Index: 0, Field: -1}}}} + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Raw": sig}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "fp-return.ll", "fp-return.o", ir) + } + source := "GLOBL saved<>(SB),8,$32\nTEXT Raw(SB),4,$0-0\nMOVL $17,AX\nMOVL AX,saved<>+4(SB)\nMOVL saved<>+4(SB),AX\n" + encoding + "\nRET\n" + requireX86GoAssemblerResult(t, target.arch, source, true) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Ret: I32}}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "static-return.ll", "static-return.o", ir) + } + } +} + +func TestX86RawReturnPrepartitionRetainsFinalFrameValidation(t *testing.T) { + for _, target := range x86RawReturnTargets { + move, word, typ := "MOVQ", 8, I64 + if target.arch == "386" { + move, word, typ = "MOVL", 4, I32 + } + source := fmt.Sprintf("TEXT Raw(SB),4,$0-%d\n%s x+0(FP),AX\nBYTE $0xc3\nRET\n", word, move) + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), target.arch) + if err != nil { + t.Fatal("pure preparation must not discard the future typed frame:", err) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Raw": {Name: "Raw", Args: []LLVMType{typ}, Ret: typ}}} + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("prepartition preparation erased the final missing-frame contract:", err) + } + sig := opt.Sigs["Raw"] + sig.Frame.Params = []FrameSlot{{Offset: 0, Type: typ, Index: 0, Field: -1}} + opt.Sigs["Raw"] = sig + ctx = llvm.NewContext() + module, err = TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if err != nil { + t.Fatal("existing consumer's bound signature was not consumed:", err) + } + } +} + +func TestX86RawReturnActualGoDeclarationContract(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc not found") + } + for _, target := range x86RawReturnTargets { + move, word := "MOVQ", 8 + if target.arch == "386" { + move, word = "MOVL", 4 + } + for _, test := range []struct { + name, declaration, body string + argSize int + context bool + }{ + {"pointer-store", "func Raw(*byte)", move + " p+0(FP),AX\nMOVB $0,(AX)", word, true}, + {"unknown-fp", "func Raw(uintptr) uintptr", move + " p+100(FP),AX", 2 * word, true}, + {"typed-pointer-transport", "func Raw(*byte) *byte", fmt.Sprintf("%s p+0(FP),AX\n%s AX,ret+%d(FP)", move, move, word), 2 * word, false}, + {"typed-static-global", "func Raw() uint32", "MOVL $17,AX\nMOVL AX,saved<>+4(SB)\nMOVL saved<>+4(SB),AX\nMOVL AX,ret+0(FP)", 4, false}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source := fmt.Sprintf("GLOBL saved<>(SB),8,$32\nTEXT ·Raw(SB),4,$0-%d\n%s\nBYTE $0xc3\nRET\n", test.argSize, test.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + pkg := mustGoPackage(t, "test/rawret", "package rawret\n"+test.declaration+"\n") + options := GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple, + ResolveSym: func(s string) string { return strings.TrimPrefix(s, "·") }} + tr, err := TranslateGoModule(pkg, []byte(source), options) + if tr != nil { + defer tr.Module.Dispose() + } + if test.context { + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("a Go pointer/declaration invented a return-PC contract:", err) + } + return + } + if err != nil { + t.Fatal("actual declaration's bounded typed frame rejected:", err) + } + compileLLVMToObject(t, llc, target.triple, "declared-return.ll", "declared-return.o", tr.Module.String()) + }) + } + } +} + +func mustParseX86RawReturn(t *testing.T, source string) *File { + t.Helper() + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + return file +} diff --git a/amd64_raw_return_source_address_runtime_test.go b/amd64_raw_return_source_address_runtime_test.go new file mode 100644 index 000000000..07315cbe1 --- /dev/null +++ b/amd64_raw_return_source_address_runtime_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// The original native reference executes Go's actual LEA and raw C3 bytes, +// not an encoding manufactured by the translator. Register arguments here +// are an explicit numerical custom ABI, never a native-pointer permission. +func TestCrossLinuxRuntimeMatrixX86RawReturnSourceAddressNumerical(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareSSE) + var assembly, oracle, checks strings.Builder + oracle.WriteString(`#include +#include +#if defined(__APPLE__) +#define LABEL(name) "_" name +#else +#define LABEL(name) name +#endif +`) + sigs := make(map[string]FuncSig) + for _, width := range []string{"L", "Q"} { + for _, scale := range []int{1, 2, 4, 8} { + name := fmt.Sprintf("address%s%d", width, scale) + source := fmt.Sprintf("TEXT %s(SB),4,$0-0\nMOV%s $-7(AX)(CX*%d),AX\nBYTE $0xc3\nRET\n", name, width, scale) + assembly.WriteString(source) + code := x86PackedShiftActualGoBytes(t, "amd64", source) + instruction, err := x86asm.Decode(code, 64) + if err != nil || instruction.Op != x86asm.LEA || instruction.Len >= len(code) || code[instruction.Len] != 0xc3 { + t.Fatalf("actual Go address rewrite/raw return lost: %x / %v", code, err) + } + var bytes strings.Builder + for _, value := range code[:instruction.Len+1] { + if bytes.Len() != 0 { + bytes.WriteByte(',') + } + fmt.Fprintf(&bytes, "0x%02x", value) + } + fmt.Fprintf(&oracle, ` +extern uint64_t %[1]s(uint64_t,uint64_t); +extern uint64_t original%[1]s(uint64_t,uint64_t); +__asm__(".text\n.globl " LABEL("original%[1]s") "\n" + LABEL("original%[1]s") ":\n" + "movq %%rdi,%%rax\nmovq %%rsi,%%rcx\n.byte %[2]s\n"); +`, name, bytes.String()) + want := fmt.Sprintf("a+%d*b-7", scale) + if width == "L" { + want = "(uint64_t)(uint32_t)(" + want + ")" + } + fmt.Fprintf(&checks, ` + { uint64_t want=%[1]s; + if (%[2]s(a,b)!=want || original%[2]s(a,b)!=want) { + fprintf(stderr,"raw RET address %[2]s seed %%u\n",seed); return 1; + } + } +`, want, name) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{I64, I64}, ArgRegs: []Reg{AX, CX}, Ret: I64} + } + } + oracle.WriteString(`int main(void) { + for (unsigned seed=0; seed<256; seed++) { + uint64_t a=UINT64_C(0x1313939382827272)*seed,b=~a; +` + checks.String() + ` + } + puts("RAW_RET_NUMERICAL_ADDRESS_2048_OK"); + return 0; +} +`) + file, err := Parse(ArchAMD64, assembly.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + output := execution.compileAndRunLLVM(t, "raw-return-address", ir, oracle.String(), level) + if string(output) != "RAW_RET_NUMERICAL_ADDRESS_2048_OK\n" { + t.Fatalf("missing complete numerical original/LLVM oracle: %q", output) + } + } +} diff --git a/amd64_raw_return_source_address_test.go b/amd64_raw_return_source_address_test.go new file mode 100644 index 000000000..8ff3cecae --- /dev/null +++ b/amd64_raw_return_source_address_test.go @@ -0,0 +1,131 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Go progedit rewrites the whole MOVL/Q TYPE_ADDR family to LEAL/Q before +// classification. A raw near RET must not prohibit a purely numerical, +// stack-unobserving address calculation or inherit unrelated source proofs. +func TestX86RawReturnSourceAddressLeafThreeAPIs(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + widths := []string{"L"} + if target.arch == "amd64" { + widths = append(widths, "Q") + } + for _, width := range widths { + for _, address := range []string{"8(AX)", "-8(AX)(CX*1)", "16(AX)(CX*2)", "32(AX)(CX*4)", "64(AX)(CX*8)", "8(CX*4)", "(1<<4)(AX)"} { + t.Run(target.triple+"/"+width+"/"+address, func(t *testing.T) { + source := fmt.Sprintf("TEXT ·addr(SB),4,$0-0\nMOVL $3,AX\nMOVL $2,CX\nMOV%s $%s,BX\nBYTE $0xc3\nRET\n", width, address) + x86LEAActualTargetGoObject(t, target.arch, target.triple, source) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + before := fmt.Sprintf("%#v", file.Funcs) + pkg := x86AddSubLiteralPackage(t, target.arch, "func addr()\n") + resolve := func(symbol string) string { return strings.Replace(symbol, "·", "literal.", 1) } + sigs, err := goSigsForAsmFile(pkg, file, resolve, target.arch, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: target.arch, TargetTriple: target.triple, ResolveSym: resolve, Sigs: sigs} + for _, api := range []string{"text", "module", "GoModule"} { + t.Run(api, func(t *testing.T) { + context := llvm.NewContext() + defer context.Dispose() + var ir string + var err error + switch api { + case "text": + ir, err = Translate(file, options) + case "module": + var module llvm.Module + module, err = TranslateModuleInContext(context, file, options) + if !module.IsNil() { + defer module.Dispose() + ir = module.String() + } + case "GoModule": + var result *GoModuleTranslation + result, err = translateGoModuleInContext(context, pkg, []byte(source), GoModuleOptions{ + GOARCH: target.arch, TargetTriple: target.triple, ResolveSym: resolve}) + if result != nil && !result.Module.IsNil() { + defer result.Module.Dispose() + ir = result.Module.String() + } + } + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-return-address.ll", "raw-return-address.o", ir) + if before != fmt.Sprintf("%#v", file.Funcs) { + t.Fatal("derived address changed the original source") + } + }) + } + }) + } + } + } +} + +func TestX86RawReturnSourceAddressCannotInheritProof(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + file, err := Parse(ArchAMD64, "TEXT addr(SB),4,$0-0\nMOVL $8(AX)(CX*4),BX\nBYTE $0xc3\nRET\n") + if err != nil { + t.Fatal(err) + } + options := Options{Goarch: arch, Sigs: map[string]FuncSig{"addr": {Name: "addr", Ret: Void}}} + normalized, err := normalizeX86RawFile(file, arch, options) + if err != nil { + t.Fatal(err) + } + for _, field := range []string{"sourceStack", "fixedLocal", "rawText", "LOCK", "frame", "scalarConstant", "g", "memory", "encodedSibling"} { + t.Run(arch+"/"+field, func(t *testing.T) { + copy := *normalized + copy.Funcs = append([]Func(nil), normalized.Funcs...) + fn := ©.Funcs[0] + fn.Instrs = append([]Instr(nil), fn.Instrs...) + fn.Instrs[1].Args = append([]Operand(nil), fn.Instrs[1].Args...) + switch field { + case "sourceStack": + fn.x86SourceStack = &x86SourceStackProof{} + case "fixedLocal": + fn.x86FixedLocal = &x86FixedLocalProof{} + case "rawText": + fn.X86RawText = []byte{0xc3} + case "LOCK": + fn.Instrs = append(fn.Instrs, Instr{Op: "LOCK"}) + case "frame": + fn.FrameSize = 8 + case "scalarConstant": + fn.Instrs[1].Op = "MOVW" + case "g": + fn.Instrs = append(fn.Instrs, Instr{Op: "MOVQ", Args: []Operand{{Kind: OpReg, Reg: "R14"}, {Kind: OpReg, Reg: AX}}}) + case "memory": + fn.Instrs = append(fn.Instrs, Instr{Op: "MOVL", Args: []Operand{{Kind: OpMem, Mem: MemRef{Base: AX}}, {Kind: OpReg, Reg: DX}}}) + case "encodedSibling": + fn.Instrs = append(fn.Instrs, Instr{Op: "MOVL", Args: []Operand{{Kind: OpImm, Imm: 1}, {Kind: OpReg, Reg: AX}}, x86Encoded: true}) + } + before := fmt.Sprintf("%#v", fn) + if _, err := normalizeX86SourceIntegerAddresses(©, options); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("adapter inherited %s proof or native effect: %v", field, err) + } + if before != fmt.Sprintf("%#v", fn) || !reflect.DeepEqual(file.Funcs[0].Instrs[1].Args, normalized.Funcs[0].Instrs[1].Args) { + t.Fatal("rejection modified original instructions") + } + }) + } + } +} diff --git a/amd64_raw_return_source_stack.go b/amd64_raw_return_source_stack.go new file mode 100644 index 000000000..a0f4c29b8 --- /dev/null +++ b/amd64_raw_return_source_stack.go @@ -0,0 +1,845 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" +) + +// This is a source ABI0 stack, not LLVM's physical stack or an incoming native +// pointer. It is enabled only after all returns and all stack uses are proved. +// Ordinary pointers need a separately consumed complete Go caller proof. +// Nonzero TEXT frames, calls and private-stack address escape cannot use it. +type x86SourceStackProof struct { + minimum int64 + word int64 + caller *x86GoCallerRegionProof + // Exact public normalized source bytes, excluding private certificates. + normalized [32]byte + // Diagnostic-only finite replay counts, never used as permission inputs. + countedIterations [4]int +} + +// Bind raw stack producer identity before consulting Go's named PUSH/POP +// deltasp rules. Edited normalized instructions cannot borrow that origin. +type x86SourceRawInstruction struct { + op Op + args []Operand +} + +type x86SourceStackAddress struct { + // original distinguishes the caller's opaque BP from an entry-SP offset. + original bool + offset int64 +} + +type x86SourceStackCell struct { + offset int64 + width int64 + bp *x86SourceStackAddress + flags bool + numericFlags bool + partialNumericFlags uint8 + caller *x86CallerPointerValue + known uint8 + gOriginal bool +} + +type x86SourceStackState struct { + sp int64 + bp x86SourceStackAddress + cells []x86SourceStackCell + numericFlags bool + partialNumericFlags uint8 + callers map[Reg]x86CallerPointerValue + known map[Reg]uint8 + vectors map[Reg]uint64 + masks map[Reg]bool + gOriginal bool +} + +const x86SourceStackLimit = int64(1 << 20) + +// Recompute the proof on every final normalization. No caller-supplied signature, +// changed source, or prepartition result inherits a cached admission flag. +func normalizeX86RawReturnSourceStack(fn Func, opt Options, bindFrame bool, original ...Func) Func { + input := fn + if len(original) != 0 { + input = original[0] + } + fn.x86SourceStack = nil + fn.x86SourceStackFailure = "" + reject := func(index int, reason string) Func { + fn.x86SourceStackFailure = fmt.Sprintf("normalized instruction %d: %s", index, reason) + return fn + } + if fn.FrameSize != 0 || strings.Contains(fn.Sym, "") { + return fn + } + hasRawReturn := false + for _, ins := range fn.Instrs { + if form := ins.x86RawNearReturn; form != nil { + if !form.nativeWidth || form.cleanup != 0 { + return fn + } + hasRawReturn = true + } + } + if !hasRawReturn { + return fn + } + word, nativeMove, nativeSuffix := int64(8), Op("MOVQ"), "Q" + if opt.Goarch == "386" { + word, nativeMove, nativeSuffix = 4, "MOVL", "L" + } + name := fn.Sym + if opt.ResolveSym != nil { + name = opt.ResolveSym(name) + } + sig, hasSig := opt.Sigs[name] + caller := bindFrame && hasSig && x86CallerRegionSourceMatches(sig, input, opt.Goarch) + if !caller { + for index, instruction := range fn.Instrs { + if _, known, _ := x86ExplicitIMULProperties(instruction.Op, instruction, opt.Goarch); known { + return reject(index, "explicit multiply needs a complete typed caller input proof") + } + if _, known, _ := x86ConditionalMoveProperties(instruction.Op, instruction, opt.Goarch); known { + return reject(index, "conditional move needs a complete typed caller input and flags proof") + } + if _, conditionalSet, _ := x86ConditionalSetProperties(instruction.Op, instruction, opt.Goarch); conditionalSet { + return reject(index, "conditional byte output needs a complete typed caller input proof") + } + if _, extension := amd64ScalarExtensionMoveSpecs[instruction.Op]; extension || instruction.x86RawMOVSXDNeedsCaller { + if x86RawMOVSXDLiteralInput(instruction) { + continue + } + // The old scalar slot contract proves frame bounds, not GP + // input availability. It cannot authorize a new extension read + // of unknown or partially initialized native entry registers. + return reject(index, "scalar extension needs a complete typed caller input proof") + } + } + } + if !caller && x86RawReturnLeafStackUnobserved(fn) { + return fn + } + if bindFrame && (!hasSig || !caller && !x86SourceStackScalarABI0(sig, fn.ArgSize, opt.Goarch)) { + return fn + } + + // Validate the complete source, including dead instructions. A dead CALL + // still changes Go's source prologue; unknown stack effects stay Context. + labels := map[string]int{} + var instructions []int + goDeltas := map[int]int64{} + var goDelta int64 + hasText := false + for index, ins := range fn.Instrs { + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpTEXT { + if hasText || !x86RawReturnLeafStackUnobserved(Func{Instrs: []Instr{ins}}) { + return fn + } + hasText = true + } + if op == OpLABEL { + if len(ins.Args) != 1 || ins.Args[0].Kind != OpLabel { + return fn + } + if _, duplicate := labels[ins.Args[0].Sym]; duplicate { + return fn + } + labels[ins.Args[0].Sym] = len(instructions) + continue + } + vectorMove := caller && x86CallerVectorMoveKnown(ins) + if !x86SourceStackInstructionKnown(ins, nativeMove, nativeSuffix, opt.Goarch) && !vectorMove { + return reject(index, "unregistered source-stack effect: "+string(ins.Op)) + } + if x86LEAWidth(ins.Op, opt.Goarch) != 0 && ins.Op != Op("LEA"+nativeSuffix) && !caller { + // The older scalar stack proof establishes a continuation, not + // numerical GP initialization. Only the actual caller's complete + // must-defined transfer can authorize these additional widths. + return reject(index, "non-native-width LEA requires actual numerical caller source") + } + if opt.Goarch == "amd64" && !caller && x86SourceGOperand(ins) { + if _, stack := amd64StackWidthSpecs[op]; !stack { + return reject(index, "g numerical/address use requires actual caller source") + } + } + if caller && !vectorMove && !x86CallerScalarOperands(ins) { + return reject(index, "unregistered caller-region operand bank: "+string(ins.Op)) + } + if _, stack := amd64StackWidthSpecs[op]; stack && ins.x86SourceRaw != nil && + (ins.x86SourceRaw.op != op || !reflect.DeepEqual(ins.x86SourceRaw.args, ins.Args)) { + return fn + } + for _, operand := range ins.Args { + if operand.Kind == OpFPAddr || (operand.Kind == OpFP && operand.FPOffset < 0) { + return fn + } + if operand.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + // Address-of is not an ordinary numeric immediate, regardless of + // the register spelling or whether its later value is overwritten. + return fn + } + } + instructions = append(instructions, index) + goDeltas[index] = goDelta + // cmd/internal/obj/x86 preprocess adjusts named FP operands using + // its linear deltasp. BYTE/WORD pushes are opaque to that pass. + if spec, stack := amd64StackWidthSpecs[op]; stack && ins.x86SourceRaw == nil { + width := int64(spec.bits / 8) + if spec.mode == amd64StackPushValue || spec.mode == amd64StackPushFlags { + goDelta += width + } else { + goDelta -= width + } + if goDelta < -x86SourceStackLimit || goDelta > x86SourceStackLimit { + return fn + } + } + } + if !hasText || len(instructions) == 0 { + return fn + } + + copyFn := fn + copyFn.Instrs = append([]Instr(nil), fn.Instrs...) + for index := range copyFn.Instrs { + copyFn.Instrs[index].Args = append([]Operand(nil), fn.Instrs[index].Args...) + copyFn.Instrs[index].x86SourceStackFP = nil + copyFn.Instrs[index].x86CallerMemory = nil + } + states := map[int]x86SourceStackState{} + work := []int{0} + states[0] = x86SourceStackState{bp: x86SourceStackAddress{original: true}, gOriginal: true} + proof := &x86SourceStackProof{word: word} + if caller { + proof.caller = sig.x86CallerRegions + states[0] = x86SourceStackState{ + bp: x86SourceStackAddress{original: true}, + callers: make(map[Reg]x86CallerPointerValue), + known: make(map[Reg]uint8), + vectors: make(map[Reg]uint64), + masks: make(map[Reg]bool), + gOriginal: true, + } + } + transfer := func(pc int, ins *Instr, state *x86SourceStackState) (bool, string) { + op := Op(normalizeInstructionOpcode(ins.Op)) + if op == OpRET { + if word == 8 && !state.gOriginal { + return false, "unrestored original g identity at source return" + } + if len(ins.Args) != 0 || state.sp != 0 || !state.bp.original || goDeltas[instructions[pc]] != 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + return true, "" + } + for _, operand := range ins.Args { + if operand.Kind == OpFP && state.sp+goDeltas[instructions[pc]] != 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + } + + if spec, stack := amd64StackWidthSpecs[op]; stack { + if !x86SourceStackStep(spec, ins, state, word) { + return false, "unproved source-stack or ABI0 continuation effect" + } + } else if !x86SourceStackAddressStep(ins, state, nativeMove, nativeSuffix, word) { + return false, "unproved source-stack or ABI0 continuation effect" + } + if state.sp < -x86SourceStackLimit || state.sp > 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + if state.sp < proof.minimum { + proof.minimum = state.sp + } + for operandIndex, operand := range ins.Args { + if operand.Kind != OpMem || !x86RawReturnStackMemory(operand.Mem) { + continue + } + if x86LEAWidth(ins.Op, opt.Goarch) != 0 { + continue // The address step proved its destination and range. + } + width := x86ScalarMoveMemoryBytes(*ins, operandIndex, opt.Goarch) + memory := operand.Mem + if width == 0 || memory.Sym != "" || memory.Index != "" || memory.Segment != "" || + ins.x86AddressBits != 0 { + return false, "unproved source-stack or ABI0 continuation effect" + } + base := state.sp + if memory.Base == BP { + if state.bp.original { + return false, "unproved source-stack or ABI0 continuation effect" + } + base = state.bp.offset + } else if memory.Base != SP { + return false, "unproved source-stack or ABI0 continuation effect" + } + // Source entry SP[0:word] is the return PC. Only declaration-backed + // data at entry SP+word can be mapped to FP shadow storage. + if memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit { + return false, "unproved source-stack or ABI0 continuation effect" + } + offset := base + memory.Off - word + if offset < 0 || offset > fn.ArgSize || width > fn.ArgSize-offset { + return false, "unproved source-stack or ABI0 continuation effect" + } + if bindFrame { + if !x86RawReturnFPBound(offset, width, sig, opt.Goarch) { + return false, "unproved source-stack or ABI0 continuation effect" + } + if ins.x86SourceStackFP == nil { + ins.x86SourceStackFP = map[int]int64{} + } + ins.x86SourceStackFP[operandIndex] = offset + } + } + pointerFlags := false + if caller { + var valid bool + if x86CallerVectorMoveKnown(*ins) { + valid = x86CallerVectorMoveStep(ins, state, sig.x86CallerRegions) + } else { + pointerFlags, valid = x86CallerScalarStep(ins, state, sig.x86CallerRegions) + } + if !valid { + return false, "unproved caller-region effect: " + string(ins.Op) + } + x86CallerGDefinition(*ins, state) + } + if !x86SourceStackFlagsStepForArch(*ins, state, opt.Goarch) { + return false, "unproved source-stack or ABI0 continuation effect" + } + if pointerFlags { + x86SourceStackSetNumericFlags(state, 0) + } + + return false, "" + } + returned := false + acyclic := caller && x86CallerReachableAcyclic(copyFn, instructions, labels) + if caller && !acyclic { + if plan := deriveX86CallerCountedPlan(input, copyFn, instructions, labels); plan != nil { + first, reason := replayX86CallerCountedPlan(plan, ©Fn, instructions, states[0], proof.caller, transfer) + if reason != "" { + return reject(0, reason) + } + minimum := proof.minimum + proof.minimum = 0 + for index := range copyFn.Instrs { + copyFn.Instrs[index].x86CallerMemory = nil + copyFn.Instrs[index].x86SourceStackFP = nil + } + finalPlan := deriveX86CallerCountedPlan(input, copyFn, instructions, labels) + if !reflect.DeepEqual(plan, finalPlan) { + return reject(0, "counted-loop plan changed during independent replay") + } + second, reason := replayX86CallerCountedPlan(finalPlan, ©Fn, instructions, states[0], proof.caller, transfer) + if reason != "" || minimum != proof.minimum || !reflect.DeepEqual(first, second) { + return reject(0, "counted-loop final access/state replay changed: "+reason) + } + for _, visit := range second { + if _, header := finalPlan.headers[visit.pc]; header { + proof.countedIterations[visit.residue]++ + } + } + var err error + proof.normalized, err = x86CallerSourceFingerprint(copyFn) + if err != nil { + return fn + } + copyFn.x86SourceStack = proof + return copyFn + } + } + final := false + for budget := 0; ; budget++ { + if len(work) == 0 { + if !acyclic || final { + break + } + // Discard every provisional certificate. Independently replay all + // final reachable inputs, once, after the finite merge fixed point. + final = true + budget = 0 + for pc, index := range instructions { + copyFn.Instrs[index].x86CallerMemory = nil + copyFn.Instrs[index].x86SourceStackFP = nil + if _, reachable := states[pc]; reachable { + work = append(work, pc) + } + } + } + if budget >= 8192 { + return fn + } + pc := work[0] + work = work[1:] + state := states[pc] + if caller { + state = x86CloneCallerState(state) + } else { + state.cells = append([]x86SourceStackCell(nil), state.cells...) + } + ins := ©Fn.Instrs[instructions[pc]] + ins.x86CallerMemory = nil + ins.x86SourceStackFP = nil + op := Op(normalizeInstructionOpcode(ins.Op)) + isReturn, reason := transfer(pc, ins, &state) + if reason != "" { + return reject(instructions[pc], reason) + } + if isReturn { + returned = true + continue + } + successors := []int{pc + 1} + if op == OpJMP || isAMD64ConditionalBranch(op) { + if len(ins.Args) != 1 { + return fn + } + operand := ins.Args[0] + label := operand.Sym + if operand.Kind == OpIdent { + label = operand.Ident + } + target, known := labels[label] + if operand.Kind == OpMem && operand.Mem.Base == PC && operand.Mem.Index == "" && + operand.Mem.Sym == "" && operand.Mem.Off >= -int64(len(instructions)) && operand.Mem.Off <= int64(len(instructions)) { + target, known = pc+int(operand.Mem.Off), true + } else if operand.Kind != OpLabel && operand.Kind != OpIdent { + return fn + } + if !known { + return fn + } + if op == OpJMP { + successors = []int{target} + } else { + successors = append(successors, target) + } + } + for _, next := range successors { + if next < 0 || next >= len(instructions) { + return fn // Fallthrough cannot invent a source continuation. + } + if old, visited := states[next]; visited { + if !reflect.DeepEqual(old, state) { + if !acyclic { + return reject(instructions[next], "different source-stack or caller-region states at join") + } + merged, valid := x86CallerAcyclicMerge(old, state, sig.x86CallerRegions) + if !valid { + return reject(instructions[next], "different source-stack or caller-region states at join") + } + if reflect.DeepEqual(old, merged) { + continue + } + if final { + return reject(instructions[next], "caller-region changed during final certificate replay") + } + states[next] = merged + work = append(work, next) + } + continue + } + if final { + return reject(instructions[next], "new caller-region state during final certificate replay") + } + states[next] = state + work = append(work, next) + } + } + if !returned { + return fn + } + if caller { + var err error + proof.normalized, err = x86CallerSourceFingerprint(copyFn) + if err != nil { + return fn + } + } + copyFn.x86SourceStack = proof + return copyFn +} + +func x86SourceStackScalarABI0(sig FuncSig, argSize int64, arch string) bool { + if len(sig.ArgRegs) != 0 || argSize <= 0 || argSize > x86SourceStackLimit || len(sig.Frame.Results) != 1 || + len(sig.Frame.Params) != len(sig.Args) || len(sig.Args) == 0 { + return false + } + seen := map[int]bool{} + for _, slot := range sig.Frame.Params { + width := x86FrameScalarBytes(slot.Type, arch) + if !x86SourceStackScalarType(slot.Type) || len(frameSlotFields(slot)) != 0 || + slot.Index < 0 || slot.Index >= len(sig.Args) || slot.Type != sig.Args[slot.Index] || + width == 0 || !x86RawReturnFPBound(slot.Offset, width, sig, arch) || + slot.Offset > argSize || width > argSize-slot.Offset || seen[slot.Index] { + return false + } + seen[slot.Index] = true + } + slot := sig.Frame.Results[0] + width := x86FrameScalarBytes(slot.Type, arch) + return x86SourceStackScalarType(slot.Type) && slot.Type == sig.Ret && width != 0 && + x86RawReturnFPBound(slot.Offset, width, sig, arch) && slot.Offset <= argSize && width <= argSize-slot.Offset +} + +func x86SourceStackScalarType(typ LLVMType) bool { + switch typ { + case I1, I8, I16, I32, I64: + return true + default: + return false + } +} + +func x86SourceStackInstructionKnown(ins Instr, nativeMove Op, suffix, arch string) bool { + if _, known, err := x86ExplicitIMULProperties(ins.Op, ins, arch); known { + return err == nil + } + if _, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + return err == nil + } + if _, known, err := x86OrderingHintProperties(ins.Op, ins, arch); known { + return err == nil + } + op := Op(normalizeInstructionOpcode(ins.Op)) + if _, recognized, err := x86ConditionalSetProperties(ins.Op, ins, arch); recognized { + return err == nil + } + if op == OpTEXT || op == OpRET || op == OpJMP || isAMD64ConditionalBranch(op) { + return true + } + if spec, stack := amd64StackWidthSpecs[op]; stack { + return string(ins.Op) == string(op) && (spec.bits == 16 || (spec.bits == 32 && arch == "386") || (spec.bits == 64 && arch == "amd64")) + } + if op == nativeMove || x86LEAWidth(ins.Op, arch) != 0 || x86RawReturnScalarMoveBytes(op) != 0 { + return true + } + if _, recognized, err := x86ScalarExtensionProperties(op, ins, arch); recognized { + return err == nil + } + // This bounded numerical contract does not infer effects for opaque raw, + // native/system calls, aggregate/vector memory, or instructions with hidden + // stack inputs. These remain Context, not unsupported decoder forms. + for _, base := range []string{"ADD", "SUB", "ADC", "SBB", "AND", "OR", "XOR", "CMP", "TEST", "INC", "DEC", "NEG", "NOT", "SHL", "SHR", "SAR"} { + for _, width := range []string{"B", "W", "L", "Q"} { + if op == Op(base+width) && (width != "Q" || arch == "amd64") { + return true + } + } + } + return false +} + +func x86SourceStackStep(spec amd64StackWidthSpec, ins *Instr, state *x86SourceStackState, word int64) bool { + width := int64(spec.bits / 8) + push := spec.mode == amd64StackPushValue || spec.mode == amd64StackPushFlags + flags := spec.mode == amd64StackPushFlags || spec.mode == amd64StackPopFlags + if flags && len(ins.Args) != 0 || !flags && len(ins.Args) != 1 { + return false + } + cell := x86SourceStackCell{width: width, flags: flags, numericFlags: state.numericFlags, partialNumericFlags: state.partialNumericFlags} + if !flags { + operand := ins.Args[0] + if operand.Kind != OpReg && (!push || operand.Kind != OpImm) { + return false // Memory/segment stack transfers need a separate contract. + } + if operand.Kind == OpReg { + if x86SegmentRegisterOperand(operand) || operand.Reg == SP { + return false + } + if x86RawReturnStackRegister(operand.Reg) { + if operand.Reg != BP || width != word { + return false + } + value := state.bp + cell.bp = &value + } + if x86SourceGRegister(operand.Reg) && state.gOriginal && push { + if word != 8 || width != word || operand.Reg != "R14" { + return false // A partial token cannot preserve the opaque g identity. + } + cell.gOriginal = true + } + if state.callers != nil && operand.Reg != BP { + root, valid := x86CallerGP(operand.Reg) + if !valid { + return false + } + if pointer, present := state.callers[root]; present { + if width != word || operand.Reg != root { + return false + } + value := pointer + cell.caller = &value + } + cell.known = state.known[root] + } + } + } + if push { + state.sp -= width + cell.offset = state.sp + // A new store replaces the covered old cell, never reuses stale data. + for len(state.cells) != 0 && state.cells[len(state.cells)-1].offset <= state.sp { + state.cells = state.cells[:len(state.cells)-1] + } + state.cells = append(state.cells, cell) + return true + } + if len(state.cells) == 0 { + return false + } + // Restoring a saved SP may discard lower private cells. The next POP must + // still read a complete initialized source PUSH of the same physical width. + for len(state.cells) != 0 && state.cells[len(state.cells)-1].offset < state.sp { + state.cells = state.cells[:len(state.cells)-1] + } + if len(state.cells) == 0 { + return false + } + saved := state.cells[len(state.cells)-1] + if saved.offset != state.sp || saved.width != width { + return false + } + if flags != saved.flags { + return false // Scalar bytes and complete hardware FLAGS are not interchangeable. + } + if flags { + state.numericFlags = saved.numericFlags + state.partialNumericFlags = saved.partialNumericFlags + } + if saved.bp != nil { + if flags || ins.Args[0].Reg != BP { + return false // A saved frame pointer cannot become numeric output. + } + state.bp = *saved.bp + } else if !flags && ins.Args[0].Reg == BP { + return false + } + if saved.gOriginal { + if flags || word != 8 || width != word || ins.Args[0].Reg != "R14" { + return false // The saved opaque g never becomes an ordinary GP value. + } + state.gOriginal = true + delete(state.callers, "R14") + delete(state.known, "R14") + } else if !flags && x86SourceGRegister(ins.Args[0].Reg) { + state.gOriginal = false + } + if state.callers != nil && !flags && ins.Args[0].Reg != BP { + root, valid := x86CallerGP(ins.Args[0].Reg) + if !valid { + return false + } + if _, pointer := state.callers[root]; pointer && width != word { + return false + } + delete(state.callers, root) + if saved.caller != nil { + if width != word || ins.Args[0].Reg != root { + return false + } + state.callers[root] = *saved.caller + } + if saved.gOriginal { + delete(state.known, root) + } else if width == word { + state.known[root] = saved.known + } else { + mask := x86CallerByteMask(ins.Args[0].Reg, width) + state.known[root] = state.known[root]&^mask | saved.known&mask + } + } + state.cells = state.cells[:len(state.cells)-1] + state.sp += width + return true +} + +// Source SP arithmetic operates on a Go stack address, not our LLVM private +// allocation's address. Its flags cannot be assumed equivalent. This bounded +// proof permits numerical redefinitions and matching FLAGS save/restore, but +// never exposes incomplete FLAGS as scalar data or invents incoming flags. +func x86SourceStackFlagsStep(ins Instr, state *x86SourceStackState) bool { + return x86SourceStackFlagsStepForArch(ins, state, "amd64") +} + +func x86SourceStackFlagsStepForArch(ins Instr, state *x86SourceStackState, arch string) bool { + if form, known, err := x86ConditionalMoveProperties(ins.Op, ins, arch); known { + return err == nil && x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(form.condition)) + } + op := Op(normalizeInstructionOpcode(ins.Op)) + if condition, conditionalSet := x86ConditionalSetConditions[op]; conditionalSet { + return x86SourceStackNumericFlagsRead(state, x86ByteConditionFlags(condition)) // SETcc preserves flags. + } + if isAMD64ConditionalBranch(op) { + return isAMD64CounterZeroBranch(op) || x86SourceStackNumericFlagsRead(state, x86ByteBranchFlags(op)) + } + if state.callers != nil { + if effect, known, err := x86TypedGPByteEffect(ins.Op, ins, arch); known { + mask := x86SourceStackNumericFlagMask(state) + if err != nil || mask&effect.flagsRead != effect.flagsRead { + return false + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + x86SourceStackSetNumericFlags(state, 0) + return true // Source address arithmetic never defines numerical flags. + } + } + x86SourceStackSetNumericFlags(state, mask&^effect.flagsKilled|effect.flagsDefined) + return true + } + } + name := string(op) + if len(name) < 2 { + return true + } + base := name[:len(name)-1] + switch base { + case "ADC", "SBB": + return state.numericFlags + case "ADD", "SUB", "AND", "OR", "XOR", "CMP", "TEST", "NEG": + x86SourceStackSetNumericFlags(state, x86ByteFlagsAll) + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + x86SourceStackSetNumericFlags(state, 0) + } + } + case "SHL", "SHR", "SAR": + // Masked zero counts preserve prior flags; OF is undefined for larger + // counts. They need a per-bit contract before serving as a new witness. + x86SourceStackSetNumericFlags(state, 0) + } + return true +} + +func x86SourceStackAddressStep(ins *Instr, state *x86SourceStackState, nativeMove Op, suffix string, word int64) bool { + op := Op(normalizeInstructionOpcode(ins.Op)) + arch := "amd64" + if word == 4 { + arch = "386" + } + lea := x86LEAWidth(ins.Op, arch) != 0 + if lea { + if _, err := x86LEAOperandWidth(*ins, arch); err != nil { + return false + } + } + if form, recognized, err := x86ScalarExtensionProperties(op, *ins, arch); recognized { + if err != nil || x86RawReturnStackRegister(form.result) || + form.source.Kind == OpReg && x86RawReturnStackRegister(form.source.Reg) { + return false + } + } + if len(ins.Args) == 2 && op == nativeMove && ins.Args[0].Kind == OpReg && ins.Args[1].Kind == OpReg && + x86RawReturnStackRegister(ins.Args[0].Reg) && x86RawReturnStackRegister(ins.Args[1].Reg) { + source, destination := ins.Args[0].Reg, ins.Args[1].Reg + if source != SP && source != BP || destination != SP && destination != BP { + return false + } + value := state.bp + if source == SP { + value = x86SourceStackAddress{offset: state.sp} + } + if destination == BP { + state.bp = value + return true + } + if value.original { + return false + } + state.sp = value.offset + return true + } + if len(ins.Args) == 2 && ins.Args[1].Kind == OpReg && ins.Args[1].Reg == SP { + if ins.Args[0].Kind == OpImm && ins.Args[0].ImmRaw == "" { + value := int64(ins.Args[0].Imm) + if word == 4 { + value = int64(int32(value)) // Raw 386 immediates retain unsigned bits. + } + if value < -x86SourceStackLimit || value > x86SourceStackLimit { + return false + } + switch op { + case Op("ADD" + suffix): + state.sp += value + return true + case Op("SUB" + suffix): + state.sp -= value + return true + case Op("AND" + suffix): + alignment := -value + if alignment < 1 || alignment > word || alignment&(alignment-1) != 0 { + return false // The Go entry contract proves native-word alignment only. + } + state.sp &= value + return true + } + } + if op == Op("LEA"+suffix) && ins.Args[0].Kind == OpMem { + memory := ins.Args[0].Mem + if memory.Index != "" || memory.Sym != "" || memory.Segment != "" || + memory.Off < -x86SourceStackLimit || memory.Off > x86SourceStackLimit || ins.x86AddressBits != 0 { + return false + } + base := state.sp + if memory.Base == BP && !state.bp.original { + base = state.bp.offset + } else if memory.Base != SP { + return false + } + state.sp = base + memory.Off + return true + } + } + for _, operand := range ins.Args { + if operand.Kind == OpReg && x86RawReturnStackRegister(operand.Reg) { + return false // No implicit narrowing, arithmetic, or address escape. + } + if lea && operand.Kind == OpMem && x86RawReturnStackMemory(operand.Mem) { + return false // An owned stack address may not escape into an ordinary GP. + } + } + return true +} + +// Source pushes and pops must use the same byte-addressable SP as MOV/LEA/ +// alignment. The legacy amd64 virtual value stack is intentionally unchanged +// for sources without this complete source-stack proof. +func (c *amd64Ctx) pushSourceStackValue(typ LLVMType, width int64, value string) error { + sp, err := c.loadReg(SP) + if err != nil { + return err + } + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = sub i64 %s, %d\n", next, sp, width) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return err + } + ptr := c.ptrFromAddrI64("%" + next) + fmt.Fprintf(c.b, " store %s %s, ptr %s, align 1\n", typ, value, ptr) + return nil +} + +func (c *amd64Ctx) popSourceStackValue(typ LLVMType, width int64) (string, error) { + sp, err := c.loadReg(SP) + if err != nil { + return "", err + } + ptr := c.ptrFromAddrI64(sp) + value := c.newTmp() + fmt.Fprintf(c.b, " %%%s = load %s, ptr %s, align 1\n", value, typ, ptr) + next := c.newTmp() + fmt.Fprintf(c.b, " %%%s = add i64 %s, %d\n", next, sp, width) + if err := c.storeRegUnchecked(SP, "%"+next); err != nil { + return "", err + } + return "%" + value, nil +} diff --git a/amd64_raw_return_source_stack_runtime_test.go b/amd64_raw_return_source_stack_runtime_test.go new file mode 100644 index 000000000..773dbce88 --- /dev/null +++ b/amd64_raw_return_source_stack_runtime_test.go @@ -0,0 +1,253 @@ +package plan9asm + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +type x86SourceStackRuntimeCase struct { + name string + width int + body string + flags bool +} + +func x86SourceStackRuntimeCases(arch string) []x86SourceStackRuntimeCase { + word, suffix, rawMove := 8, "Q", "WORD $0x8948\nBYTE $0xe5" + rawRestore := "WORD $0x8948\nBYTE $0xec" + rawAlign := "LONG $0xf8e48348" + if arch == "386" { + word, suffix, rawMove = 4, "L", "WORD $0xe589" + rawRestore, rawAlign = "WORD $0xec89", "WORD $0xe483\nBYTE $0xfc" + } + var cases []x86SourceStackRuntimeCase + for _, width := range []int{8, 16, 32, 64} { + if arch == "386" && width == 64 { + continue + } + move := map[int]string{8: "MOVB", 16: "MOVW", 32: "MOVL", 64: "MOVQ"}[width] + cases = append(cases, x86SourceStackRuntimeCase{ + name: fmt.Sprintf("Physical%d", width), width: width, + body: fmt.Sprintf("%s %d(SP),AX\nADD%s $7,AX\n%s AX,%d(SP)\nBYTE $0xc3", move, word, suffix, move, 2*word), + }) + } + finish := fmt.Sprintf("ADD%s $7,AX\nMOV%s AX,ret+%d(FP)\nBYTE $0xc3", suffix, suffix, word) + start := "MOV" + suffix + " x+0(FP),AX\n" + cases = append(cases, + x86SourceStackRuntimeCase{name: "BalancedRaw", width: word * 8, + body: start + "BYTE $0x55\n" + rawMove + "\n" + rawAlign + "\n" + rawRestore + "\nBYTE $0x5d\n" + finish}, + x86SourceStackRuntimeCase{name: "BalancedNamed", width: word * 8, + body: start + "PUSH" + suffix + " BP\nMOV" + suffix + " SP,BP\nPUSH" + suffix + " AX\nMOV" + suffix + " $0,AX\nPOP" + suffix + " AX\nMOV" + suffix + " BP,SP\nPOP" + suffix + " BP\n" + finish}, + x86SourceStackRuntimeCase{name: "WordData", width: word * 8, + body: start + "PUSHW AX\nMOVW $0,AX\nPOPW AX\n" + finish}, + x86SourceStackRuntimeCase{name: "WordImmediate", width: word * 8, + body: start + "PUSHW $65530\nPOPW AX\n" + finish}, + x86SourceStackRuntimeCase{name: "Branch", width: word * 8, + body: start + "CMP" + suffix + " AX,$1\nJE done\nADD" + suffix + " $0,BX\ndone:\n" + finish}, + x86SourceStackRuntimeCase{name: "FlagsStack", width: word * 8, flags: true, + body: start + "CMP" + suffix + " AX,$1\nPUSHF" + suffix + "\n" + + fmt.Sprintf("SUB%s $%d,SP\nADD%s $%d,SP\n", suffix, word, suffix, word) + + "POPF" + suffix + "\nADC" + suffix + " $0,AX\n" + finish}, + ) + for _, flagSuffix := range []string{"W", suffix} { + cases = append(cases, x86SourceStackRuntimeCase{ + name: "Flags" + flagSuffix, width: word * 8, flags: true, + body: start + "CMP" + suffix + " AX,$1\nPUSHF" + flagSuffix + "\nADD" + suffix + " $0,BX\nPOPF" + flagSuffix + "\nADC" + suffix + " $0,AX\n" + finish, + }) + } + return cases +} + +func x86SourceStackRuntimeSource(arch string) (string, map[string]FuncSig) { + var source strings.Builder + sigs := map[string]FuncSig{} + for _, test := range x86SourceStackRuntimeCases(arch) { + word, move := int64(8), "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + ty := map[int]LLVMType{8: I8, 16: I16, 32: I32, 64: I64}[test.width] + text, sig, _ := x86SourceStackTypedFixture(arch, fmt.Sprintf("uint%d", test.width), ty, word, move, test.body) + source.WriteString(strings.ReplaceAll(text, "·Value", "·"+test.name)) + sig.Name = test.name + sigs[test.name] = sig + } + word := int64(8) + if arch == "386" { + word = 4 + } + text, sig, _ := x86SourceStackTypedFixture(arch, "bool", I1, word, "MOVB", + "MOVB @WORD(SP),AX\nMOVB AX,@RESULT(SP)\nBYTE $0xc3") + source.WriteString(strings.ReplaceAll(text, "·Value", "·PhysicalBool")) + sig.Name = "PhysicalBool" + sigs[sig.Name] = sig + return source.String(), sigs +} + +func x86SourceStackActualGoNumerical(t *testing.T, arch string, runner []string) { + t.Helper() + dir := t.TempDir() + source, _ := x86SourceStackRuntimeSource(arch) + var goSource strings.Builder + goSource.WriteString("package sourceframe\nimport \"testing\"\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&goSource, "func %s(x uint%d) uint%d\n", test.name, test.width, test.width) + } + goSource.WriteString("func PhysicalBool(x bool) bool\n") + goSource.WriteString("func TestNumerical(t *testing.T) {\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&goSource, ` + for iteration := 0; iteration < 64; iteration++ { + for _, input := range []uint%d{0, 1, 17, 253, ^uint%d(0)-3} { + want := input + 7 +`, test.width, test.width) + if test.flags { + goSource.WriteString("\t\t\tif input == 0 { want++ }\n") + } + if test.name == "WordImmediate" { + fmt.Fprintf(&goSource, "\t\t\twant = ((input & ^uint%d(65535)) | 65530) + 7\n", test.width) + } + fmt.Fprintf(&goSource, ` + canary := [4]uint64{11, 23, 37, 41} + if got := %s(input); got != want || canary != [4]uint64{11, 23, 37, 41} { + t.Fatalf("%s(%%#x)=%%#x, want %%#x; canary=%%v", input, got, want, canary) + } + } + } +`, test.name, test.name) + } + goSource.WriteString("\tif PhysicalBool(false) || !PhysicalBool(true) { t.Fatal(\"bool byte slot\") }\n}\n") + for name, text := range map[string]string{ + "go.mod": "module example.invalid/sourceframeoracle\n\ngo 1.20\n", + "value_" + arch + ".s": source, + "value_test.go": goSource.String(), + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(text), 0600); err != nil { + t.Fatal(err) + } + } + args := []string{"test", "-p=2", "-vet=off", "-count=1"} + if len(runner) != 0 { + args = append(args, "-exec", strings.Join(runner, " ")) + } + args = append(args, ".") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + env := append(os.Environ(), "GOWORK=off", "GOENV=off", "GOTOOLCHAIN=local", "GOARCH="+arch, "CGO_ENABLED=0", "GOMAXPROCS=2") + command := exec.CommandContext(ctx, "go", args...) + command.Dir, command.Env, command.WaitDelay = dir, env, 2*time.Second + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("actual Go %s original source numerical oracle: %v\n%s", arch, err, output) + } +} + +func x86SourceStackCOracle(arch string) string { + var source strings.Builder + source.WriteString("#include \n") + source.WriteString("#include \nextern bool PhysicalBool(bool);\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&source, "extern uint%d_t %s(uint%d_t);\n", test.width, test.name, test.width) + } + source.WriteString("int main(void) {\n") + for _, test := range x86SourceStackRuntimeCases(arch) { + fmt.Fprintf(&source, ` + for (unsigned iteration = 0; iteration < 64; iteration++) { + const uint%d_t inputs[] = {0, 1, 17, 253, (uint%d_t)-4}; + for (unsigned index = 0; index < sizeof(inputs)/sizeof(inputs[0]); index++) { + const uint%d_t input = inputs[index]; + uint%d_t want = input + 7; +`, test.width, test.width, test.width, test.width) + if test.flags { + source.WriteString("\t\t\tif (input == 0) want++;\n") + } + if test.name == "WordImmediate" { + fmt.Fprintf(&source, "\t\t\twant = (input & ~(uint%d_t)65535) | 65530;\n\t\t\twant += 7;\n", test.width) + } + fmt.Fprintf(&source, ` + volatile uint64_t canary = UINT64_C(0x123456789abcdef0); + if (%s(input) != want || canary != UINT64_C(0x123456789abcdef0)) return 1; + } + } +`, test.name) + } + source.WriteString("\tif (PhysicalBool(false) || !PhysicalBool(true)) return 2;\n\treturn 0;\n}\n") + return source.String() +} + +func TestX86RawReturnBoundSourceStackRuntimeForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + source, sigs := x86SourceStackRuntimeSource(target.arch) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "source-stack-forms.ll", "source-stack-forms.o", ir) + } +} + +func TestCrossLinuxRuntimeMatrixX86RawSourceStackNumerical(t *testing.T) { + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("x86 source-stack numerical oracle runs in the required Linux/amd64 cross-runtime gate") + } + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + var runner []string + if crossRosetta { + runner = []string{"/usr/bin/arch", "-x86_64"} + } + x86SourceStackActualGoNumerical(t, "amd64", nil) + source, sigs := x86SourceStackRuntimeSource("amd64") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "source_stack64", triple, ir, x86SourceStackCOracle("amd64"), runner) + if os.Getenv("PLAN9ASM_CROSS_EXEC") == "1" { + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("PLAN9ASM_CROSS_EXEC source-stack matrix requires Linux/amd64") + } + runner = []string{"qemu-i386", "-L", "/usr/i686-linux-gnu"} + x86SourceStackActualGoNumerical(t, "386", runner) + source, sigs = x86SourceStackRuntimeSource("386") + file, err = Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err = Translate(file, Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", Sigs: sigs, + ResolveSym: func(symbol string) string { return strings.TrimPrefix(symbol, "·") }, + }) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestWithCompiler(t, llc, []string{"i686-linux-gnu-gcc", "-no-pie"}, + "source_stack32", "i386-unknown-linux-gnu", ir, x86SourceStackCOracle("386"), runner) + } +} diff --git a/amd64_raw_return_source_stack_test.go b/amd64_raw_return_source_stack_test.go new file mode 100644 index 000000000..ee4a777ec --- /dev/null +++ b/amd64_raw_return_source_stack_test.go @@ -0,0 +1,389 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86SourceStackNumericalFixture(arch, body string) (string, FuncSig, GoPackage) { + word, move, typ, llvmType := int64(8), "MOVQ", "uint64", I64 + if arch == "386" { + word, move, typ, llvmType = 4, "MOVL", "uint32", I32 + } + return x86SourceStackTypedFixture(arch, typ, llvmType, word, move, body) +} + +func x86SourceStackTypedFixture(arch, typ string, llvmType LLVMType, word int64, move, body string) (string, FuncSig, GoPackage) { + body = strings.NewReplacer("@MOV", move, "@WORD", fmt.Sprint(word), "@RESULT", fmt.Sprint(2*word)).Replace(body) + source := fmt.Sprintf("TEXT ·Value(SB),4,$0-%d\n%s\n", 2*word, body) + sig := FuncSig{Name: "sourceframe.Value", Args: []LLVMType{llvmType}, Ret: llvmType, + Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: llvmType, Index: 0, Field: -1}}, + Results: []FrameSlot{{Offset: word, Type: llvmType, Index: 0, Field: -1}}, + }} + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "value.go", "package sourceframe\nfunc Value(x "+typ+") "+typ+"\nfunc callee()\n", 0) + if err != nil { + panic(err) + } + pkg, err := (&types.Config{Sizes: types.SizesFor("gc", arch)}).Check("sourceframe", fset, []*ast.File{file}, nil) + if err != nil { + panic(err) + } + return source, sig, GoPackage{Path: "sourceframe", Types: pkg, Syntax: []*ast.File{file}} +} + +func x86SourceStackOptions(arch, triple string, sig FuncSig) Options { + return Options{Goarch: arch, TargetTriple: triple, + ResolveSym: func(symbol string) string { return strings.Replace(symbol, "·", "sourceframe.", 1) }, + Sigs: map[string]FuncSig{sig.Name: sig}, + } +} + +func TestX86RawReturnBoundSourceStackNumerical(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + push, pop, arithmetic, address, align := "PUSHQ", "POPQ", "ADDQ", "LEAQ", 8 + if target.arch == "386" { + push, pop, arithmetic, address, align = "PUSHL", "POPL", "ADDL", "LEAL", 4 + } + for _, fixture := range []struct{ name, body string }{ + {"physical", "@MOV @WORD(SP),AX\n" + arithmetic + " $7,AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3"}, + {"balanced", "@MOV x+0(FP),AX\n" + push + " BP\n@MOV SP,BP\nAND" + strings.TrimPrefix(arithmetic, "ADD") + fmt.Sprintf(" $-%d,SP\n", align) + arithmetic + " $7,AX\n@MOV BP,SP\n" + pop + " BP\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3"}, + {"nested", "@MOV x+0(FP),AX\n" + push + " BP\n@MOV SP,BP\n" + push + " BX\n" + push + " AX\n" + address + " -2*@WORD(BP),SP\n" + pop + " AX\n" + pop + " BX\n@MOV BP,SP\n" + pop + " BP\n" + arithmetic + " $7,AX\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3"}, + } { + t.Run(target.triple+"/"+fixture.name, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, fixture.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + ir, err := Translate(file, opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "source-frame.ll", "source-frame.o", ir) + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + module.Dispose() + ctx.Dispose() + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + translation.Module.Dispose() + ctx.Dispose() + }) + } + } +} + +func TestX86RawReturnBoundSourceStackRetainsContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + push, pop, sub, align := "PUSHQ", "POPQ", "SUBQ", 16 + if target.arch == "386" { + push, pop, sub, align = "PUSHL", "POPL", "SUBL", 8 + } + for _, body := range []string{ + "@MOV @WORD(SP),AX\n@MOV AX,0(SP)\nBYTE $0xc3", + "@MOV @WORD(SP),AX\n@MOV AX,@WORD-1(SP)\nBYTE $0xc3", + push + " AX\nBYTE $0xc3", + push + " BP\n@MOV SP,BP\n" + pop + " AX\nBYTE $0xc3", + push + " BP\n@MOV SP,BP\nAND" + strings.TrimPrefix(sub, "SUB") + fmt.Sprintf(" $-%d,SP\n", align) + "@MOV BP,SP\n" + pop + " BP\nBYTE $0xc3", + push + " BP\n@MOV SP,BP\n@MOV BP,AX\n@MOV BP,SP\n" + pop + " BP\nBYTE $0xc3", + "LEA" + strings.TrimPrefix(sub, "SUB") + " (SP),AX\nBYTE $0xc3", + "@MOV @WORD(SP),AX\n@MOV AX,(BX)\nBYTE $0xc3", + "CALL ·callee(SB)\nBYTE $0xc3", + "@MOV $0,SP\nBYTE $0xc3", + "CMP" + strings.TrimPrefix(sub, "SUB") + " $0,AX\nJE alternate\n" + push + " AX\nalternate:\nBYTE $0xc3", + } { + t.Run(target.triple+"/"+body, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, body) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + if _, err := Translate(file, opt); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("unproved source continuation became callable: %v", err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module accepted unproved source continuation: %v", err) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err == nil { + translation.Module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("declaration API accepted unproved source continuation: %v", err) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackGoFPProducerOffsets(t *testing.T) { + for _, target := range x86RawReturnTargets { + word, push, pop, move, rawPush, rawPop := int64(8), "PUSHQ", "POPQ", "MOVQ", "BYTE $0x50", "BYTE $0x59" + if target.arch == "386" { + word, push, pop, move = 4, "PUSHL", "POPL", "MOVL" + } + for _, test := range []struct { + name, body string + accepted bool + }{ + {"named FP after named push", push + " CX\n" + move + " x+0(FP),AX\n" + pop + " CX\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", true}, + {"raw push physical shifted argument", rawPush + "\n" + move + " @RESULT(SP),AX\n" + rawPop + "\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", true}, + {"raw push cannot shift Go FP", rawPush + "\n" + move + " x+0(FP),AX\n" + rawPop + "\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", false}, + {"partial scalar access", fmt.Sprintf("MOVB %d(SP),AX\n%s AX,ret+@WORD(FP)\nBYTE $0xc3", word, move), false}, + {"stack index", move + " @WORD(SP)(CX*1),AX\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", false}, + {"original BP is not argument frame", move + " @WORD(BP),AX\n" + move + " AX,ret+@WORD(FP)\nBYTE $0xc3", false}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, test.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + opt := x86SourceStackOptions(target.arch, target.triple, sig) + _, err = Translate(file, opt) + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("source ABI0 transport accepted=%t: %v", test.accepted, err) + } + ctx := llvm.NewContext() + module, moduleErr := TranslateModuleInContext(ctx, file, opt) + if moduleErr == nil { + module.Dispose() + } + ctx.Dispose() + if test.accepted && moduleErr != nil || !test.accepted && !errors.Is(moduleErr, ErrProbeNeedsContext) { + t.Fatalf("module ABI0 transport accepted=%t: %v", test.accepted, moduleErr) + } + ctx = llvm.NewContext() + translation, bindingErr := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if bindingErr == nil { + translation.Module.Dispose() + } + ctx.Dispose() + if test.accepted && bindingErr != nil || !test.accepted && !errors.Is(bindingErr, ErrProbeNeedsContext) { + t.Fatalf("declaration ABI0 transport accepted=%t: %v", test.accepted, bindingErr) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackScalarMoveWidths(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + word, add := int64(8), "ADDQ" + if target.arch == "386" { + word, add = 4, "ADDL" + } + for _, typ := range []struct { + goType string + llvm LLVMType + move string + }{ + {"bool", I1, "MOVB"}, + {"uint8", I8, "MOVB"}, {"uint16", I16, "MOVW"}, {"uint32", I32, "MOVL"}, {"uint64", I64, "MOVQ"}, + } { + if target.arch == "386" && typ.llvm == I64 { + continue // Go's 386 GP ABI transports this value as two slots. + } + t.Run(target.triple+"/"+typ.goType, func(t *testing.T) { + body := "@MOV @WORD(SP),AX\n" + add + " $7,AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3" + if typ.llvm == I1 { + body = "@MOV @WORD(SP),AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3" + } + source, sig, pkg := x86SourceStackTypedFixture(target.arch, typ.goType, typ.llvm, word, typ.move, body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "scalar-sp.ll", "scalar-sp.o", ir) + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + module.Dispose() + ctx.Dispose() + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + translation.Module.Dispose() + ctx.Dispose() + }) + } + } +} + +func TestX86RawReturnBoundSourceStackDoesNotTrustChangedSignature(t *testing.T) { + for _, target := range x86RawReturnTargets { + source, sig, _ := x86SourceStackNumericalFixture(target.arch, "@MOV @WORD(SP),AX\n@MOV AX,@RESULT(SP)\nBYTE $0xc3") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + prepared, err := NormalizeRawFileForTranslation(file, target.arch) + if err != nil { + t.Fatal(err) + } + for _, change := range []string{"no frame", "wrong type", "wrong offset", "wrong index", "custom registers", "pointer", "wrong size"} { + t.Run(target.triple+"/"+change, func(t *testing.T) { + changed := sig + changed.Frame.Params = append([]FrameSlot(nil), sig.Frame.Params...) + changed.Frame.Results = append([]FrameSlot(nil), sig.Frame.Results...) + changed.Args = append([]LLVMType(nil), sig.Args...) + switch change { + case "no frame": + changed.Frame = FrameLayout{} + case "wrong type": + changed.Frame.Params[0].Type = I8 + case "wrong offset": + changed.Frame.Params[0].Offset = 1 + case "wrong index": + changed.Frame.Params[0].Index = 1 + case "custom registers": + changed.ArgRegs = []Reg{AX} + case "pointer": + changed.Args[0], changed.Frame.Params[0].Type = Ptr, Ptr + case "wrong size": + changed.Frame.Results[0].Offset = 1 << 20 + } + if _, err := Translate(prepared, x86SourceStackOptions(target.arch, target.triple, changed)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("prepartition metadata trusted a changed contract: %v", err) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackFlagsNeedNumericalProvenance(t *testing.T) { + for _, target := range x86RawReturnTargets { + suffix, word := "Q", 8 + if target.arch == "386" { + suffix, word = "L", 4 + } + start := "@MOV @WORD(SP),AX\n" + finish := "@MOV AX,@RESULT(SP)\nBYTE $0xc3" + movement := fmt.Sprintf("SUB%s $%d,SP\nADD%s $%d,SP\n", suffix, word, suffix, word) + for _, test := range []struct { + name, body string + accepted bool + }{ + {"entry carry", start + "ADC" + suffix + " $0,AX\n" + finish, false}, + {"SP-derived flags", start + movement + "JS done\ndone:\n" + finish, false}, + {"flags exposed as scalar", start + "CMP" + suffix + " AX,$1\nPUSHF" + suffix + "\nPOP" + suffix + " AX\n" + finish, false}, + {"scalar replaces flags", start + "PUSH" + suffix + " AX\nPOPF" + suffix + "\nADC" + suffix + " $0,AX\n" + finish, false}, + {"different flag origins join", start + "CMP" + suffix + " AX,$1\nJE done\n" + movement + "done:\nADC" + suffix + " $0,AX\n" + finish, false}, + {"saved numeric flags restored", start + "CMP" + suffix + " AX,$1\nPUSHF" + suffix + "\n" + movement + "POPF" + suffix + "\nADC" + suffix + " $0,AX\n" + finish, true}, + } { + t.Run(target.triple+"/"+test.name, func(t *testing.T) { + source, sig, pkg := x86SourceStackNumericalFixture(target.arch, test.body) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, x86SourceStackOptions(target.arch, target.triple, sig)) + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("numerical flags contract accepted=%t: %v", test.accepted, err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, x86SourceStackOptions(target.arch, target.triple, sig)) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("module flags contract accepted=%t: %v", test.accepted, err) + } + ctx = llvm.NewContext() + translation, err := translateGoModuleInContext(ctx, pkg, []byte(source), GoModuleOptions{GOARCH: target.arch, TargetTriple: target.triple}) + if err == nil { + translation.Module.Dispose() + } + ctx.Dispose() + if test.accepted && err != nil || !test.accepted && !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("declaration flags contract accepted=%t: %v", test.accepted, err) + } + }) + } + } +} + +func TestX86RawReturnBoundSourceStackDoesNotTrustEditedRawProducer(t *testing.T) { + for _, target := range x86RawReturnTargets { + source, sig, _ := x86SourceStackNumericalFixture(target.arch, + "BYTE $0x50\n@MOV @RESULT(SP),AX\nBYTE $0x59\n@MOV AX,ret+@WORD(FP)\nBYTE $0xc3") + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + prepared, err := NormalizeRawFileForTranslation(file, target.arch) + if err != nil { + t.Fatal(err) + } + for index := range prepared.Funcs[0].Instrs { + ins := &prepared.Funcs[0].Instrs[index] + if spec, stack := amd64StackWidthSpecs[ins.Op]; stack && spec.mode == amd64StackPushValue { + ins.Args = []Operand{{Kind: OpReg, Reg: BX}} + break + } + } + if _, err := Translate(prepared, x86SourceStackOptions(target.arch, target.triple, sig)); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s edited raw producer kept an original-Go prologue certificate: %v", target.triple, err) + } + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, prepared, x86SourceStackOptions(target.arch, target.triple, sig)) + if err == nil { + module.Dispose() + } + ctx.Dispose() + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("%s module trusted edited raw producer: %v", target.triple, err) + } + } +} diff --git a/amd64_raw_rorx.go b/amd64_raw_rorx.go new file mode 100644 index 000000000..e0c6494b3 --- /dev/null +++ b/amd64_raw_rorx.go @@ -0,0 +1,82 @@ +package plan9asm + +import "fmt" + +// decodedX86RORXInstruction recognizes the complete BMI2 RORXL/RORXQ Yml, +// Yrl, imm8 family, including register and every source-layout-safe ModRM/SIB +// memory form. x/arch v0.14 reports the VEX.0F3A encoding as an unknown AVX +// opcode, so it must be recovered before generic decoding. +func decodedX86RORXInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vex + } + } + +vex: + if len(code) < i+4 || code[i] != 0xc4 || code[i+1]&0x1f != 3 || code[i+3] != 0xf0 { + return Instr{}, 0, false, nil + } + ok = true + if mode != 32 && mode != 64 { + return Instr{}, 0, true, fmt.Errorf("unsupported x86 mode %d", mode) + } + vexExt, vexByte := code[i+1], code[i+2] + if vexByte&3 != 3 || vexByte&4 != 0 || vexByte&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("reserved RORX VEX prefix") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + if len(code) <= i+4 { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + rExt := int(^vexExt>>7) & 1 + xExt := int(^vexExt>>6) & 1 + bExt := int(^vexExt>>5) & 1 + width64 := vexByte&0x80 != 0 + modRM := code[i+4] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && destinationNumber >= 8 { + return Instr{}, 0, true, fmt.Errorf("extended destination register in 32-bit mode") + } + destination, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return Instr{}, 0, true, fmt.Errorf("invalid destination register %d", destinationNumber) + } + source, consumed, err := decodedX86VEXRMSource(code[i+4:], mode, bExt, xExt, segment) + if err != nil { + return Instr{}, 0, true, err + } + if len(code) <= i+4+consumed { + return Instr{}, 0, true, fmt.Errorf("missing imm8") + } + immediate := int64(code[i+4+consumed]) + op := Op("RORXL") + if width64 && mode == 64 { + op = "RORXQ" + } + instruction = Instr{ + Op: op, + Args: []Operand{ + {Kind: OpImm, Imm: immediate}, + source, + {Kind: OpReg, Reg: destination}, + }, + Raw: fmt.Sprintf("%s $%d, %s, %s", op, immediate, source.String(), destination), + } + return instruction, i + 5 + consumed, true, nil +} diff --git a/amd64_raw_rotate_variable.go b/amd64_raw_rotate_variable.go new file mode 100644 index 000000000..0f6564d27 --- /dev/null +++ b/amd64_raw_rotate_variable.go @@ -0,0 +1,85 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawVariableRotateInstruction covers the four EVEX per-lane +// variable rotates in Go's _yvblendmpd table: left/right, D/Q, X/Y/Z, +// optional mask, zeroing, and scalar-memory broadcast. +func decodedX86RawVariableRotateInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 2 || p.pp != 1 || + (p.opcode != 0x14 && p.opcode != 0x15) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX variable rotate: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 { + return fail("invalid EVEX fixed, address, or vector-length bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if mode == 32 && p.mask != 0 { + return fail("386 mask form exceeds the Go assembler frontend's operand limit") + } + modRM := code[p.modRM] + if p.broadcast && modRM>>6 == 3 { + return fail("EVEX.b broadcast requires memory") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && (p.upper >= 8 || destinationNumber >= 8 || + modRM>>6 == 3 && int(modRM&7)+p.b*8+p.x*16 >= 8) { + return fail("extended vector register in 32-bit mode") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + width := 16 << p.vectorLength + laneBytes := 4 + if p.w { + laneBytes = 8 + } + disp8Scale := width + if p.broadcast { + disp8Scale = laneBytes + } + first, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + op := Op("VPRORVD") + if p.opcode == 0x15 { + op = "VPROLVD" + } + if p.w { + op = Op(strings.TrimSuffix(string(op), "D") + "Q") + } + if p.broadcast { + op += ".BCST" + } + if p.zero { + op += ".Z" + } + second := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))} + args := []Operand{first, second} + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_rotate_variable_test.go b/amd64_raw_rotate_variable_test.go new file mode 100644 index 000000000..3b785c0b9 --- /dev/null +++ b/amd64_raw_rotate_variable_test.go @@ -0,0 +1,198 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVariableRotate(op Op, width, mask, destination, first, second int, zeroing, broadcast, memory bool) []byte { + opcode := byte(0x14) + if strings.HasPrefix(string(op), "VPROL") { + opcode = 0x15 + } + p0 := byte((1-(destination>>3)&1)<<7 | (1-(first>>4)&1)<<6 | + (1-(first>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 2) + p1 := byte(((^second)&15)<<3 | 0x05) + if strings.HasSuffix(string(op), "Q") { + p1 |= 0x80 + } + p2 := byte(width<<5 | (1-((second>>4)&1))<<3 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destination&7)<<3 | first&7) + if memory { + modRM = byte(0x40 | (destination&7)<<3) + p0 = byte((1-(destination>>3)&1)<<7 | 0x60 | (1-(destination>>4)&1)<<4 | 2) + } + code := []byte{0x62, p0, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodeX86RawVariableRotateSeBiShogunRegression(t *testing.T) { + // Go's assembler disassembles these bytes as VPROLVD Z0, Z1, Z1. + code := []byte{0x62, 0xf2, 0x75, 0x48, 0x15, 0xc8, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "rotlInt32 AVX512 VPROLVD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPROLVD" || + decoded[0].Args[0].Reg != "Z0" || decoded[0].Args[1].Reg != "Z1" || + decoded[0].Args[2].Reg != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawVariableRotateCompleteGoForms(t *testing.T) { + for _, op := range []Op{"VPROLVD", "VPROLVQ", "VPRORVD", "VPRORVQ"} { + for width := 0; width < 3; width++ { + vector := [...]string{"X", "Y", "Z"}[width] + for _, form := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + destination int + first int + second int + }{ + {name: "register", destination: 2, first: 3, second: 4}, + {name: "high registers", destination: 18, first: 19, second: 20}, + {name: "masked zeroing", mask: 7, zeroing: true, destination: 2, first: 3, second: 4}, + {name: "memory", memory: true, destination: 2, second: 4}, + {name: "broadcast memory", broadcast: true, memory: true, destination: 2, second: 4}, + {name: "broadcast zeroing memory", broadcast: true, memory: true, mask: 5, zeroing: true, destination: 2, second: 4}, + } { + name := fmt.Sprintf("%s/%s/%s", op, vector, form.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawVariableRotate( + op, width, form.mask, form.destination, form.first, form.second, + form.zeroing, form.broadcast, form.memory, + ) + got, length, ok, err := decodedX86RawVariableRotateInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := op + if form.broadcast { + wantOp += ".BCST" + } + if form.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != fmt.Sprintf("%s%d", vector, form.second) || + got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s%d", vector, form.destination) { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + if !form.memory && got.Args[0].String() != fmt.Sprintf("%s%d", vector, form.first) { + t.Fatalf("decode %x first source = %s", code, got.Args[0]) + } + }) + } + } + } +} + +func TestDecodeX86RawVariableRotateRIPLiteral(t *testing.T) { + for _, op := range []Op{"VPROLVD", "VPROLVQ", "VPRORVD", "VPRORVQ"} { + for width := 0; width < 3; width++ { + for _, broadcast := range []bool{false, true} { + code := encodeX86RawVariableRotate(op, width, 1, 2, 0, 4, false, broadcast, false) + code[5] = 0x15 // RIP-relative first source, destination 2. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << width + if broadcast { + literalWidth = 4 + if strings.HasSuffix(string(op), "Q") { + literalWidth = 8 + } + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d/broadcast%t", op, width, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || len(got[0].x86RIPLiteralData) != literalWidth { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } + } +} + +func TestDecodeX86RawVariableRotateRejectsInvalidForms(t *testing.T) { + base := encodeX86RawVariableRotate("VPROLVD", 2, 0, 2, 3, 4, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x10 }, // broadcast requires memory + func(code []byte) { code[3] |= 0x80 }, // zeroing requires a mask + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawVariableRotateInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func TestTranslateX86RawVariableRotateLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawVariableRotate(SB),$0-0\n") + for _, op := range []Op{"VPROLVD", "VPROLVQ", "VPRORVD", "VPRORVQ"} { + for width := 0; width < 3; width++ { + for _, form := range []struct { + memory, broadcast bool + }{ + {}, {memory: true}, {memory: true, broadcast: true}, + } { + code := encodeX86RawVariableRotate(op, width, 0, 2, 3, 4, false, form.broadcast, form.memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawVariableRotate": {Name: "rawVariableRotate", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-variable-rotate.ll", "raw-variable-rotate.o", ir) + }) + } +} diff --git a/amd64_raw_round.go b/amd64_raw_round.go new file mode 100644 index 000000000..7810be8e6 --- /dev/null +++ b/amd64_raw_round.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// decodedX86RawVEXRoundInstruction implements Go's four VROUND PS/PD/SS/SD +// opcode rows. The packed forms reserve vvvv; scalar forms use it as their +// upper-lane source. Every form has a trailing imm8. +func decodedX86RawVEXRoundInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 3 || p.pp != 1 || p.opcode < 0x08 || p.opcode > 0x0b { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("VROUND: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.evex || p.w || p.addressOverride { + return fail("requires VEX.W0 without address override") + } + scalar := p.opcode >= 0x0a + if scalar && p.vectorLength != 0 || !scalar && p.upper != 0 { + return fail("invalid scalar vector length or packed reserved vvvv") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + destinationNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (destinationNumber >= 8 || p.upper >= 8 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + width := "X" + if p.vectorLength != 0 { + width = "Y" + } + source, consumed, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := p.modRM + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + args := []Operand{{Kind: OpImm, Imm: int64(code[immIndex])}, source} + if scalar { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))}) + op := [...]Op{"VROUNDPS", "VROUNDPD", "VROUNDSS", "VROUNDSD"}[p.opcode-0x08] + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, immIndex + 1, true, nil +} diff --git a/amd64_raw_round_test.go b/amd64_raw_round_test.go new file mode 100644 index 000000000..4a4f4e021 --- /dev/null +++ b/amd64_raw_round_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVEXRound(opcode byte, width256 bool, destination, upper, source int, imm byte) []byte { + code := encodeX86VEXPackedWordMultiply(3, opcode, width256, false, destination, upper, source) + return append(code, imm) +} + +func TestDecodedX86RawVEXRoundCompleteFamily(t *testing.T) { + for _, form := range []struct { + opcode byte + op Op + scalar bool + }{ + {0x08, "VROUNDPS", false}, + {0x09, "VROUNDPD", false}, + {0x0a, "VROUNDSS", true}, + {0x0b, "VROUNDSD", true}, + } { + for _, width256 := range []bool{false, true} { + if form.scalar && width256 { + continue + } + for _, source := range []int{0, 7, 8, 15} { + for _, imm := range []byte{0, 9, 255} { + name := fmt.Sprintf("%s/y%t/src%d/imm%d", form.op, width256, source, imm) + t.Run(name, func(t *testing.T) { + code := encodeX86RawVEXRound(form.opcode, width256, 11, 3, source, imm) + if !form.scalar { + code = encodeX86RawVEXRound(form.opcode, width256, 11, 0, source, imm) + } + got, length, matched, err := decodedX86RawVEXRoundInstruction(code, 64) + if err != nil || !matched || length != len(code) || got.Op != form.op || + got.Args[0].Kind != OpImm || got.Args[0].Imm != int64(imm) || + got.Args[1].String() != fmt.Sprintf("%s%d", map[bool]string{false: "X", true: "Y"}[width256], source) { + t.Fatalf("decode %x = %+v length=%d matched=%t err=%v", code, got, length, matched, err) + } + if form.scalar && (len(got.Args) != 4 || got.Args[2].String() != "X3") { + t.Fatalf("scalar upper-lane operand missing: %+v", got) + } + }) + } + } + } + } + // Actual bytes in simd@v1.21.1 arith_avx2_amd64.s. + real := []byte{0xc4, 0xa3, 0x7d, 0x08, 0x04, 0x86, 0x09} + got, length, matched, err := decodedX86RawVEXRoundInstruction(real, 64) + if err != nil || !matched || length != len(real) || got.Op != "VROUNDPS" || + got.Args[1].String() != "0(SI)(R8*4)" || got.Args[2].String() != "Y0" { + t.Fatalf("decode real VROUNDPS %x = %+v length=%d matched=%t err=%v", real, got, length, matched, err) + } +} + +func TestDecodedX86RawVEXRoundRejectsInvalidForms(t *testing.T) { + packed := encodeX86RawVEXRound(0x08, true, 0, 0, 0, 9) + scalar := encodeX86RawVEXRound(0x0a, false, 0, 1, 0, 9) + for name, bad := range map[string][]byte{ + "packed reserved vvvv": func() []byte { + b := append([]byte(nil), packed...) + b[2] &^= 0x08 + return b + }(), + "scalar L1": func() []byte { + b := append([]byte(nil), scalar...) + b[2] |= 0x04 + return b + }(), + "W1": func() []byte { + b := append([]byte(nil), packed...) + b[2] |= 0x80 + return b + }(), + "truncated imm8": packed[:len(packed)-1], + "address override": append([]byte{0x67}, packed...), + } { + t.Run(name, func(t *testing.T) { + if _, _, matched, err := decodedX86RawVEXRoundInstruction(bad, 64); !matched || err == nil { + t.Fatalf("accepted invalid VROUND %x: matched=%t err=%v", bad, matched, err) + } + }) + } +} + +func x86RawVEXRoundConstant(opcode byte, width256 bool) []byte { + code := encodeX86RawVEXRound(opcode, width256, 0, 0, 0, 9) + code[len(code)-2] = 0x05 + imm := code[len(code)-1] + code = append(code[:len(code)-1], 1, 0, 0, 0, imm, 0xc3) + width := 16 + if width256 { + width = 32 + } + if opcode == 0x0a { + width = 4 + } else if opcode == 0x0b { + width = 8 + } + for i := 0; i < width; i++ { + code = append(code, byte(i*17+11)) + } + return code +} + +func TestDecodeX86RawVEXRoundConstantCompleteFamily(t *testing.T) { + for _, opcode := range []byte{0x08, 0x09, 0x0a, 0x0b} { + for _, wide := range []bool{false, true} { + if opcode >= 0x0a && wide { + continue + } + name := fmt.Sprintf("%02x/y%t", opcode, wide) + t.Run(name, func(t *testing.T) { + code := x86RawVEXRoundConstant(opcode, wide) + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Args[1].Kind != OpSym || !got[0].x86RIPLiteral || got[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want source-local round constant", code, got) + } + }) + } + } +} + +func TestDecodeX86RawVEXRoundConstantRejectsUnsafeSources(t *testing.T) { + valid := x86RawVEXRoundConstant(0x08, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[5] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[5] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe VROUND RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVEXRoundObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, opcode := range []byte{0x08, 0x09, 0x0a, 0x0b} { + code := x86RawVEXRoundConstant(opcode, opcode < 0x0a) + name := fmt.Sprintf("rawVEXRound%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-vex-round.ll", "raw-vex-round.o", ir) + }) + } +} diff --git a/amd64_raw_sad.go b/amd64_raw_sad.go new file mode 100644 index 000000000..30553fc65 --- /dev/null +++ b/amd64_raw_sad.go @@ -0,0 +1,103 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27 has three vector SAD encodings: VPSADBW is 66/0F/F6 in +// VEX X/Y and EVEX X/Y/Z, VMPSADBW is VEX 66/0F3A/42 with imm8 in X/Y, +// and VDBPSADBW is the corresponding EVEX X/Y/Z immediate form with an +// optional writemask. Legacy PSADBW and MPSADBW use the x86asm path. +func decodedX86RawPackedSADInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, recognized := decodeX86RawVectorEncoding(code) + if !recognized || p.pp != 1 { + return Instr{}, 0, false, nil + } + var op Op + withImmediate := false + switch { + case p.mapNumber == 1 && p.opcode == 0xf6: + op = "VPSADBW" + case p.mapNumber == 3 && p.opcode == 0x42 && !p.evex: + op, withImmediate = "VMPSADBW", true + case p.mapNumber == 3 && p.opcode == 0x42 && p.evex: + op, withImmediate = "VDBPSADBW", true + default: + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("%s: %s", op, message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.w || p.vectorLength > 2 || p.evex && !p.fixed { + return fail("invalid width or EVEX fixed bit") + } + if !p.evex && p.vectorLength == 2 { + return fail("VEX does not encode Z registers") + } + if p.broadcast || p.mask != 0 && op != "VDBPSADBW" || + p.zero && (op != "VDBPSADBW" || p.mask == 0) { + return fail("invalid broadcast, writemask or zeroing form") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0 || p.upper >= 8) { + return fail("extended vector register in 32-bit mode") + } + + width := [...]string{"X", "Y", "Z"}[p.vectorLength] + var first Operand + var consumed int + var err error + if p.evex { + first, consumed, err = decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, width, 16<>3&7) + p.r*8 + if mode == 32 && destinationNumber >= 8 { + return fail("extended destination register in 32-bit mode") + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, destinationNumber))} + args := []Operand{first, second} + if withImmediate { + args = append([]Operand{{Kind: OpImm, Imm: int64(code[length])}}, args...) + length++ + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + if p.zero { + op += ".Z" + } + parts := make([]string, len(args)) + for index, arg := range args { + parts[index] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(parts, ", ")), + x86Encoded: true, + }, length, true, nil +} diff --git a/amd64_raw_sad_test.go b/amd64_raw_sad_test.go new file mode 100644 index 000000000..c560519b7 --- /dev/null +++ b/amd64_raw_sad_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawPackedSAD(op Op, vectorBits, first, second, destination, mask int, zero, memory bool) []byte { + if memory { + first = 0 + } + mapNumber, opcode := byte(1), byte(0xf6) + if op != "VPSADBW" && op != "VPSADBW.EVEX" { + mapNumber, opcode = 3, 0x42 + } + modRM := byte(0xc0 | destination&7<<3 | first&7) + if memory { + modRM = byte(0x40 | destination&7<<3) + } + var code []byte + if op == "VDBPSADBW" || op == "VPSADBW.EVEX" { + p0 := byte((1-destination/8&1)<<7 | (1-first/16)<<6 | + (1-first/8&1)<<5 | (1-destination/16)<<4 | int(mapNumber)) + p1 := byte((^second&15)<<3 | 4 | 1) + p2 := byte(vectorBits<<5 | (1-second/16)<<3 | mask) + if zero { + p2 |= 0x80 + } + code = []byte{0x62, p0, p1, p2, opcode, modRM} + } else { + p0 := byte((1-destination/8&1)<<7 | 1<<6 | + (1-first/8&1)<<5 | int(mapNumber)) + p1 := byte((^second&15)<<3 | vectorBits<<2 | 1) + code = []byte{0xc4, p0, p1, opcode, modRM} + } + if memory { + code = append(code, 1) + } + if op != "VPSADBW" && op != "VPSADBW.EVEX" { + code = append(code, 0x39) + } + return code +} + +func TestTranslateRawPackedSADGorseRegression(t *testing.T) { + const source = `TEXT rawPackedSAD(SB), $0-0 + LONG $0xd3f6edc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawPackedSAD": {Name: "rawPackedSAD", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } +} + +func TestTranslateRawPackedSADLegacyForms(t *testing.T) { + const source = `TEXT rawLegacySAD(SB), $0-0 + BYTE $0x66; BYTE $0x0f; BYTE $0xf6; BYTE $0xc1 + BYTE $0x66; BYTE $0x0f; BYTE $0x3a; BYTE $0x42; BYTE $0xc1; BYTE $0x01 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawLegacySAD": {Name: "rawLegacySAD", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } +} + +func TestDecodedX86RawPackedSADGoRows(t *testing.T) { + for _, test := range []struct { + encoded Op + want Op + widths int + }{ + {"VPSADBW", "VPSADBW", 2}, + {"VPSADBW.EVEX", "VPSADBW", 3}, + {"VMPSADBW", "VMPSADBW", 2}, + {"VDBPSADBW", "VDBPSADBW", 3}, + } { + if _, ok := amd64PackedSADSpecs[test.want]; !ok { + t.Fatalf("missing lowerer for Go row %s", test.want) + } + for vectorBits := 0; vectorBits < test.widths; vectorBits++ { + for _, memory := range []bool{false, true} { + for _, mask := range []int{0, 3} { + if mask != 0 && test.encoded != "VDBPSADBW" { + continue + } + for _, zero := range []bool{false, true} { + if zero && mask == 0 { + continue + } + code := encodeX86RawPackedSAD(test.encoded, vectorBits, 4, 5, 2, mask, zero, memory) + got, length, recognized, err := decodedX86RawPackedSADInstruction(code, 64) + if err != nil || !recognized || length != len(code) { + t.Fatalf("decode %x: %+v length=%d recognized=%v err=%v", code, got, length, recognized, err) + } + wantOp := test.want + if zero { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[len(got.Args)-1].String() != fmt.Sprintf("%s2", "XYZ"[vectorBits:vectorBits+1]) { + t.Fatalf("decode %x: %+v, want %s", code, got, wantOp) + } + if memory { + firstIndex := 0 + if test.want != "VPSADBW" { + firstIndex = 1 + } + if got.Args[firstIndex].Kind != OpMem { + t.Fatalf("decode %x did not retain memory: %+v", code, got) + } + } + } + } + } + } + } +} + +func TestDecodedX86RawPackedSADRejectsInvalidForms(t *testing.T) { + base := encodeX86RawPackedSAD("VPSADBW", 1, 1, 2, 3, 0, false, false) + for _, test := range []struct { + name string + code []byte + }{ + {"VEX W1", []byte{base[0], base[1], base[2] | 0x80, base[3], base[4]}}, + {"missing ModRM", base[:4]}, + {"address override", append([]byte{0x67}, base...)}, + {"missing immediate", encodeX86RawPackedSAD("VMPSADBW", 0, 1, 2, 3, 0, false, false)[:5]}, + {"VDB broadcast", func() []byte { + code := encodeX86RawPackedSAD("VDBPSADBW", 0, 0, 2, 3, 0, false, true) + code[3] |= 0x10 + return code + }()}, + {"VPSADBW writemask", func() []byte { + code := encodeX86RawPackedSAD("VPSADBW.EVEX", 0, 1, 2, 3, 0, false, false) + code[3] |= 3 + return code + }()}, + {"VDB zero without mask", encodeX86RawPackedSAD("VDBPSADBW", 0, 1, 2, 3, 0, true, false)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawPackedSADInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } +} + +func TestTranslateRawPackedSADLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawPackedSADFamily(SB), $0-0\n") + for _, test := range []struct { + op Op + width int + mask int + zero bool + memory bool + }{ + {op: "VPSADBW", width: 1}, + {op: "VPSADBW.EVEX", width: 2, memory: true}, + {op: "VMPSADBW", width: 1, memory: true}, + {op: "VDBPSADBW", width: 2, mask: 3, zero: true, memory: true}, + } { + code := encodeX86RawPackedSAD( + test.op, test.width, 1, 2, 3, test.mask, test.zero, test.memory, + ) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawPackedSADFamily": {Name: "rawPackedSADFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-sad.ll", "raw-packed-sad.o", ir) + }) + } +} diff --git a/amd64_raw_same_width_conversion.go b/amd64_raw_same_width_conversion.go new file mode 100644 index 000000000..aea770b65 --- /dev/null +++ b/amd64_raw_same_width_conversion.go @@ -0,0 +1,230 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type decodedX86SameWidthConversionProperties struct { + op Op + laneBytes int + sae bool +} + +func decodedX86SameWidthConversionOpcode(opcode, pp byte) (decodedX86SameWidthConversionProperties, bool) { + properties := map[[2]byte]decodedX86SameWidthConversionProperties{ + {0x5b, 0}: {op: "VCVTDQ2PS", laneBytes: 4}, + {0x5b, 1}: {op: "VCVTPS2DQ", laneBytes: 4}, + {0x5b, 2}: {op: "VCVTTPS2DQ", laneBytes: 4, sae: true}, + {0x51, 0}: {op: "VSQRTPS", laneBytes: 4}, + {0x51, 1}: {op: "VSQRTPD", laneBytes: 8}, + }[[2]byte{opcode, pp}] + return properties, properties.op != "" +} + +// decodedX86SameWidthConversionInstruction recognizes every VEX and EVEX +// encoding behind Go 1.27's _yvcvtdq2ps table: VCVTDQ2PS, VCVTPS2DQ, +// VCVTTPS2DQ, VSQRTPD, and VSQRTPS. It covers X/Y/Z registers, ModRM/SIB +// memory, masking, zeroing, broadcasts, embedded rounding, SAE, and compressed +// displacements before the generic decoder can consume an incorrect length. +func decodedX86SameWidthConversionInstruction(code []byte, mode int) (instruction Instr, length int, ok bool, err error) { + return decodedX86SameWidthConversionInstructionWithMemory(code, mode, x86RawMemoryDecoder{}) +} + +func decodedX86SameWidthConversionInstructionWithMemory(code []byte, mode int, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if p, valid := decodeX86RawVectorEncoding(code); valid && p.mapNumber == 1 && p.opcode == 0x5b && + (p.pp == 1 || p.pp == 2) && (!p.evex || !p.w) { + return decodedX86PackedNumericConversionInstructionWithMemory(code, mode, memory) + } + i := 0 + segment := Reg("") + addressOverride := false + for i < len(code) { + switch code[i] { + case 0x64: + segment = FS + i++ + case 0x65: + segment = GS + i++ + case 0x67: + addressOverride = true + i++ + default: + goto vectorPrefix + } + } + +vectorPrefix: + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + if len(code) > i && code[i] == 0x62 { + return decodedX86EVEXSameWidthConversionInstructionWithMemory(code, i, mode, segment, addressOverride, memory) + } + + var vexLength, rExt, xExt, bExt int + var vexBits, opcode byte + switch { + case len(code) >= i+4 && code[i] == 0xc5: + vexLength = 2 + vexBits = code[i+1] + rExt = int(^vexBits>>7) & 1 + opcode = code[i+2] + case len(code) >= i+5 && code[i] == 0xc4 && code[i+1]&0x1f == 1: + vexLength = 3 + vexBits = code[i+2] + rExt = int(^code[i+1]>>7) & 1 + xExt = int(^code[i+1]>>6) & 1 + bExt = int(^code[i+1]>>5) & 1 + opcode = code[i+3] + default: + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86SameWidthConversionOpcode(opcode, vexBits&3) + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + // CVTDQ2PS is VEX.WIG in both machine modes (Intel Vol.2 3-269). + // Keep the other families' existing W policy independent. + if vexLength == 3 && vexBits&0x80 != 0 && properties.op != "VCVTDQ2PS" { + return Instr{}, 0, true, fmt.Errorf("VEX.W must be zero") + } + if vexBits&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("VEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + opcodeIndex := i + vexLength + modRMIndex := opcodeIndex + 1 + if len(code) <= modRMIndex { + return Instr{}, 0, true, fmt.Errorf("missing ModRM byte") + } + vectorPrefix := "X" + if vexBits&0x04 != 0 { + vectorPrefix = "Y" + } + modRM := code[modRMIndex] + destinationNumber := int(modRM>>3&7) + rExt*8 + if mode == 32 && (destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register in 32-bit mode") + } + source, consumed, decodeErr := memory.vex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + instruction = Instr{ + Op: properties.op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", properties.op, source.String(), destination.String()), + } + return instruction, modRMIndex + consumed, true, nil +} + +func decodedX86EVEXSameWidthConversionInstruction(code []byte, i, mode int, segment Reg, addressOverride bool) (instruction Instr, length int, ok bool, err error) { + return decodedX86EVEXSameWidthConversionInstructionWithMemory(code, i, mode, segment, addressOverride, x86RawMemoryDecoder{}) +} + +func decodedX86EVEXSameWidthConversionInstructionWithMemory(code []byte, i, mode int, segment Reg, addressOverride bool, memory x86RawMemoryDecoder) (instruction Instr, length int, ok bool, err error) { + if len(code) < i+6 { + return Instr{}, 0, false, nil + } + p0, p1, p2 := code[i+1], code[i+2], code[i+3] + opcode := code[i+4] + half := p0&0x0f == 5 && opcode == 0x51 && p1&3 == 0 + if p0&0x0f != 1 && !half { + return Instr{}, 0, false, nil + } + properties, recognized := decodedX86SameWidthConversionOpcode(opcode, p1&3) + if half { + properties = decodedX86SameWidthConversionProperties{op: "VSQRTPH", laneBytes: 2} + } + if !recognized { + return Instr{}, 0, false, nil + } + ok = true + if p1&0x04 == 0 { + return Instr{}, 0, true, fmt.Errorf("invalid EVEX prefix") + } + width64 := p1&0x80 != 0 + wantWidth64 := properties.laneBytes == 8 + if width64 != wantWidth64 { + return Instr{}, 0, true, fmt.Errorf("EVEX.W does not match conversion element type") + } + if p1&0x78 != 0x78 || p2&0x08 == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX.vvvv must be reserved") + } + if addressOverride { + return Instr{}, 0, true, fmt.Errorf("address-size override is not source-layout safe") + } + + modRMIndex := i + 5 + modRM := code[modRMIndex] + vectorBits := (p2 >> 5) & 3 + evexB := p2&0x10 != 0 + registerSource := modRM>>6 == 3 + embeddedControl := evexB && registerSource + broadcast := evexB && !registerSource + if !embeddedControl && vectorBits == 3 { + return Instr{}, 0, true, fmt.Errorf("reserved EVEX vector length") + } + + vectorPrefix := "Z" + vectorWidth := 64 + if !embeddedControl { + vectorPrefix = [...]string{"X", "Y", "Z"}[vectorBits] + vectorWidth = [...]int{16, 32, 64}[vectorBits] + } + disp8Scale := vectorWidth + if broadcast { + disp8Scale = properties.laneBytes + } + maskNumber := int(p2 & 7) + zeroing := p2&0x80 != 0 + if zeroing && maskNumber == 0 { + return Instr{}, 0, true, fmt.Errorf("EVEX zeroing requires a nonzero mask") + } + + rExt := int(^p0>>7) & 1 + rHighExt := int(^p0>>4) & 1 + xExt := int(^p0>>6) & 1 + bExt := int(^p0>>5) & 1 + destinationNumber := int(modRM>>3&7) + rExt*8 + rHighExt*16 + if mode == 32 && (maskNumber != 0 && !half && properties.op != "VCVTDQ2PS" || destinationNumber >= 8 || modRM>>6 == 3 && int(modRM&7)+bExt*8+xExt*16 >= 8) { + return Instr{}, 0, true, fmt.Errorf("extended vector register or mask in 32-bit mode") + } + source, consumed, decodeErr := memory.evex(code[modRMIndex:], mode, bExt, xExt, segment, vectorPrefix, disp8Scale) + if decodeErr != nil { + return Instr{}, 0, true, decodeErr + } + + op := properties.op + if broadcast { + op += ".BCST" + } + if embeddedControl { + if properties.sae { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[vectorBits] + } + } + if zeroing { + op += ".Z" + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vectorPrefix, destinationNumber))} + args := []Operand{source} + if maskNumber != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", maskNumber))}) + } + args = append(args, destination) + rawArgs := make([]string, len(args)) + for index := range args { + rawArgs[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(rawArgs, ", ")), x86Encoded: half}, modRMIndex + consumed, true, nil +} diff --git a/amd64_raw_same_width_conversion_test.go b/amd64_raw_same_width_conversion_test.go new file mode 100644 index 000000000..48d98a08b --- /dev/null +++ b/amd64_raw_same_width_conversion_test.go @@ -0,0 +1,171 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +type rawSameWidthConversionSpec struct { + opcode byte + pp byte + evexW bool + op Op + laneBytes int + sae bool +} + +var rawSameWidthConversionSpecs = []rawSameWidthConversionSpec{ + {opcode: 0x5b, pp: 0, op: "VCVTDQ2PS", laneBytes: 4}, + {opcode: 0x5b, pp: 1, op: "VCVTPS2DQ", laneBytes: 4}, + {opcode: 0x5b, pp: 2, op: "VCVTTPS2DQ", laneBytes: 4, sae: true}, + {opcode: 0x51, pp: 1, evexW: true, op: "VSQRTPD", laneBytes: 8}, + {opcode: 0x51, pp: 0, op: "VSQRTPS", laneBytes: 4}, +} + +func encodeX86VEXSameWidthConversion(spec rawSameWidthConversionSpec, vex3, width256 bool, destination, source int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if !vex3 { + vex1 := byte((1-(destination>>3)&1)<<7 | 15<<3 | int(l)<<2 | int(spec.pp)) + return []byte{0xc5, vex1, spec.opcode, modRM} + } + vex0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(source>>3)&1)<<5 | 1) + vex1 := byte(15<<3 | int(l)<<2 | int(spec.pp)) + return []byte{0xc4, vex0, vex1, spec.opcode, modRM} +} + +func encodeX86EVEXSameWidthConversion(spec rawSameWidthConversionSpec, vectorBits byte, broadcastOrControl, zeroing bool, mask, destination, source int) []byte { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source>>4)&1)<<6 | (1-(source>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte(15<<3 | 1<<2 | int(spec.pp)) + if spec.evexW { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | 1<<3 | mask) + if broadcastOrControl { + p2 |= 0x10 + } + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0x62, p0, p1, p2, spec.opcode, modRM} +} + +func TestDecodedX86VEXSameWidthConversionCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawSameWidthConversionSpecs { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + sourceLimit := 16 + if !vex3 { + sourceLimit = 8 + } + for destination := 0; destination < 16; destination++ { + for source := 0; source < sourceLimit; source++ { + code := encodeX86VEXSameWidthConversion(spec, vex3, width256, destination, source) + got, length, ok, err := decodedX86SameWidthConversionInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != spec.op || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 3840 { + t.Fatalf("covered %d VEX same-width conversion encodings, want 3840", count) + } +} + +func TestDecodedX86EVEXSameWidthConversionCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawSameWidthConversionSpecs { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + for destination := 0; destination < 32; destination++ { + for source := 0; source < 32; source++ { + code := encodeX86EVEXSameWidthConversion(spec, vectorBits, false, false, 1, destination, source) + got, length, ok, err := decodedX86SameWidthConversionInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != spec.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, spec.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 15360 { + t.Fatalf("covered %d EVEX same-width conversion encodings, want 15360", count) + } +} + +func TestDecodedX86EVEXSameWidthConversionModifiersMemoryAndInvalid(t *testing.T) { + find := func(op Op) rawSameWidthConversionSpec { + t.Helper() + for _, spec := range rawSameWidthConversionSpecs { + if spec.op == op { + return spec + } + } + t.Fatalf("missing raw same-width spec %s", op) + return rawSameWidthConversionSpec{} + } + convert := find("VCVTDQ2PS") + truncate := find("VCVTTPS2DQ") + sqrt := find("VSQRTPD") + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "vex weaviate", code: []byte{0xc5, 0xfc, 0x5b, 0xe4}, op: "VCVTDQ2PS", args: []string{"Y4", "Y4"}}, + {name: "broadcast zeroing", code: []byte{0x62, 0xf1, 0x7c, 0xdb, 0x5b, 0x50, 0x7f}, op: "VCVTDQ2PS.BCST.Z", args: []string{"508(AX)", "K3", "Z2"}}, + {name: "round up", code: encodeX86EVEXSameWidthConversion(convert, 2, true, false, 2, 4, 5), op: "VCVTDQ2PS.RU_SAE", args: []string{"Z5", "K2", "Z4"}}, + {name: "truncate sae zeroing", code: encodeX86EVEXSameWidthConversion(truncate, 0, true, true, 3, 6, 7), op: "VCVTTPS2DQ.SAE.Z", args: []string{"Z7", "K3", "Z6"}}, + {name: "sqrt round zero", code: encodeX86EVEXSameWidthConversion(sqrt, 3, true, false, 0, 8, 9), op: "VSQRTPD.RZ_SAE", args: []string{"Z9", "Z8"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86SameWidthConversionInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + zeroWithoutMask := encodeX86EVEXSameWidthConversion(convert, 0, false, true, 0, 0, 0) + reservedLength := encodeX86EVEXSameWidthConversion(convert, 3, false, false, 0, 0, 0) + wrongW := encodeX86EVEXSameWidthConversion(convert, 0, false, false, 0, 0, 0) + wrongW[2] |= 0x80 + for _, code := range [][]byte{zeroWithoutMask, reservedLength, wrongW} { + if _, _, ok, err := decodedX86SameWidthConversionInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} diff --git a/amd64_raw_scalar_broadcast.go b/amd64_raw_scalar_broadcast.go new file mode 100644 index 000000000..043ebca6e --- /dev/null +++ b/amd64_raw_scalar_broadcast.go @@ -0,0 +1,73 @@ +package plan9asm + +import "fmt" + +// Go's _yvbroadcastss/_yvbroadcastsd tables share the typed VEX/EVEX +// prefix grammar. The scalar input width, output width and mask are separate +// axes: SD has no 128-bit destination, and disp8 scales by scalar input size. +func decodedX86ScalarBroadcastInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 2 || (p.opcode != 0x18 && p.opcode != 0x19) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar broadcast: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.pp != 1 || p.upper != 0 || p.broadcast { + return fail("reserved prefix, vvvv/V' or broadcast field") + } + double := p.opcode == 0x19 + if p.vectorLength > 2 || double && p.vectorLength == 0 { + return fail("invalid vector width") + } + if p.evex { + if !p.fixed || p.w != double { + return fail("invalid EVEX fixed or width bit") + } + } else if p.w { + return fail("VEX.W must be zero") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + op, bytes := Op("VBROADCASTSS"), 4 + if double { + op, bytes = "VBROADCASTSD", 8 + } + var source Operand + var size int + var err error + if p.evex { + source, size, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", bytes) + } else { + source, size, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + reg := int(code[p.modRM]>>3&7) + p.r*8 + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", []string{"X", "Y", "Z"}[p.vectorLength], reg))} + args := []Operand{source, destination} + printed := source.String() + ", " + destination.String() + if p.mask != 0 { + mask := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))} + args = []Operand{source, mask, destination} + printed = source.String() + ", " + mask.String() + ", " + destination.String() + } + if p.zero { + op += ".Z" + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, printed)}, p.modRM + size, true, nil +} diff --git a/amd64_raw_scalar_broadcast_test.go b/amd64_raw_scalar_broadcast_test.go new file mode 100644 index 000000000..0f0a26f8e --- /dev/null +++ b/amd64_raw_scalar_broadcast_test.go @@ -0,0 +1,296 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86RawScalarBroadcastGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Go 1.27 _yvbroadcastss has 2 VEX + 6 EVEX rows; + // _yvbroadcastsd has 1 VEX + 4 EVEX rows (no X destination). + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT broadcasts(SB),4,$0-0\n") + regs := []int{0, 7} + if arch == "amd64" { + regs = append(regs, 15, 16, 31) + } + for _, op := range []string{"VBROADCASTSS", "VBROADCASTSD"} { + for _, width := range []string{"X", "Y", "Z"} { + if op == "VBROADCASTSD" && width == "X" { + continue + } + for _, reg := range regs { + for _, input := range []string{fmt.Sprintf("X%d", reg), "(AX)", "16(BX)(CX*4)", "-64(SI)"} { + for _, form := range []struct{ suffix, mask string }{{"", ""}, {"", "K1,"}, {".Z", "K7,"}} { + fmt.Fprintf(&source, "%s%s %s,%s%s%d\n", op, form.suffix, input, form.mask, width, reg) + } + } + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "broadcasts", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"broadcasts": {Name: "broadcasts", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-scalar-broadcast.ll", "raw-scalar-broadcast.o", ir) + } + }) + } +} + +func TestX86RawScalarBroadcastEncodingAxes(t *testing.T) { + count := 0 + for _, evex := range []bool{false, true} { + limit, widths := 16, 2 + if evex { + limit, widths = 32, 3 + } + for _, double := range []bool{false, true} { + for width := 0; width < widths; width++ { + if double && width == 0 { + continue + } + for src := 0; src < limit; src++ { + for dst := 0; dst < limit; dst++ { + mask, zero := 0, false + if evex { + mask = (src + dst) & 7 + zero = mask != 0 && dst&1 != 0 + } + code := encodeRawScalarMove(evex, 1, 0x10, src, 0, dst, mask, zero) + op, opcode := Op("VBROADCASTSS"), byte(0x18) + if double { + op, opcode = "VBROADCASTSD", 0x19 + } + if evex { + code[1] = code[1]&0xf0 | 2 + code[3] |= byte(width) << 5 + code[4] = opcode + if double { + code[2] |= 0x80 + } + } else { + code[1] = code[1]&0xe0 | 2 + code[2] |= byte(width) << 2 + code[3] = opcode + } + got, size, ok, err := decodedX86ScalarBroadcastInstruction(code, 64) + if zero { + op += ".Z" + } + args := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", src))}} + if mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", []string{"X", "Y", "Z"}[width], dst))}) + if err != nil || !ok || size != len(code) || got.Op != op || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x=%+v size=%d ok=%v err=%v; want %s %+v", code, got, size, ok, err, op, args) + } + count++ + } + } + } + } + } + if count != 5888 { + t.Fatalf("tested %d encodings, want 5888", count) + } +} + +func TestX86RawScalarBroadcastRejectsReservedEncodings(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe2, 0x79, 0x18}, // Missing ModRM. + {0xc4, 0xe2, 0x79, 0x18, 0x04}, // Missing SIB. + {0xc4, 0xe2, 0x79, 0x18, 0x45}, // Missing displacement. + {0xc4, 0xe2, 0x79, 0x19, 0xc0}, // SD has no 128-bit output. + {0xc4, 0xe2, 0xf9, 0x18, 0xc0}, // VEX.W must be zero. + {0xc4, 0xe2, 0x71, 0x18, 0xc0}, // Reserved vvvv. + {0x67, 0xc4, 0xe2, 0x79, 0x18, 0x00}, + {0x62, 0xf2, 0x79, 0x08, 0x18, 0xc0}, // Missing EVEX fixed bit. + {0x62, 0xf2, 0xfd, 0x08, 0x18, 0xc0}, // SS W1. + {0x62, 0xf2, 0x7d, 0x28, 0x19, 0xc0}, // SD W0. + {0x62, 0xf2, 0x7d, 0x00, 0x18, 0xc0}, // Reserved V'. + {0x62, 0xf2, 0x7d, 0x68, 0x18, 0xc0}, // Reserved LL. + {0x62, 0xf2, 0x7d, 0x18, 0x18, 0xc0}, // Reserved EVEX.b. + {0x62, 0xf2, 0x7d, 0x88, 0x18, 0xc0}, // Zeroing with K0. + } { + if _, _, matched, err := decodedX86ScalarBroadcastInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x matched=%v err=%v", code, matched, err) + } + } +} + +func TestX86ScalarBroadcastGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"VBROADCASTSS", "VBROADCASTSD"} { + instructions := []string{op + " AX,Y0", op + " Y0,Y1", op + " X0,AX", op + " X0,K0,Y0", op + ".Z X0,Y0", op + ".SAE X0,Y0", op + " X0,K1,(AX)"} + if op == "VBROADCASTSD" { + instructions = append(instructions, op+" X0,X1") + } + if arch == "386" { + instructions = append(instructions, op+" X0,Z8") + } + for _, instruction := range instructions { + source := "TEXT invalidbroadcast(SB),4,$0-0\n" + instruction + "\nRET\n" + dir := t.TempDir() + name := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(name, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), name) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + if output, err := cmd.CombinedOutput(); err == nil { + if instruction != op+".Z X0,Y0" || goToolchainAtLeast(runtime.Version(), 1, 27) { + t.Fatalf("Go %s accepted %s; repair the oracle expectation: %s", arch, instruction, output) + } + // Old Go accepts maskless .Z but emits a reserved EVEX + // zeroing/K0 encoding. Verify both that exact historical + // mismatch and the raw decoder's rejection, then require + // the named translator's rejection below too. + code := assembleX87ControlBytes(t, arch, source) + mode := 64 + if arch == "386" { + mode = 32 + } + p, prefixMatched := decodeX86RawVectorEncoding(code) + _, _, matched, err := decodedX86ScalarBroadcastInstruction(code, mode) + if !prefixMatched || !p.evex || !p.zero || p.mask != 0 || !matched || err == nil || !strings.Contains(err.Error(), "zeroing requires a nonzero mask") { + t.Fatalf("historical maskless .Z did not emit rejected EVEX zeroing/K0: %x %v", code, err) + } + t.Logf("%s/%s emits reserved zeroing/K0; both translators reject it", runtime.Version(), arch) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + continue + } + _, err = Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"invalidbroadcast": {Name: "invalidbroadcast", Ret: Void}}}) + if err == nil { + t.Errorf("%s accepted Go-rejected %s", arch, instruction) + } + } + } + } +} + +func TestX86ScalarBroadcastMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; separate five-target object tests are required") + } + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, spec := range []struct { + op string + bits int + }{{"VBROADCASTSS", 32}, {"VBROADCASTSD", 64}} { + for width, reg := range []string{"X0", "Y0", "Z0"} { + if spec.bits == 64 && width == 0 { + continue + } + for form := 0; form < 6; form++ { + name := fmt.Sprintf("broadcast%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ old+0(FP),BX\nMOVQ mem+8(FP),AX\nMOVQ out+16(FP),DI\nMOVQ mask+24(FP),CX\nKMOVQ CX,K7\nVMOVUPS (BX),Z0\n", name) + input := "(AX)" + if form >= 3 { + source.WriteString("VMOVUPS (AX),X1\n") + input = "X1" + } + op, mask := spec.op, "" + if form%3 != 0 { + mask = "K7," + } + if form%3 == 2 { + op += ".Z" + } + fmt.Fprintf(&source, "%s %s,%s%s\nVMOVUPS Z0,(DI)\nVMOVUPS Y0,64(DI)\nVMOVUPS X0,96(DI)\nRET\n", op, input, mask, reg) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *,uint64_t);\n", name) + faultcheck := 0 + if form < 3 && form%3 != 0 { + faultcheck = 1 + } + fmt.Fprintf(&checks, "if(check(%s,%d,%d,%d,%d)) return %d;\n", name, 16< +#include +` + declarations.String() + ` +static int check(void (*fn)(const void *,const void *,void *,uint64_t),int bytes,int lane,int form,int faultcheck) { + uint64_t masks[]={0,1,2,UINT64_C(0xaaaaaaaaaaaaaaaa),UINT64_MAX,UINT64_C(1)<<63}; + for(int m=0;m<6;m++) { + _Alignas(64) uint8_t old[64],mem[16],out[112]; + for(int i=0;i<64;i++) old[i]=(uint8_t)(197-i*3); + for(int i=0;i<16;i++) mem[i]=(uint8_t)(17+i*7); + uint64_t active=masks[m]&((UINT64_C(1)<<(bytes/lane))-1); + fn(old,faultcheck && !active ? 0 : mem,out,masks[m]); + for(int view=0;view<3;view++) for(int i=0;i<(64>>view);i++) { + int enabled=form==0 || ((masks[m]>>(i/lane))&1); + uint8_t want=i>=bytes ? 0 : enabled ? mem[i%lane] : form==2 ? 0 : old[i]; + int offset=view==0 ? 0 : view==1 ? 64 : 96; + if(out[offset+i]!=want) {fprintf(stderr,"broadcast mismatch bytes=%d lane=%d form=%d mask=%d view=%d byte=%d\n",bytes,lane,form,m,view,i);return 1;} + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + compileAndRunRuntimeTestForTarget(t, llc, clang, "scalar_broadcast", triple, ir, mainC, prefix) +} diff --git a/amd64_raw_scalar_extract_test.go b/amd64_raw_scalar_extract_test.go new file mode 100644 index 000000000..fe211e74e --- /dev/null +++ b/amd64_raw_scalar_extract_test.go @@ -0,0 +1,197 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +type rawPackedScalarExtractSpec struct { + op Op + mapNumber byte + opcode byte + width64 bool +} + +var rawPackedScalarExtractSpecs = []rawPackedScalarExtractSpec{ + {op: "VPEXTRB", mapNumber: 3, opcode: 0x14}, + {op: "VPEXTRW", mapNumber: 3, opcode: 0x15}, + {op: "VPEXTRW", mapNumber: 1, opcode: 0xc5}, + {op: "VPEXTRD", mapNumber: 3, opcode: 0x16}, + {op: "VPEXTRQ", mapNumber: 3, opcode: 0x16, width64: true}, + {op: "VEXTRACTPS", mapNumber: 3, opcode: 0x17}, +} + +func encodeX86RawVEXPackedScalarExtract(spec rawPackedScalarExtractSpec, source, destination int, immediate byte) []byte { + p0 := byte((^source>>3)&1)<<7 | 0x40 | byte((^destination>>3)&1)<<5 | spec.mapNumber + p1 := byte(0x79) + if spec.width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | source&7<<3 | destination&7) + return []byte{0xc4, p0, p1, spec.opcode, modRM, immediate} +} + +func encodeX86RawEVEXPackedScalarExtract(spec rawPackedScalarExtractSpec, source, destination int, immediate byte) []byte { + p0 := byte((^source>>3)&1)<<7 | + 0x40 | + byte((^destination>>3)&1)<<5 | + byte((^source>>4)&1)<<4 | + spec.mapNumber + p1 := byte(0x7d) + if spec.width64 { + p1 |= 0x80 + } + modRM := byte(0xc0 | source&7<<3 | destination&7) + return []byte{0x62, p0, p1, 0x08, spec.opcode, modRM, immediate} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVEXTRACTPS(t *testing.T) { + // vmovss %xmm0, (%rdi); vextractps $1, %xmm0, 4(%rdi) + code := []byte{0xc5, 0xfa, 0x11, 0x07, 0xc4, 0xe3, 0x79, 0x17, 0x47, 0x04, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VEXTRACTPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VMOVSS X0, 0(DI) ") || !strings.HasPrefix(decoded[1].Raw, "VEXTRACTPS $1, X0, 4(DI) ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawPackedScalarExtractCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawPackedScalarExtractSpecs { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawVEXPackedScalarExtract(spec, source, destination, immediate) + got, length, ok, err := decodedX86PackedScalarExtractInstruction(code, 64) + want := fmt.Sprintf("%s $%d, X%d, %s", spec.op, immediate, source, mustDecodedX86GeneralRegister(t, destination)) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for source := 0; source < 32; source++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(source*16 + destination) + code := encodeX86RawEVEXPackedScalarExtract(spec, source, destination, immediate) + got, length, ok, err := decodedX86PackedScalarExtractInstruction(code, 64) + want := fmt.Sprintf("%s $%d, X%d, %s", spec.op, immediate, source, mustDecodedX86GeneralRegister(t, destination)) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + if count != 4608 { + t.Fatalf("covered %d packed scalar-extract register encodings, want 4608", count) + } +} + +func TestDecodedX86RawPackedScalarExtractMemoryAndInvalidForms(t *testing.T) { + vexMemory := []byte{0xc4, 0x03, 0x79, 0x17, 0x7c, 0x8b, 0x01, 0xff} + got, length, ok, err := decodedX86PackedScalarExtractInstruction(vexMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(vexMemory) || got.Op != "VEXTRACTPS" || len(got.Args) != 3 || got.Args[1].String() != "X15" || got.Args[2].Kind != OpMem || !reflect.DeepEqual(got.Args[2].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + + evexMemory := []byte{0x62, 0x03, 0xfd, 0x08, 0x16, 0x7c, 0x8b, 0x02, 0x01} + got, length, ok, err = decodedX86PackedScalarExtractInstruction(evexMemory, 64) + wantMemory.Off = 16 + if err != nil || !ok || length != len(evexMemory) || got.Op != "VPEXTRQ" || len(got.Args) != 3 || got.Args[1].String() != "X31" || got.Args[2].Kind != OpMem || !reflect.DeepEqual(got.Args[2].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + twoByteVEX := []byte{0xc5, 0xf9, 0xc5, 0xc1, 0x01} + got, length, ok, err = decodedX86PackedScalarExtractInstruction(twoByteVEX, 64) + if err != nil || !ok || length != len(twoByteVEX) || got.Raw != "VPEXTRW $1, X0, CX" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", twoByteVEX, got, length, ok, err) + } + + validVEX := encodeX86RawVEXPackedScalarExtract(rawPackedScalarExtractSpecs[0], 7, 7, 1) + validEVEX := encodeX86RawEVEXPackedScalarExtract(rawPackedScalarExtractSpecs[0], 31, 15, 1) + for name, invalid := range map[string][]byte{ + "vex reserved vvvv": {validVEX[0], validVEX[1], validVEX[2] &^ 0x08, validVEX[3], validVEX[4], validVEX[5]}, + "vex vector length": {validVEX[0], validVEX[1], validVEX[2] | 0x04, validVEX[3], validVEX[4], validVEX[5]}, + "evex mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 1, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex vector length": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x20, validEVEX[4], validEVEX[5], validEVEX[6]}, + "evex missing fixed bit": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x04, validEVEX[3], validEVEX[4], validEVEX[5], validEVEX[6]}, + "truncated vex immediate": validVEX[:len(validVEX)-1], + "truncated evex immediate": validEVEX[:len(validEVEX)-1], + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86PackedScalarExtractInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + if instruction, _, matched, decodeErr := decodedX86PackedScalarExtractInstruction(validEVEX, 32); !matched || decodeErr == nil { + t.Fatalf("386 EVEX encoding %x decoded as %+v, ok=%v, err=%v", validEVEX, instruction, matched, decodeErr) + } + extended386 := encodeX86RawVEXPackedScalarExtract(rawPackedScalarExtractSpecs[0], 8, 0, 1) + if instruction, _, matched, decodeErr := decodedX86PackedScalarExtractInstruction(extended386, 32); !matched || decodeErr == nil { + t.Fatalf("386 extended encoding %x decoded as %+v, ok=%v, err=%v", extended386, instruction, matched, decodeErr) + } +} + +func TestTranslateX86RawPackedScalarExtractTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_packed_scalar_extract(SB),NOSPLIT,$0-0\n") + for _, spec := range rawPackedScalarExtractSpecs { + code := encodeX86RawVEXPackedScalarExtract(spec, target.registerMax, target.registerMax, 1) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + if target.goarch == "amd64" { + code = encodeX86RawEVEXPackedScalarExtract(spec, 31, 15, 1) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_packed_scalar_extract": {Name: "raw_packed_scalar_extract", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-packed-scalar-extract-"+target.name+".ll", "raw-packed-scalar-extract-"+target.name+".o", ir) + }) + } +} + +func mustDecodedX86GeneralRegister(t *testing.T, number int) Reg { + t.Helper() + reg, ok := decodedX86GeneralRegister(number) + if !ok { + t.Fatalf("missing general register %d", number) + } + return reg +} diff --git a/amd64_raw_scalar_flag_compare.go b/amd64_raw_scalar_flag_compare.go new file mode 100644 index 000000000..1f4c245a7 --- /dev/null +++ b/amd64_raw_scalar_flag_compare.go @@ -0,0 +1,71 @@ +package plan9asm + +import "fmt" + +// Go's four _yvcomisd entries share X/m,X and EVEX register-only SAE. +// Prefix fields stay typed: EVEX.b means SAE here, never memory broadcast; +// disp8 scales by the scalar width. VEX W and EVEX LL are ignored. Intel +// SDM COMISS/UCOMISS explicitly warns that VEX.L=1 can behave unpredictably +// across processor generations; decoder acceptance is not a portable opcode +// contract and cannot authorize changing those original bytes to VEX.L=0. +func decodedX86ScalarFlagCompareInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || (p.opcode != 0x2e && p.opcode != 0x2f) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar flag compare: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if (p.pp != 0 && p.pp != 1) || p.upper != 0 || p.mask != 0 || p.zero { + return fail("reserved prefix, vvvv/V', mask or zeroing field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + double := p.pp == 1 + if p.evex && (!p.fixed || p.w != double) { + return fail("invalid EVEX fixed or scalar width bit") + } + if p.broadcast && code[p.modRM]>>6 != 3 { + return fail("SAE requires a register source") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + op := Op("VCOMISS") + if p.opcode == 0x2e { + op = "VUCOMISS" + } + bytes := 4 + if double { + op = op[:len(op)-1] + "D" + bytes = 8 + } + if p.broadcast { + op += ".SAE" + } + var source Operand + var size int + var err error + if p.evex { + source, size, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", bytes) + } else { + source, size, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + if !p.evex && p.vectorLength != 0 { + return Instr{}, 0, true, fmt.Errorf("%w: scalar flag comparison VEX.L=1 lacks a portable physical opcode contract", ErrProbeNeedsContext) + } + dst := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", int(code[p.modRM]>>3&7)+p.r*8))} + return Instr{Op: op, Args: []Operand{source, dst}, Raw: fmt.Sprintf("%s %s, %s", op, source.String(), dst.String()), + x86ScalarFlagCompareEVEX: p.evex, + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_scalar_flag_compare_literal_test.go b/amd64_raw_scalar_flag_compare_literal_test.go new file mode 100644 index 000000000..1b6ffc039 --- /dev/null +++ b/amd64_raw_scalar_flag_compare_literal_test.go @@ -0,0 +1,143 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawScalarFlagCompareLiteral(op string, encoding string) []byte { + pp, opcode := 0, 0x2f + if strings.HasPrefix(op, "VU") { + opcode = 0x2e + } + width := 4 + if strings.HasSuffix(op, "SD") { + pp, width = 1, 8 + } + var code []byte + if encoding == "vex2" { + code = []byte{0xc5, byte(0xf8 | pp), byte(opcode), 0x05} + } else { + evex := encoding == "evex" + code = encodeRawScalarMove(evex, pp, opcode, 0, 0, 0, 0, false) + if evex && width == 8 { + code[2] |= 0x80 + } + code[len(code)-1] = 0x05 + } + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*37+21)) + } + return code +} + +func TestDecodeX86RawScalarFlagCompareRIPDataCompleteFamily(t *testing.T) { + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + name := op + "/" + encoding + t.Run(name, func(t *testing.T) { + code := x86RawScalarFlagCompareLiteral(op, encoding) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != Op(op) || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local scalar", code, decoded, op) + } + }) + } + } +} + +func TestDecodeX86RawScalarFlagCompareRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawScalarFlagCompareLiteral("VUCOMISD", "evex") + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + "EVEX SAE memory": func(code []byte) []byte { + code[3] |= 0x10 + return code + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawScalarFlagCompareRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + index := 0 + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + code := x86RawScalarFlagCompareLiteral(op, encoding) + name := fmt.Sprintf("flagCompareLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + width := 4 + if strings.HasSuffix(op, "SD") { + width = 8 + } + pools = append(pools, code[len(code)-width:]) + index++ + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "flag-compare-literal.ll", "flag-compare-literal.o", ir) + }) + } +} diff --git a/amd64_raw_scalar_flag_compare_test.go b/amd64_raw_scalar_flag_compare_test.go new file mode 100644 index 000000000..faa0502b7 --- /dev/null +++ b/amd64_raw_scalar_flag_compare_test.go @@ -0,0 +1,312 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawScalarFlagCompareGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + // Go 1.27 _yvcomisd has one VEX and one EVEX X/m,X row for + // each of the four opcodes. Only EVEX register sources permit SAE. + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + regs := []int{0, 7} + if arch == "amd64" { + regs = append(regs, 8, 15, 16, 31) + } + var source strings.Builder + source.WriteString("TEXT flagcompare(SB),4,$0-0\n") + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, dst := range regs { + for _, src := range regs { + fmt.Fprintf(&source, "%s X%d,X%d\n%s.SAE X%d,X%d\n", op, src, dst, op, src, dst) + } + for _, mem := range []string{"(AX)", "12(BX)(CX*4)", "-32(SI)", "8192(DI)"} { + fmt.Fprintf(&source, "%s %s,X%d\n", op, mem, dst) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "flagcompare", code) + triples := []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} + if arch == "386" { + triples = []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} + } + for _, triple := range triples { + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"flagcompare": {Name: "flagcompare", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-flag-compare.ll", "raw-flag-compare.o", ir) + } + t.Logf("checked %d Go encodings", len(want)-1) + }) + } +} + +func TestX86RawScalarFlagCompareEncodingAxes(t *testing.T) { + count, contexts := 0, 0 + for _, evex := range []bool{false, true} { + limit, variants := 16, 4 // WIG; L1 is an explicit source-context boundary. + if evex { + limit, variants = 32, 8 // EVEX.LL ignored, SAE on/off. + } + for _, double := range []bool{false, true} { + for _, unordered := range []bool{false, true} { + pp, opcode, op := 0, 0x2f, Op("VCOMISS") + if unordered { + opcode, op = 0x2e, "VUCOMISS" + } + if double { + pp, op = 1, op[:len(op)-1]+"D" + } + for src := 0; src < limit; src++ { + for dst := 0; dst < limit; dst++ { + for variant := 0; variant < variants; variant++ { + code := encodeRawScalarMove(evex, pp, opcode, src, 0, dst, 0, false) + wantOp := op + if evex { + code[2] &^= 0x80 + if double { + code[2] |= 0x80 + } + code[3] |= byte(variant&3) << 5 + if variant&4 != 0 { + code[3] |= 0x10 + wantOp += ".SAE" + } + } else { + code[2] |= byte(variant&1)<<2 | byte(variant>>1)<<7 + } + got, size, matched, err := decodedX86ScalarFlagCompareInstruction(code, 64) + if !evex && variant&1 != 0 { + if !matched || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("VEX.L1 lost physical opcode boundary: %x %v", code, err) + } + count++ + contexts++ + continue + } + want := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", src))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}} + if err != nil || !matched || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v size=%d matched=%v err=%v; want %s %+v", code, got, size, matched, err, wantOp, want) + } + _, _, matched, err = decodedX86ScalarFlagCompareInstruction(code, 32) + if !matched || (err != nil) != (src >= 8 || dst >= 8) { + t.Fatalf("386 decode %x matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 36864 || contexts != 2048 { + t.Fatalf("tested %d encodings with %d VEX.L1 contexts, want 36864/2048", count, contexts) + } +} + +func TestX86RawScalarFlagCompareInvalidAndMemoryForms(t *testing.T) { + for _, code := range [][]byte{nil, {0xc5}, {0xc4, 0xe2, 0x78, 0x2e, 0xc0}, {0xc5, 0xf8, 0x58, 0xc0}} { + if _, _, matched, _ := decodedX86ScalarFlagCompareInstruction(code, 64); matched { + t.Fatalf("matched unrelated encoding %x", code) + } + } + if _, _, matched, err := decodedX86ScalarFlagCompareInstruction([]byte{0xc5, 0xf8, 0x2e, 0xc0}, 16); !matched || err == nil { + t.Fatalf("accepted unsupported mode: matched=%v err=%v", matched, err) + } + for _, code := range [][]byte{ + {0xc5, 0xf8, 0x2e}, + {0xc5, 0xf8, 0x2e, 0x04}, // Missing SIB. + {0xc5, 0xf8, 0x2e, 0x45}, // Missing displacement. + {0xc5, 0xfa, 0x2e, 0xc0}, // Wrong pp. + {0xc5, 0xfb, 0x2e, 0xc0}, + {0xc5, 0xf0, 0x2e, 0xc0}, // Reserved vvvv. + {0x67, 0xc5, 0xf8, 0x2e, 0x00}, + {0x62, 0xf1, 0x78, 0x08, 0x2e, 0xc0}, // Missing fixed bit. + {0x62, 0xf1, 0xfc, 0x08, 0x2e, 0xc0}, // SS W1. + {0x62, 0xf1, 0x7d, 0x08, 0x2e, 0xc0}, // SD W0. + {0x62, 0xf1, 0x7c, 0x00, 0x2e, 0xc0}, // Reserved V'. + {0x62, 0xf1, 0x7c, 0x88, 0x2e, 0xc0}, // Zeroing. + {0x62, 0xf1, 0x7c, 0x09, 0x2e, 0xc0}, // Masking. + {0x62, 0xf1, 0x7c, 0x18, 0x2e, 0x00}, // SAE with memory. + } { + if _, _, matched, err := decodedX86ScalarFlagCompareInstruction(code, 64); !matched || err == nil { + t.Fatalf("accepted %x matched=%v err=%v", code, matched, err) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, double := range []bool{false, true} { + for _, evex := range []bool{false, true} { + code := []byte{segment.prefix, 0xc4, 0x01, 0x78, 0x2e, 0x64, 0x88, 0xf9} + wantOff, op := int64(-7), Op("VUCOMISS") + if double { + code[3] |= 1 + op = "VUCOMISD" + } + if evex { + code = []byte{segment.prefix, 0x62, 0x01, 0x7c, 0x08, 0x2e, 0x64, 0x88, 0xf9} + wantOff *= 4 + if double { + code[3] |= 0x81 + wantOff *= 2 + } + } + dst := Reg("X12") + if evex { + dst = "X28" + } + want := []Operand{{Kind: OpMem, Mem: MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: wantOff}}, {Kind: OpReg, Reg: dst}} + got, size, matched, err := decodedX86ScalarFlagCompareInstruction(code, 64) + if err != nil || !matched || size != len(code) || got.Op != op || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x=%+v %v; want %s %+v", code, got, err, op, want) + } + } + } + } +} + +func TestX86ScalarFlagCompareGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, args := range []string{" Y0,X1", " X0,Y1", " Z0,X1", " (AX),(BX)", " X0,K1,X1", ".Z X0,X1", ".BCST (AX),X1", ".SAE (AX),X1", ".RN_SAE X0,X1"} { + line := op + args + source := "TEXT rejected(SB),4,$0-0\n" + line + "\nRET\n" + goAssemblerOracleOutput(t, arch, source, "", false) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + if _, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"rejected": {Name: "rejected", Ret: Void}}}); err == nil { + t.Fatalf("translator accepted %s for %s", line, arch) + } + } + } + } +} + +func x86COMIRawNativeCompare(t *testing.T) { + t.Helper() + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + source, sigs, mainC, _ := x86COMIRawNativeFixture(t) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: execution.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("IR_O2=%v", optimized), func(t *testing.T) { + input := ir + if optimized { + input = x86MulDivRoundOptimize(t, input) + } + for _, backend := range []string{"-O0", "-O2"} { + t.Run("backend"+backend, func(t *testing.T) { + execution.compileAndRunLLVM(t, "raw-comi", input, mainC, backend) + }) + } + }) + } +} + +func x86COMIRawNativeFixture(t *testing.T) (string, map[string]FuncSig, string, int) { + t.Helper() + var source, declarations, checks strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, op := range []string{"VCOMISS", "VCOMISD", "VUCOMISS", "VUCOMISD"} { + for _, reg := range []int{0, 16} { + for _, form := range []string{"register", "memory", "sae"} { + name := fmt.Sprintf("flagcompare%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-24\nMOVQ a+0(FP),AX\nMOVQ b+8(FP),BX\nMOVQ out+16(FP),DI\nVMOVUPS (AX),Z%d\nVMOVUPS (BX),Z%d\nMOVL $2147483647,CX\nADDL $1,CX\n", name, reg, reg+1) + input, suffix := fmt.Sprintf("X%d", reg+1), "" + if form == "memory" { + input = "(BX)" + } + if form == "sae" { + suffix = ".SAE" + } + code := assembleX87ControlBytes(t, "amd64", fmt.Sprintf("TEXT probe(SB),4,$0-0\n%s%s %s,X%d\nRET\n", op, suffix, input, reg)) + for _, b := range code[:len(code)-1] { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("SETEQ (DI)\nSETCS 1(DI)\nSETPS 2(DI)\nSETMI 3(DI)\nSETOS 4(DI)\n") + fmt.Fprintf(&source, "VMOVUPS Z%d,8(DI)\nVMOVUPS Z%d,72(DI)\nRET\n", reg, reg+1) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}}}} + fmt.Fprintf(&declarations, "extern void %s(const void *,const void *,void *);\n", name) + double := 0 + if strings.HasSuffix(op, "SD") { + double = 1 + } + fmt.Fprintf(&checks, "if(check(%s,%d)) return %d;\n", name, double, index) + } + } + } + // Compare EFLAGS with native UCOMIS{S,D}. This oracle checks comparison, + // NaN, signed-zero and register-preservation semantics, not MXCSR exception + // delivery or SAE suppression, which require a separate floating-state audit. + mainC := `#include +#include +#include +` + declarations.String() + ` +static const uint64_t doubles[]={0,0x8000000000000000ULL,0x3ff0000000000000ULL,0xbff0000000000000ULL,0x4004000000000000ULL,0xc004000000000000ULL,0x7ff0000000000000ULL,0xfff0000000000000ULL,0x7ff8000000000001ULL,0x7ff0000000000001ULL,1,0x7fefffffffffffffULL}; +static const uint32_t singles[]={0,0x80000000,0x3f800000,0xbf800000,0x40200000,0xc0200000,0x7f800000,0xff800000,0x7fc00001,0x7f800001,1,0x7f7fffff}; +static int check(void (*fn)(const void *,const void *,void *),int isDouble) { + for(int i=0;i<12;i++) for(int j=0;j<12;j++) { + uint8_t a[64],b[64],out[136]={0},want[5]; + for(int k=0;k<64;k++) {a[k]=(uint8_t)(k*7+91);b[k]=(uint8_t)(k*31+5);} + if(isDouble) { + double x,y;memcpy(a,&doubles[i],8);memcpy(b,&doubles[j],8);memcpy(&x,a,8);memcpy(&y,b,8); + __asm__ volatile("ucomisd %6,%5; sete %0; setb %1; setp %2; sets %3; seto %4":"=qm"(want[0]),"=qm"(want[1]),"=qm"(want[2]),"=qm"(want[3]),"=qm"(want[4]):"x"(x),"x"(y):"cc"); + } else { + float x,y;memcpy(a,&singles[i],4);memcpy(b,&singles[j],4);memcpy(&x,a,4);memcpy(&y,b,4); + __asm__ volatile("ucomiss %6,%5; sete %0; setb %1; setp %2; sets %3; seto %4":"=qm"(want[0]),"=qm"(want[1]),"=qm"(want[2]),"=qm"(want[3]),"=qm"(want[4]):"x"(x),"x"(y):"cc"); + } + fn(a,b,out); + if(memcmp(out,want,5)||memcmp(out+8,a,64)||memcmp(out+72,b,64)) { + fprintf(stderr,"flag compare mismatch double=%d i=%d j=%d\n",isDouble,i,j);return 1; + } + } + return 0; +} +int main(void) { +` + checks.String() + "return 0;\n}\n" + return source.String(), sigs, mainC, index +} diff --git a/amd64_raw_scalar_float_to_int.go b/amd64_raw_scalar_float_to_int.go new file mode 100644 index 000000000..60a7c82bd --- /dev/null +++ b/amd64_raw_scalar_float_to_int.go @@ -0,0 +1,108 @@ +package plan9asm + +import "fmt" + +// Decode the complete Go 1.27 scalar floating-point-to-GP family. Signed +// forms have VEX and EVEX encodings; unsigned forms are EVEX-only. The named +// lowerer and this decoder share amd64ScalarFloatToIntSpecs so W, F2/F3, +// truncation, and signedness cannot drift between the two entry points. +func decodedX86ScalarFloatToIntegerInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || p.mapNumber != 1 || (p.pp != 2 && p.pp != 3) { + return Instr{}, 0, false, nil + } + if p.opcode != 0x2c && p.opcode != 0x2d && p.opcode != 0x78 && p.opcode != 0x79 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar float-to-integer: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + unsigned := p.opcode == 0x78 || p.opcode == 0x79 + truncating := p.opcode == 0x2c || p.opcode == 0x78 + if unsigned && !p.evex { + return fail("unsigned conversion requires EVEX") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.upper != 0 { + return fail("reserved vvvv/V' field") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.b != 0 || p.x != 0) { + return fail("extended register encoding in 32-bit mode") + } + if p.evex { + if !p.fixed || p.mask != 0 || p.zero || p.r >= 2 { + return fail("invalid EVEX fixed, mask, zeroing or GP extension field") + } + if p.broadcast && code[p.modRM]>>6 != 3 { + return fail("SAE or embedded rounding requires register source") + } + // EVEX.L'L is ignored without embedded rounding (and for CVTT SAE). + // Unlike VEX.L=1, Intel defines these as LLIG encodings. + } else if p.vectorLength != 0 { + return fail("scalar conversion requires VEX.128") + } + + inputBits := 32 + if p.pp == 3 { + inputBits = 64 + } + outputBits := 32 + if p.w { + outputBits = 64 + } + var op Op + for name, spec := range amd64ScalarFloatToIntSpecs { + if spec.vector && spec.inputBits == inputBits && spec.outputBits == outputBits && + spec.truncating == truncating && spec.unsigned == unsigned { + op = name + break + } + } + if op == "" { + return fail("encoding has no Go scalar conversion form") + } + if p.evex && p.broadcast { + if truncating { + op += ".SAE" + } else { + op += [...]Op{".RN_SAE", ".RD_SAE", ".RU_SAE", ".RZ_SAE"}[p.vectorLength] + } + } + var source Operand + var consumed int + var err error + if p.evex { + source, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", inputBits/8) + } else { + source, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + destinationNumber := int(code[p.modRM]>>3&7) + (p.r&1)*8 + destinationReg, valid := decodedX86GeneralRegister(destinationNumber) + if !valid { + return fail("invalid GP destination") + } + destination := Operand{Kind: OpReg, Reg: destinationReg} + instruction := Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + x86ScalarFloatIntegerEVEX: p.evex, + } + arch := "amd64" + if mode == 32 { + arch = "386" + } + _, _, err = x86ScalarFloatIntegerProperties(instruction, arch) + return instruction, p.modRM + consumed, true, err +} diff --git a/amd64_raw_scalar_float_to_int_test.go b/amd64_raw_scalar_float_to_int_test.go new file mode 100644 index 000000000..2fe030bda --- /dev/null +++ b/amd64_raw_scalar_float_to_int_test.go @@ -0,0 +1,266 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func rawScalarFloatToIntBytes(spec amd64ScalarFloatToIntSpec, memory, evex bool) []byte { + opcode := byte(0x2d) + if spec.truncating { + opcode = 0x2c + } + if spec.unsigned { + opcode = 0x79 + if spec.truncating { + opcode = 0x78 + } + } + pp := byte(2) // F3: single precision. + if spec.inputBits == 64 { + pp = 3 // F2: double precision. + } + modRM := byte(0xc1) // X1 -> AX. + if memory { + modRM = 0x08 // (AX) -> CX. + } + if evex { + p1 := byte(0x7c) | pp + if spec.outputBits == 64 { + p1 |= 0x80 + } + return []byte{0x62, 0xf1, p1, 0x08, opcode, modRM} + } + if spec.outputBits == 64 { + return []byte{0xc4, 0xe1, 0xf8 | pp, opcode, modRM} + } + return []byte{0xc5, 0xf8 | pp, opcode, modRM} +} + +func TestX86RawScalarFloatToIntegerCompleteGo127Family(t *testing.T) { + count := 0 + for op, spec := range amd64ScalarFloatToIntSpecs { + if !spec.vector { + continue + } + for _, evex := range []bool{false, true} { + if spec.unsigned && !evex { + continue // Go has no unsigned VEX encoding. + } + for _, memory := range []bool{false, true} { + name := fmt.Sprintf("%s/evex=%t/memory=%t", op, evex, memory) + t.Run(name, func(t *testing.T) { + code := rawScalarFloatToIntBytes(spec, memory, evex) + fn := Func{} + for _, value := range code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, "amd64") + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", code, got.Instrs, op) + } + wantSource := Operand{Kind: OpReg, Reg: "X1"} + wantDestination := AX + if memory { + wantSource = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + wantDestination = CX + } + args := got.Instrs[0].Args + if args[0].Kind != wantSource.Kind || args[0].Reg != wantSource.Reg || args[0].Mem != wantSource.Mem || args[1].Reg != wantDestination { + t.Fatalf("decoded %#x operands as %#v, want %+v, %s", code, args, wantSource, wantDestination) + } + }) + count++ + } + } + } + if count != 48 { + t.Fatalf("tested %d raw VEX/EVEX forms, want 48", count) + } +} + +func TestX86RawScalarFloatToIntegerRoundingAndEncodingAxes(t *testing.T) { + base := []byte{0x62, 0xf1, 0x7f, 0x08, 0x2d, 0xc1} + for _, tc := range []struct { + name string + p2 byte + want Op + }{ + {"round-nearest", 0x18, "VCVTSD2SI.RN_SAE"}, + {"round-down", 0x38, "VCVTSD2SI.RD_SAE"}, + {"round-up", 0x58, "VCVTSD2SI.RU_SAE"}, + {"round-zero", 0x78, "VCVTSD2SI.RZ_SAE"}, + } { + t.Run(tc.name, func(t *testing.T) { + code := append([]byte(nil), base...) + code[3] = tc.p2 + got, length, recognized, err := decodedX86ScalarFloatToIntegerInstruction(code, 64) + if !recognized || err != nil || length != len(code) || got.Op != tc.want { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s", code, got, length, recognized, err, tc.want) + } + }) + } + for _, tc := range []struct { + name string + code []byte + want Op + src Reg + dst Reg + }{ + {"signed-SAE", []byte{0x62, 0xf1, 0x7f, 0x18, 0x2c, 0xc1}, "VCVTTSD2SI.SAE", "X1", AX}, + {"unsigned-SAE", []byte{0x62, 0xf1, 0x7f, 0x18, 0x78, 0xc1}, "VCVTTSD2USIL.SAE", "X1", AX}, + {"extended-vector", []byte{0x62, 0xb1, 0x7f, 0x08, 0x78, 0xc1}, "VCVTTSD2USIL", "X17", AX}, + {"extended-GP", []byte{0x62, 0x71, 0x7f, 0x08, 0x78, 0xc1}, "VCVTTSD2USIL", "X1", "R8"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86ScalarFloatToIntegerInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) || got.Op != tc.want || + got.Args[0].Reg != tc.src || got.Args[1].Reg != tc.dst { + t.Fatalf("decode %#x = %+v, %d, %v, %v; want %s %s, %s", tc.code, got, length, recognized, err, tc.want, tc.src, tc.dst) + } + }) + } + for _, tc := range []struct { + name string + code []byte + off int64 + }{ + {"SS-disp8-scale", []byte{0x62, 0xf1, 0x7e, 0x08, 0x79, 0x48, 0x02}, 8}, + {"SD-disp8-scale", []byte{0x62, 0xf1, 0x7f, 0x08, 0x79, 0x48, 0x02}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + got, _, recognized, err := decodedX86ScalarFloatToIntegerInstruction(tc.code, 64) + if !recognized || err != nil || got.Args[0].Kind != OpMem || got.Args[0].Mem.Off != tc.off { + t.Fatalf("decode %#x = %+v, %v, %v; want displacement %d", tc.code, got, recognized, err, tc.off) + } + }) + } +} + +func TestX86RawScalarFloatToIntegerRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"VEX-vector-length", []byte{0xc5, 0xff, 0x2c, 0xc1}, 64}, + {"VEX-vvvv", []byte{0xc5, 0xeb, 0x2c, 0xc1}, 64}, + {"EVEX-mask", []byte{0x62, 0xf1, 0x7f, 0x09, 0x2c, 0xc1}, 64}, + {"EVEX-zero", []byte{0x62, 0xf1, 0x7f, 0x88, 0x2c, 0xc1}, 64}, + {"EVEX-vvvv", []byte{0x62, 0xf1, 0x6f, 0x08, 0x2c, 0xc1}, 64}, + {"EVEX-SAE-memory", []byte{0x62, 0xf1, 0x7f, 0x18, 0x2c, 0x00}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfb, 0x2c, 0xc1}, 64}, + {"missing-ModRM", []byte{0xc5, 0xfb, 0x2c}, 64}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86ScalarFloatToIntegerInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } +} + +func TestX86RawScalarFloatToIntegerCompilesEveryGoForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawscalarconvert(SB),4,$0-0\n") + for _, spec := range amd64ScalarFloatToIntSpecs { + if !spec.vector { + continue + } + for _, evex := range []bool{false, true} { + if spec.unsigned && !evex { + continue + } + for _, memory := range []bool{false, true} { + for _, value := range rawScalarFloatToIntBytes(spec, memory, evex) { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawscalarconvert": {Name: "rawscalarconvert", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-scalar-convert.ll", "raw-scalar-convert.o", ir) + }) + } +} + +func x86RawScalarFloatToIntegerNativeRuntime(t *testing.T) { + execution := x86FloatCompareRuntimeTools(t, x86FloatCompareEVEX) + const source = `TEXT rawfloattointsemantics(SB),4,$0-16 + MOVQ input+0(FP), BX + MOVQ out+8(FP), DI + BYTE $0xc5; BYTE $0xfb; BYTE $0x2c; BYTE $0x03 + MOVL AX, 0(DI) + BYTE $0xc5; BYTE $0xfb; BYTE $0x2d; BYTE $0x03 + MOVL AX, 4(DI) + BYTE $0x62; BYTE $0xf1; BYTE $0x7f; BYTE $0x08; BYTE $0x78; BYTE $0x03 + MOVL AX, 8(DI) + BYTE $0x62; BYTE $0xf1; BYTE $0x7f; BYTE $0x08; BYTE $0x79; BYTE $0x03 + MOVL AX, 12(DI) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: execution.triple, + Sigs: map[string]FuncSig{"rawfloattointsemantics": { + Name: "rawfloattointsemantics", Args: []LLVMType{Ptr, Ptr}, Ret: Void, + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, + {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern void rawfloattointsemantics(const double *, uint32_t *); +int main(void) { + double input = 2.75; + uint32_t out[4] = {0}; + rawfloattointsemantics(&input, out); + if (out[0] != 2) return 1; + if (out[1] != 3) return 2; + if (out[2] != 2) return 3; + if (out[3] != 3) return 4; + return 0; +} +` + execution.compileAndRunLLVM(t, "raw-float-to-integer", ir, mainC, "-O2") +} diff --git a/amd64_raw_scalar_integer_float_test.go b/amd64_raw_scalar_integer_float_test.go new file mode 100644 index 000000000..0a5ddc014 --- /dev/null +++ b/amd64_raw_scalar_integer_float_test.go @@ -0,0 +1,182 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXScalarIntegerFloat(pp byte, width64, vex3 bool, destination, passthrough, source int) []byte { + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if !vex3 { + vex1 := byte((1-(destination>>3)&1)<<7 | (^passthrough&15)<<3 | int(pp)) + return []byte{0xc5, vex1, 0x2a, modRM} + } + vex0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(source>>3)&1)<<5 | 1) + vex1 := byte((^passthrough&15)<<3 | int(pp)) + if width64 { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, 0x2a, modRM} +} + +func encodeX86EVEXScalarIntegerFloat(unsigned bool, pp byte, width64 bool, vectorBits byte, rounding bool, destination, passthrough, source int) []byte { + opcode := byte(0x2a) + if unsigned { + opcode = 0x7b + } + p0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(source>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte((^passthrough&15)<<3 | 1<<2 | int(pp)) + if width64 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^passthrough>>4&1)<<3) + if rounding { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func scalarIntegerFloatRawOp(unsigned bool, pp byte, width64 bool) Op { + prefix := "VCVTSI2" + if unsigned { + prefix = "VCVTUSI2" + } + result := "SS" + if pp == 3 { + result = "SD" + } + source := "L" + if width64 { + source = "Q" + } + return Op(prefix + result + source) +} + +func TestTranslateRawVEXScalarIntegerFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawScalarIntegerFloat(SB), $0-0 + LONG $0xc82aeac5 + LONG $0xc02a9ac5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawScalarIntegerFloat": {Name: "rawScalarIntegerFloat", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if strings.Count(ir, "call float @llvm.experimental.constrained.sitofp.f32.i32") != 2 { + t.Fatalf("raw VCVTSI2SSL lowering omitted signed conversions:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-scalar-integer-float.ll", "amd64-raw-scalar-integer-float.o", ir) +} + +func TestDecodedX86VEXScalarIntegerFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, pp := range []byte{2, 3} { + for destination := 0; destination < 16; destination++ { + for passthrough := 0; passthrough < 16; passthrough++ { + for source := 0; source < 8; source++ { + code := encodeX86VEXScalarIntegerFloat(pp, false, false, destination, passthrough, source) + assertDecodedX86ScalarIntegerFloat(t, code, scalarIntegerFloatRawOp(false, pp, false), destination, passthrough, source, "") + count++ + } + } + } + for _, width64 := range []bool{false, true} { + for destination := 0; destination < 16; destination++ { + for passthrough := 0; passthrough < 16; passthrough++ { + for source := 0; source < 16; source++ { + code := encodeX86VEXScalarIntegerFloat(pp, width64, true, destination, passthrough, source) + assertDecodedX86ScalarIntegerFloat(t, code, scalarIntegerFloatRawOp(false, pp, width64), destination, passthrough, source, "") + count++ + } + } + } + } + } + if count != 20480 { + t.Fatalf("covered %d VEX scalar integer-to-float encodings, want 20480", count) + } +} + +func TestDecodedX86EVEXScalarIntegerFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, unsigned := range []bool{false, true} { + for _, pp := range []byte{2, 3} { + for _, width64 := range []bool{false, true} { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + for destination := 0; destination < 32; destination++ { + for passthrough := 0; passthrough < 32; passthrough++ { + for source := 0; source < 16; source++ { + code := encodeX86EVEXScalarIntegerFloat(unsigned, pp, width64, vectorBits, false, destination, passthrough, source) + assertDecodedX86ScalarIntegerFloat(t, code, scalarIntegerFloatRawOp(unsigned, pp, width64), destination, passthrough, source, "") + count++ + } + } + } + } + } + } + } + if count != 393216 { + t.Fatalf("covered %d EVEX scalar integer-to-float encodings, want 393216", count) + } +} + +func TestDecodedX86EVEXScalarIntegerFloatRoundingMemoryAndInvalid(t *testing.T) { + for vectorBits, suffix := range []string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"} { + code := encodeX86EVEXScalarIntegerFloat(false, 2, true, byte(vectorBits), true, 17, 18, 9) + assertDecodedX86ScalarIntegerFloat(t, code, "VCVTSI2SSQ."+Op(suffix), 17, 18, 9, "") + } + memory := []byte{0x62, 0xe1, 0xff, 0x00, 0x2a, 0x60, 0x02} + got, length, ok, err := decodedX86ScalarIntegerFloatInstruction(memory, 64) + if err != nil || !ok || length != len(memory) || got.Op != "VCVTSI2SDQ" || len(got.Args) != 3 || got.Args[0].String() != "16(AX)" || got.Args[1].String() != "X16" || got.Args[2].String() != "X20" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", memory, got, length, ok, err) + } + + roundingMemory := encodeX86EVEXScalarIntegerFloat(false, 2, true, 0, true, 0, 0, 0) + roundingMemory[len(roundingMemory)-1] = 0 + nonRounding := encodeX86EVEXScalarIntegerFloat(false, 3, false, 0, true, 0, 0, 0) + invalid := [][]byte{ + encodeX86VEXScalarIntegerFloat(1, false, false, 0, 0, 0), + encodeX86EVEXScalarIntegerFloat(false, 2, false, 3, false, 0, 0, 0), + roundingMemory, + nonRounding, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86ScalarIntegerFloatInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} + +func assertDecodedX86ScalarIntegerFloat(t *testing.T, code []byte, op Op, destination, passthrough, source int, sourceText string) { + t.Helper() + got, length, ok, err := decodedX86ScalarIntegerFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if sourceText == "" { + sourceText = decodedX86ScalarIntegerFloatTestRegister(source) + } + wantArgs := []string{sourceText, fmt.Sprintf("X%d", passthrough), fmt.Sprintf("X%d", destination)} + if got.Op != op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, op, strings.Join(wantArgs, ", ")) + } +} + +func decodedX86ScalarIntegerFloatTestRegister(number int) string { + register, ok := decodedX86GeneralRegister(number) + if !ok { + return fmt.Sprintf("invalid%d", number) + } + return string(register) +} diff --git a/amd64_raw_scalar_move.go b/amd64_raw_scalar_move.go new file mode 100644 index 000000000..944e5d004 --- /dev/null +++ b/amd64_raw_scalar_move.go @@ -0,0 +1,167 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// x86RawVectorEncoding separates the shared VEX/EVEX prefix axes from a +// family's operand grammar. Register numbers are decoded (not complemented). +type x86RawVectorEncoding struct { + modRM, mapNumber, pp, opcode int + r, x, b, upper int + vectorLength, mask int + evex, w, zero, broadcast bool + fixed, addressOverride bool + segment Reg +} + +func decodeX86RawVectorEncoding(code []byte) (p x86RawVectorEncoding, ok bool) { + i := 0 + for i < len(code) { + switch code[i] { + case 0x64: + p.segment = FS + case 0x65: + p.segment = GS + case 0x67: + p.addressOverride = true + default: + goto vector + } + i++ + } + return p, false +vector: + if i >= len(code) { + return p, false + } + var bits byte + switch code[i] { + case 0xc5: + if len(code) < i+3 { + return p, false + } + bits = code[i+1] + p.r, p.mapNumber, p.modRM = int(^bits>>7)&1, 1, i+3 + case 0xc4: + if len(code) < i+4 { + return p, false + } + first := code[i+1] + bits = code[i+2] + p.r, p.x, p.b = int(^first>>7)&1, int(^first>>6)&1, int(^first>>5)&1 + p.mapNumber, p.modRM, p.w = int(first&31), i+4, bits&0x80 != 0 + case 0x62: + if len(code) < i+5 { + return p, false + } + first, last := code[i+1], code[i+3] + bits = code[i+2] + p.evex, p.w, p.fixed = true, bits&0x80 != 0, bits&4 != 0 + p.r, p.x, p.b = (int(^first>>7)&1)|(int(^first>>4)&1)<<1, int(^first>>6)&1, int(^first>>5)&1 + p.mapNumber, p.modRM = int(first&15), i+5 + p.vectorLength, p.mask = int(last>>5)&3, int(last&7) + p.zero, p.broadcast = last&0x80 != 0, last&0x10 != 0 + p.upper = (int(^last>>3) & 1) << 4 + default: + return p, false + } + p.opcode, p.pp = int(code[p.modRM-1]), int(bits&3) + p.upper |= int(^bits>>3) & 15 + if !p.evex { + p.vectorLength = int(bits>>2) & 1 + } + return p, true +} + +// Go's _yvmovsd grammar is shared by VMOVSS and VMOVSD. Opcodes 10 and 11 +// both encode three-register merges; memory forms reserve vvvv and EVEX V'. +func decodedX86ScalarMoveInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || (p.opcode != 0x10 && p.opcode != 0x11) { + return Instr{}, 0, false, nil + } + fp16 := p.evex && p.mapNumber == 5 && p.pp == 2 + if !fp16 && (p.mapNumber != 1 || (p.pp != 2 && p.pp != 3)) { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("scalar move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + // EVEX VMOVSH is LLIG; VMOVSS/SD reserve every nonzero length. + if (!fp16 && p.vectorLength != 0) || p.broadcast { + return fail("scalar moves require 128-bit encoding without broadcast/rounding") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + spec := map[int]struct { + op Op + bytes int + }{2: {"VMOVSS", 4}, 3: {"VMOVSD", 8}}[p.pp] + if fp16 { + spec.op, spec.bytes = "VMOVSH", 2 + } + if p.evex && (!p.fixed || p.w != (spec.bytes == 8)) { + return fail("invalid EVEX fixed or width bit") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + registerForm := code[p.modRM]>>6 == 3 + if !registerForm && p.upper != 0 { + return fail("memory forms reserve vvvv and V'") + } + store := p.opcode == 0x11 + if store && !registerForm && p.zero { + return fail("memory stores cannot zero-mask") + } + regNumber := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (regNumber >= 8 || p.upper >= 8 || p.b != 0 || p.x != 0) { + return fail("extended register in 32-bit mode") + } + var rm Operand + var consumed int + var err error + if p.evex { + rm, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", spec.bytes) + } else { + rm, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + reg := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", regNumber))} + source, destination := rm, reg + if store { + source, destination = reg, rm + } + args := []Operand{source} + if registerForm { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}) + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, destination) + op := spec.op + if p.zero { + op += ".Z" + } + printed := make([]string, len(args)) + for i := range args { + printed[i] = args[i].String() + } + return Instr{ + Op: op, Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + x86Encoded: fp16, + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_scalar_move_literal_test.go b/amd64_raw_scalar_move_literal_test.go new file mode 100644 index 000000000..fe0346c1c --- /dev/null +++ b/amd64_raw_scalar_move_literal_test.go @@ -0,0 +1,163 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawScalarMoveLiteral(double bool, encoding string, mask int, zeroing bool) []byte { + pp := byte(2) + width := 4 + if double { + pp = 3 + width = 8 + } + var code []byte + switch encoding { + case "VEX2": + code = []byte{0xc5, 0xf8 | pp, 0x10, 0x05} + case "VEX3": + code = []byte{0xc4, 0xe1, 0x78 | pp, 0x10, 0x05} + case "EVEX": + p1 := byte(0x7c) | pp + if double { + p1 |= 0x80 + } + p2 := byte(0x08 | mask) + if zeroing { + p2 |= 0x80 + } + code = []byte{0x62, 0xf1, p1, p2, 0x10, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*17+5)) + } + return code +} + +func TestDecodeX86RawScalarMoveRIPDataCompleteLoadForms(t *testing.T) { + type masking struct { + mask int + zeroing bool + } + for _, double := range []bool{false, true} { + for _, encoding := range []string{"VEX2", "VEX3", "EVEX"} { + maskings := []masking{{}} + if encoding == "EVEX" { + maskings = append(maskings, masking{mask: 3}, masking{mask: 7, zeroing: true}) + } + for _, masking := range maskings { + name := fmt.Sprintf("double%t/%s/mask%d/zero%t", double, encoding, masking.mask, masking.zeroing) + t.Run(name, func(t *testing.T) { + code := x86RawScalarMoveLiteral(double, encoding, masking.mask, masking.zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op("VMOVSS") + if double { + want = "VMOVSD" + } + if masking.zeroing { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } +} + +func TestTranslateX86RawScalarMoveRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for _, double := range []bool{false, true} { + for _, encoding := range []string{"VEX2", "VEX3", "EVEX"} { + name := fmt.Sprintf("scalarMoveLiteral%d", len(sigs)) + mask, zeroing := 0, false + if encoding == "EVEX" { + mask, zeroing = 7, true + } + code := x86RawScalarMoveLiteral(double, encoding, mask, zeroing) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + width := 4 + if double { + width = 8 + } + pools = append(pools, code[len(code)-width:]) + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "scalar-move-literal.ll", "scalar-move-literal.o", ir) + }) + } +} + +func TestDecodeX86RawScalarMoveRIPDataRejectsUnsafeSources(t *testing.T) { + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "missing_pool", edit: func(code []byte) []byte { + code[6] = 0x40 + return code + }}, + {name: "instruction_overlap", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x65}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.edit(x86RawScalarMoveLiteral(true, "EVEX", 0, false)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe scalar-move source %x", code) + } + }) + } +} diff --git a/amd64_raw_scalar_move_test.go b/amd64_raw_scalar_move_test.go new file mode 100644 index 000000000..6710f9062 --- /dev/null +++ b/amd64_raw_scalar_move_test.go @@ -0,0 +1,342 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestDecodeX86RawScalarMoveCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT scalarmoves(SB),4,$0-0\n") + var lines []string + for _, op := range []string{"VMOVSS", "VMOVSD"} { + for _, registers := range [][]int{{0, 1, 7}, {8, 9, 15}, {16, 23, 31}} { + for _, mask := range []string{"", "K1, ", "K7, "} { + for _, suffix := range []string{"", ".Z"} { + if mask == "" && suffix != "" { + continue + } + for _, memory := range []string{"(AX)", "-8(R11)(R9*4)", "1024(R11)"} { + lines = append(lines, fmt.Sprintf("%s%s %s, %sX%d", op, suffix, memory, mask, registers[2])) + if suffix == "" { + lines = append(lines, fmt.Sprintf("%s X%d, %s%s", op, registers[0], mask, memory)) + } + } + lines = append(lines, fmt.Sprintf("%s%s X%d, X%d, %sX%d", op, suffix, registers[0], registers[1], mask, registers[2])) + } + } + } + } + source.WriteString(strings.Join(lines, "\n") + "\nRET\n") + code := assembleX87ControlBytes(t, "amd64", source.String()) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := file.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, ins := range decoded.Instrs { + if ins.Op != want[i].Op || !reflect.DeepEqual(ins.Args, want[i].Args) { + t.Fatalf("instruction %d decoded %+v, want %+v", i, ins, want[i]) + } + } + rawFile := x86RawUnprovedReturnFormProbeFile(t, "amd64", "scalarmoves", code) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"} { + ir, err := Translate(rawFile, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{"scalarmoves": {Name: "scalarmoves", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-scalar-moves.ll", "raw-scalar-moves.o", ir) + } +} + +func TestTranslate386VectorScalarMoveCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + var source strings.Builder + source.WriteString("TEXT moves386(SB),4,$0-0\n") + for _, op := range []string{"VMOVSS", "VMOVSD"} { + for _, x := range []string{"X1", "X15", "X31"} { + fmt.Fprintf(&source, "%s (AX),%s\n%s %s,(BX)\n%s %s,X2,X3\n%s (AX),K7,%s\n%s.Z (AX),K7,%s\n%s %s,K7,(BX)\n", op, x, op, x, op, x, op, x, op, x, op, x) + } + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "386", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"moves386": {Name: "moves386", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "scalar-moves386.ll", "scalar-moves386.o", ir) + } + // Raw 32-bit encodings use the architectural X0-X7 range; named Go + // compatibility rows above also expose high EVEX register classes. + rawSource := strings.NewReplacer("X15", "X1", "X31", "X1").Replace(source.String()) + code := assembleX87ControlBytes(t, "386", rawSource) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "386") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, rawSource) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("386 raw instruction count=%d, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("386 instruction %d=%+v, want %+v", i, got, want[i]) + } + } +} + +// Unlike Go's canonical encoder, exercise both equivalent register directions +// and every independent R/R'/B/X/vvvv/V' bit, including EVEX masking. +func encodeRawScalarMove(evex bool, pp, opcode, low, upper, dst, mask int, zero bool) []byte { + reg, rm := dst, low + if opcode == 0x11 { + reg, rm = low, dst + } + modrm := byte(0xc0 | (reg&7)<<3 | rm&7) + if !evex { + return []byte{0xc4, byte((^reg>>3&1)<<7 | 0x40 | (^rm>>3&1)<<5 | 1), byte((^upper&15)<<3 | pp), byte(opcode), modrm} + } + p0 := byte((^reg>>3&1)<<7 | (^rm>>4&1)<<6 | (^rm>>3&1)<<5 | (^reg>>4&1)<<4 | 1) + p1 := byte((^upper&15)<<3 | 4 | pp) + if pp == 3 { + p1 |= 0x80 + } + p2 := byte((^upper>>4&1)<<3 | mask) + if zero { + p2 |= 0x80 + } + return []byte{0x62, p0, p1, p2, byte(opcode), modrm} +} + +func TestDecodedX86RawScalarMoveRegisterEncodings(t *testing.T) { + count := 0 + for _, evex := range []bool{false, true} { + limit := 16 + if evex { + limit = 32 + } + for pp, op := range map[int]Op{2: "VMOVSS", 3: "VMOVSD"} { + for _, opcode := range []int{0x10, 0x11} { + for low := 0; low < limit; low++ { + for upper := 0; upper < limit; upper++ { + for dst := 0; dst < limit; dst++ { + mask, zero := 0, false + if evex { + mask = (low + upper + dst) & 7 + zero = mask != 0 && dst&1 != 0 + } + code := encodeRawScalarMove(evex, pp, opcode, low, upper, dst, mask, zero) + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + wantOp := op + if zero { + wantOp += ".Z" + } + want := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", low))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", upper))}} + if mask != 0 { + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + want = append(want, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", dst))}) + if err != nil || !ok || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, want) { + t.Fatalf("decode %x = %+v, size=%d, ok=%v, err=%v; want %s %+v", code, got, size, ok, err, wantOp, want) + } + count++ + } + } + } + } + } + } + if count != 147456 { + t.Fatalf("covered %d encodings, want 147456", count) + } +} + +func TestDecodedX86RawScalarMoveMemoryAndInvalidForms(t *testing.T) { + for pp, op := range map[int]Op{2: "VMOVSS", 3: "VMOVSD"} { + for _, segment := range []byte{0x64, 0x65} { + code := []byte{segment, 0x62, 0x01, byte(0x7c | pp), 0x08, 0x10, 0x7c, 0x8b, 0xff} + scale := int64(4) + if pp == 3 { + code[3] |= 0x80 + scale = 8 + } + got, size, ok, err := decodedX86ScalarMoveInstruction(code, 64) + seg := FS + if segment == 0x65 { + seg = GS + } + wantMem := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: -scale, Segment: seg} + if err != nil || !ok || size != len(code) || got.Op != op || len(got.Args) != 2 || !reflect.DeepEqual(got.Args[0].Mem, wantMem) || got.Args[1].Reg != "X31" { + t.Fatalf("decode %x = %+v, size=%d, ok=%v, err=%v; want memory %+v", code, got, size, ok, err, wantMem) + } + } + } + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"missing-modrm", []byte{0xc5, 0xfa, 0x10}, 64}, + {"missing-sib", []byte{0xc5, 0xfa, 0x10, 0x04}, 64}, + {"missing-displacement", []byte{0xc5, 0xfa, 0x10, 0x45}, 64}, + {"address-override", []byte{0x67, 0xc5, 0xfa, 0x10, 0xc0}, 64}, + {"vex-length", []byte{0xc5, 0xfe, 0x10, 0xc0}, 64}, + {"vex-memory-vvvv", []byte{0xc5, 0xf2, 0x10, 0x00}, 64}, + {"evex-fixed", []byte{0x62, 0xf1, 0x7a, 0x08, 0x10, 0xc0}, 64}, + {"evex-ss-width", []byte{0x62, 0xf1, 0xfe, 0x08, 0x10, 0xc0}, 64}, + {"evex-sd-width", []byte{0x62, 0xf1, 0x7f, 0x08, 0x10, 0xc0}, 64}, + {"evex-length", []byte{0x62, 0xf1, 0x7e, 0x28, 0x10, 0xc0}, 64}, + {"evex-broadcast", []byte{0x62, 0xf1, 0x7e, 0x18, 0x10, 0xc0}, 64}, + {"zero-k0", []byte{0x62, 0xf1, 0x7e, 0x88, 0x10, 0xc0}, 64}, + {"memory-vprime", []byte{0x62, 0xf1, 0x7e, 0x00, 0x10, 0x00}, 64}, + {"zero-store", []byte{0x62, 0xf1, 0x7e, 0x89, 0x11, 0x00}, 64}, + {"386-high-register", []byte{0xc5, 0x7a, 0x10, 0xc0}, 32}, + {"wrong-mode", []byte{0xc5, 0xfa, 0x10, 0xc0}, 16}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, ok, err := decodedX86ScalarMoveInstruction(tc.code, tc.mode); !ok || err == nil { + t.Fatalf("invalid %x: matched=%v, err=%v", tc.code, ok, err) + } + }) + } + for _, code := range [][]byte{nil, {0x64}, {0xc5}, {0xc4, 0xe1}, {0x62, 0xf1, 0x7e}, {0xc5, 0xf9, 0x10, 0xc0}, {0xc5, 0xfa, 0x5a, 0xc0}} { + if _, _, ok, err := decodedX86ScalarMoveInstruction(code, 64); ok || err != nil { + t.Fatalf("unrelated/truncated prefix %x: matched=%v, err=%v", code, ok, err) + } + } +} + +func TestX86RawScalarMoveRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; three-target object coverage is required separately") + } + var source, declarations, calls strings.Builder + sigs := make(map[string]FuncSig) + index := 0 + for _, pp := range []int{2, 3} { + width := 4 + if pp == 3 { + width = 8 + } + for _, kind := range []string{"merge10", "merge11", "load", "store"} { + for _, encoding := range []string{"vex", "evex", "masked", "zero"} { + if kind == "store" && encoding == "zero" { + continue + } + mask, zero := 0, encoding == "zero" + if encoding == "masked" || zero { + mask = 1 + } + opcode := 0x10 + if kind == "merge11" || kind == "store" { + opcode = 0x11 + } + code := encodeRawScalarMove(encoding != "vex", pp, opcode, 0, 1, 2, mask, zero) + if kind == "load" || kind == "store" { + // Memory is (AX), destination/source X2. vvvv/V' must be reserved. + code = encodeRawScalarMove(encoding != "vex", pp, opcode, 2, 0, 2, mask, zero) + code[len(code)-1] = 0x10 + } + name := fmt.Sprintf("rawscalarmove%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ out+0(FP),DI\nMOVQ in+8(FP),SI\nMOVQ mem+16(FP),AX\nMOVQ mask+24(FP),CX\nKMOVQ CX,K1\nVMOVUPS 32(SI),Y2\nVMOVUPS 32(SI),Z2\nMOVOU (SI),X0\nMOVOU 16(SI),X1\nMOVOU 32(SI),X2\n", name) + for _, b := range code { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + source.WriteString("MOVOU X2,(DI)\nVMOVUPS Y2,16(DI)\nVMOVUPS Z2,48(DI)\nRET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}, + }}} + fmt.Fprintf(&declarations, "extern void %s(void *,const void *,void *,uint64_t);\n", name) + zeroFlag := 0 + if zero { + zeroFlag = 1 + } + fmt.Fprintf(&calls, "if (check(%s,%q,%d,%d,%d)) return %d;\n", name, kind, width, mask, zeroFlag, index) + } + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var runPrefix []string + if crossRosetta { + triple, runPrefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + mainC := ` +#include +#include +#include +` + declarations.String() + ` +static int check(void (*fn)(void *,const void *,void *,uint64_t), const char *kind, int width, int masked, int zero) { + for (int m=0;m<4;m++) { + uint8_t in[96], out[112], want[16], mem[32], old[32]; + for (int i=0;i<96;i++) in[i]=(uint8_t)(17+i*3); + for (int i=0;i<32;i++) mem[i]=old[i]=(uint8_t)(255-i*7); + int active=!masked || (m&1), store=strcmp(kind,"store")==0, load=strcmp(kind,"load")==0; + memcpy(want,in+32,16); + if (!store) { + if (load) memset(want,0,16); else memcpy(want,in+16,16); + if (active) memcpy(want,load ? mem+1 : in,width); + else if (zero) memset(want,0,width); else memcpy(want,in+32,width); + } + fn(out,in,mem+1,(uint64_t)m); + if (memcmp(out,want,16)) { fprintf(stderr,"%s width=%d mask=%d zero=%d register mismatch\n",kind,width,m,zero); return 1; } + for (int v=0;v<2;v++) { + int start=v ? 48 : 16, bytes=v ? 64 : 32; + for (int b=0;b>3&1 | destination>>3&1<<2) + code := []byte{spec.legacyPrefix} + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0x5a, byte(0xc0|destination&7<<3|source&7)) +} + +func encodeX86RawVEXScalarPrecisionConvert(spec rawScalarPrecisionConvertSpec, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | 1 + pp := byte(3) + if spec.inputBytes == 4 { + pp = 2 + } + p1 := byte(^second&15)<<3 | pp + return []byte{0xc4, p0, p1, 0x5a, byte(0xc0 | destination&7<<3 | first&7)} +} + +func encodeX86RawEVEXScalarPrecisionConvert(spec rawScalarPrecisionConvertSpec, control int, mask int, zeroing bool, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + pp := byte(3) + if spec.inputBytes == 4 { + pp = 2 + } + p1 := byte(^second&15)<<3 | 4 | pp + if spec.inputBytes == 8 { + p1 |= 0x80 + } + p2 := byte((^second>>4)&1)<<3 | byte(mask) + if control >= 0 { + p2 |= 0x10 | byte(control)<<5 + } + if zeroing { + p2 |= 0x80 + } + return []byte{0x62, p0, p1, p2, 0x5a, byte(0xc0 | destination&7<<3 | first&7)} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVCVTSD2SS(t *testing.T) { + // vcvtsd2ss %xmm0, %xmm0, %xmm0; + // vmovsd (%rcx), %xmm1 + code := []byte{0xc5, 0xfb, 0x5a, 0xc0, 0xc5, 0xfb, 0x10, 0x09} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VCVTSD2SS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VCVTSD2SS X0, X0, X0 ") || !strings.HasPrefix(decoded[1].Raw, "VMOVSD 0(CX), X1 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawScalarPrecisionConvertCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawScalarPrecisionConvertSpecs { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyScalarPrecisionConvert(spec, source, destination) + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d", spec.legacy, source, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXScalarPrecisionConvert(spec, first, second, destination) + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d, X%d", spec.vector, first, second, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + controls := []int{-1, 0} + if spec.inputBytes == 8 { + controls = []int{-1, 0, 1, 2, 3} + } + for _, control := range controls { + for _, masking := range []struct { + mask int + zeroing bool + }{{}, {mask: 3}, {mask: 7, zeroing: true}} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXScalarPrecisionConvert(spec, control, masking.mask, masking.zeroing, first, second, destination) + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(code, 64) + suffix := "" + if control >= 0 { + if spec.inputBytes == 4 { + suffix = ".SAE" + } else { + suffix = "." + [...]string{"RN_SAE", "RD_SAE", "RU_SAE", "RZ_SAE"}[control] + } + } + if masking.zeroing { + suffix += ".Z" + } + args := []string{fmt.Sprintf("X%d", first), fmt.Sprintf("X%d", second)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("X%d", destination)) + want := string(spec.vector) + suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 696832 { + t.Fatalf("covered %d scalar precision-conversion register encodings, want 696832", count) + } +} + +func TestDecodedX86RawScalarPrecisionConvertMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0xf3, 0x47, 0x0f, 0x5a, 0x7c, 0x8b, 0x01} + got, length, ok, err := decodedX86ScalarPrecisionConvertInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "CVTSS2SD" || got.Args[1].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x01, 0x73, 0x5a, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86ScalarPrecisionConvertInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VCVTSD2SS" || got.Args[1].String() != "X1" || got.Args[2].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0xf1, 0x56, 0x83, 0x5a, 0x50, 0x02} + got, length, ok, err = decodedX86ScalarPrecisionConvertInstruction(evexMemory, 64) + wantMemory = MemRef{Base: "AX", Off: 8} + if err != nil || !ok || length != len(evexMemory) || got.Op != "VCVTSS2SD.Z" || got.Args[1].String() != "X21" || got.Args[2].String() != "K3" || got.Args[3].String() != "X2" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[0], 7, 6, 5) + validEVEXSS := encodeX86RawEVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[0], -1, 0, false, 7, 6, 5) + validEVEXSD := encodeX86RawEVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[1], -1, 0, false, 7, 6, 5) + for name, invalid := range map[string][]byte{ + "vex W": {validVEX[0], validVEX[1], validVEX[2] | 0x80, validVEX[3], validVEX[4]}, + "vex L": {validVEX[0], validVEX[1], validVEX[2] | 0x04, validVEX[3], validVEX[4]}, + "evex fixed bit": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2] &^ 0x04, validEVEXSS[3], validEVEXSS[4], validEVEXSS[5]}, + "evex SS wrong W": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2] | 0x80, validEVEXSS[3], validEVEXSS[4], validEVEXSS[5]}, + "evex SD wrong W": {validEVEXSD[0], validEVEXSD[1], validEVEXSD[2] &^ 0x80, validEVEXSD[3], validEVEXSD[4], validEVEXSD[5]}, + "evex length without SAE": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2], validEVEXSS[3] | 0x20, validEVEXSS[4], validEVEXSS[5]}, + "zero without mask": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2], validEVEXSS[3] | 0x80, validEVEXSS[4], validEVEXSS[5]}, + "SS explicit rounding": {validEVEXSS[0], validEVEXSS[1], validEVEXSS[2], validEVEXSS[3] | 0x30, validEVEXSS[4], validEVEXSS[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86ScalarPrecisionConvertInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + roundingMemory := []byte{0x62, 0xf1, 0xff, 0x10, 0x5a, 0x00} + if instruction, _, matched, decodeErr := decodedX86ScalarPrecisionConvertInstruction(roundingMemory, 64); !matched || decodeErr == nil { + t.Fatalf("rounding memory encoding %x decoded as %+v, ok=%v, err=%v", roundingMemory, instruction, matched, decodeErr) + } + masked386 := encodeX86RawEVEXScalarPrecisionConvert(rawScalarPrecisionConvertSpecs[0], -1, 1, false, 0, 1, 2) + if instruction, _, matched, decodeErr := decodedX86ScalarPrecisionConvertInstruction(masked386, 32); !matched || decodeErr == nil { + t.Fatalf("386 masked encoding %x decoded as %+v, ok=%v, err=%v", masked386, instruction, matched, decodeErr) + } +} + +func TestTranslateX86RawScalarPrecisionConvertTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_scalar_precision_convert(SB),NOSPLIT,$0-0\n") + for _, spec := range rawScalarPrecisionConvertSpecs { + var encodings [][]byte + if target.goarch == "amd64" { + control := 0 + if spec.inputBytes == 8 { + control = 3 + } + encodings = append(encodings, + encodeX86RawLegacyScalarPrecisionConvert(spec, 15, 14), + encodeX86RawVEXScalarPrecisionConvert(spec, 15, 14, 13), + encodeX86RawEVEXScalarPrecisionConvert(spec, -1, 3, true, 31, 30, 29), + encodeX86RawEVEXScalarPrecisionConvert(spec, control, 7, true, 31, 30, 29), + ) + } else { + control := 0 + if spec.inputBytes == 8 { + control = 2 + } + encodings = append(encodings, + encodeX86RawLegacyScalarPrecisionConvert(spec, 7, 6), + encodeX86RawVEXScalarPrecisionConvert(spec, 7, 6, 5), + encodeX86RawEVEXScalarPrecisionConvert(spec, -1, 0, false, 20, 21, 22), + encodeX86RawEVEXScalarPrecisionConvert(spec, control, 0, false, 20, 21, 22), + ) + } + for _, encoding := range encodings { + for _, value := range encoding { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "raw_scalar_precision_convert": {Name: "raw_scalar_precision_convert", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-scalar-precision-convert-"+target.name+".ll", "raw-scalar-precision-convert-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_scaled_round.go b/amd64_raw_scaled_round.go new file mode 100644 index 000000000..fb0fb3029 --- /dev/null +++ b/amd64_raw_scaled_round.go @@ -0,0 +1,110 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86RawScaledRoundForm struct { + op Op + laneBytes int + scalar bool +} + +// Go 1.27's eight EVEX RNDSCALE/REDUCE opcode rows. Both operations share +// the same packed/scalar immediate and modifier grammar. +var x86RawScaledRoundForms = map[[2]int]x86RawScaledRoundForm{ + {0x08, 0}: {op: "VRNDSCALEPS", laneBytes: 4}, + {0x09, 1}: {op: "VRNDSCALEPD", laneBytes: 8}, + {0x0a, 0}: {op: "VRNDSCALESS", laneBytes: 4, scalar: true}, + {0x0b, 1}: {op: "VRNDSCALESD", laneBytes: 8, scalar: true}, + {0x56, 0}: {op: "VREDUCEPS", laneBytes: 4}, + {0x56, 1}: {op: "VREDUCEPD", laneBytes: 8}, + {0x57, 0}: {op: "VREDUCESS", laneBytes: 4, scalar: true}, + {0x57, 1}: {op: "VREDUCESD", laneBytes: 8, scalar: true}, +} + +func decodedX86RawScaledRoundInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.mapNumber != 3 || p.pp != 1 { + return Instr{}, 0, false, nil + } + w := 0 + if p.w { + w = 1 + } + form, recognized := x86RawScaledRoundForms[[2]int{p.opcode, w}] + if !recognized { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("EVEX scaled round: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if !p.fixed || p.addressOverride || p.vectorLength > 2 || + form.scalar && (mode == 32 || p.vectorLength != 0) || + !form.scalar && p.upper != 0 { + return fail("invalid EVEX fixed, address, vector length, or vvvv") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if p.zero && p.mask == 0 { + return fail("zeroing requires a nonzero mask") + } + if mode == 32 && p.mask != 0 { + return fail("386 mask form exceeds the Go assembler frontend's operand limit") + } + modRM := code[p.modRM] + registerSource := modRM>>6 == 3 + sae := p.broadcast && registerSource + broadcast := p.broadcast && !registerSource + if form.scalar && broadcast || !form.scalar && sae && p.vectorLength != 2 { + return fail("SAE or broadcast is unavailable for this operand width") + } + destinationNumber := int(modRM>>3&7) + p.r*8 + if mode == 32 && p.vectorLength == 2 && + (destinationNumber >= 8 || registerSource && int(modRM&7)+p.b*8+p.x*16 >= 8) { + return fail("386 Z-register exceeds the Go assembler frontend's register class") + } + vector := [...]string{"X", "Y", "Z"}[p.vectorLength] + disp8Scale := 16 << p.vectorLength + if form.scalar || broadcast { + disp8Scale = form.laneBytes + } + source, consumed, err := decodedX86EVEXRMOperand( + code[p.modRM:], mode, p.b, p.x, p.segment, vector, disp8Scale, + ) + if err != nil { + return Instr{}, 0, true, err + } + immIndex := p.modRM + consumed + if len(code) <= immIndex { + return fail("missing imm8") + } + op := form.op + if broadcast { + op += ".BCST" + } + if sae { + op += ".SAE" + } + if p.zero { + op += ".Z" + } + args := []Operand{{Kind: OpImm, Imm: int64(code[immIndex])}, source} + if form.scalar { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}) + } + if p.mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", p.mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", vector, destinationNumber))}) + printed := make([]string, len(args)) + for index := range args { + printed[index] = args[index].String() + } + return Instr{Op: op, Args: args, Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", "))}, immIndex + 1, true, nil +} diff --git a/amd64_raw_scaled_round_test.go b/amd64_raw_scaled_round_test.go new file mode 100644 index 000000000..819e1b5a5 --- /dev/null +++ b/amd64_raw_scaled_round_test.go @@ -0,0 +1,216 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawScaledRound(op Op, vectorBits, mask int, zeroing, evexB, memory bool) []byte { + var opcode byte + var w, scalar bool + for encoding, form := range x86RawScaledRoundForms { + if form.op == op { + opcode = byte(encoding[0]) + w = encoding[1] != 0 + scalar = form.scalar + break + } + } + p1 := byte(0x7d) + if scalar { + p1 = 0x6d // EVEX.vvvv = X2 passthrough. + } + if w { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if evexB { + p2 |= 0x10 + } + modRM := byte(0xd1) // first register 1, destination register 2. + if scalar { + modRM = 0xd9 // first register 1, destination register 3. + } + if memory { + modRM = 0x50 // disp8(AX), destination register 2. + if scalar { + modRM = 0x58 // disp8(AX), destination register 3. + } + } + code := []byte{0x62, 0xf3, p1, p2, opcode, modRM} + if memory { + code = append(code, 1) + } + return append(code, 9) +} + +func TestDecodeX86RawScaledRoundSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VRNDSCALEPS $9, 0(SI)(R8*4), Z0. + code := []byte{0x62, 0xb3, 0x7d, 0x48, 0x08, 0x04, 0x86, 0x09, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "floorFloat32 AVX512 VRNDSCALEPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VRNDSCALEPS" || + decoded[0].Args[0].Imm != 9 || decoded[0].Args[2].Reg != "Z0" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawScaledRoundCompleteGoForms(t *testing.T) { + if len(x86RawScaledRoundForms) != 8 { + t.Fatalf("modeled %d scaled-round rows, want eight", len(x86RawScaledRoundForms)) + } + for _, form := range x86RawScaledRoundForms { + maxLength := 2 + if form.scalar { + maxLength = 0 + } + for vectorBits := 0; vectorBits <= maxLength; vectorBits++ { + for _, variant := range []struct { + name string + mask int + zeroing bool + evexB bool + memory bool + suffix string + }{ + {name: "register"}, + {name: "masked zero register", mask: 7, zeroing: true, suffix: ".Z"}, + {name: "memory", memory: true}, + {name: "broadcast memory", evexB: true, memory: true, suffix: ".BCST"}, + {name: "SAE register", evexB: true, suffix: ".SAE"}, + } { + if form.scalar && variant.suffix == ".BCST" || + !form.scalar && variant.suffix == ".SAE" && vectorBits != 2 { + continue + } + name := fmt.Sprintf("%s/length%d/%s", form.op, vectorBits, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawScaledRound(form.op, vectorBits, variant.mask, variant.zeroing, variant.evexB, variant.memory) + got, length, ok, err := decodedX86RawScaledRoundInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != form.op+Op(variant.suffix) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantDestination := fmt.Sprintf("%s2", [...]string{"X", "Y", "Z"}[vectorBits]) + if form.scalar { + wantDestination = "X3" + } + if got.Args[0].Imm != 9 || got.Args[len(got.Args)-1].String() != wantDestination { + t.Fatalf("decode %x = %+v, want $9 ... %s", code, got, wantDestination) + } + }) + } + } + } +} + +func TestDecodeX86RawScaledRoundRIPLiteral(t *testing.T) { + for _, form := range x86RawScaledRoundForms { + for _, broadcast := range []bool{false, true} { + if broadcast && form.scalar { + continue + } + vectorBits := 2 + if form.scalar { + vectorBits = 0 + } + code := encodeX86RawScaledRound(form.op, vectorBits, 1, false, broadcast, false) + code = code[:6] + code[5] = 0x15 // RIP-relative first source, destination 2. + if form.scalar { + code[5] = 0x1d // destination 3. + } + code = append(code, 1, 0, 0, 0, 9, 0xc3) + literalWidth := 16 << vectorBits + if form.scalar || broadcast { + literalWidth = form.laneBytes + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/broadcast%t", form.op, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || len(got[0].x86RIPLiteralData) != literalWidth { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } +} + +func TestDecodeX86RawScaledRoundRejectsInvalidForms(t *testing.T) { + base := encodeX86RawScaledRound("VRNDSCALEPS", 2, 0, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[3] |= 0x80 }, // zeroing needs a mask + func(code []byte) { code[3] |= 0x20 }, // reserved vector length + func(code []byte) { code[2] &^= 0x08 }, // packed reserved vvvv + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86RawScaledRoundInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + scalarBroadcast := encodeX86RawScaledRound("VRNDSCALESS", 0, 0, false, true, true) + if _, _, ok, err := decodedX86RawScaledRoundInstruction(scalarBroadcast, 64); !ok || err == nil { + t.Fatalf("accepted scalar broadcast %x: ok=%v err=%v", scalarBroadcast, ok, err) + } +} + +func TestTranslateX86RawScaledRoundLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawScaledRound(SB),$0-0\n") + for _, op := range []Op{"VRNDSCALEPS", "VRNDSCALEPD", "VREDUCEPS", "VREDUCEPD"} { + code := encodeX86RawScaledRound(op, 0, 0, false, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + if target.goarch == "amd64" { + for _, op := range []Op{"VRNDSCALESS", "VRNDSCALESD", "VREDUCESS", "VREDUCESD"} { + code := encodeX86RawScaledRound(op, 0, 0, false, false, false) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawScaledRound": {Name: "rawScaledRound", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-scaled-round.ll", "raw-scaled-round.o", ir) + }) + } +} diff --git a/amd64_raw_scatter_test.go b/amd64_raw_scatter_test.go new file mode 100644 index 000000000..3cd4c831f --- /dev/null +++ b/amd64_raw_scatter_test.go @@ -0,0 +1,150 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestDecodeX86RawScatterSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VSCATTERQPS Y2, K1, 0(DI)(Z1*4). + code := []byte{0x62, 0xf2, 0x7d, 0x49, 0xa3, 0x14, 0x8f, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "convolveFloat32 AVX512 VSCATTERQPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VSCATTERQPS" || + decoded[0].Args[0].Reg != "Y2" || decoded[0].Args[1].Reg != "K1" || + decoded[0].Args[2].Mem.Index != "Z1" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func x86RawScatterGoForms(goarch string) string { + var source strings.Builder + source.WriteString("TEXT rawScatterForms(SB),4,$0-0\n") + widths := [...]string{"X", "Y", "Z"} + for _, family := range x86ScatterOps { + for _, op := range family { + spec := amd64ScatterSpecs[string(op)] + for width := 0; width < 3; width++ { + dataWidth := widths[width] + indexWidth := dataWidth + switch spec.table { + case amd64GatherDPD: + if width > 0 { + indexWidth = widths[width-1] + } + case amd64GatherQPS: + if width > 0 { + dataWidth = widths[width-1] + } + } + fmt.Fprintf(&source, "\t%s %s1, K1, 8(BX)(%s2*4)\n", op, dataWidth, indexWidth) + if goarch == "amd64" { + fmt.Fprintf(&source, "\t%s %s21, K7, -32(R8)(%s18*2)\n", op, dataWidth, indexWidth) + } + } + } + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestDecodeX86RawScatterCompleteGoForms(t *testing.T) { + expected := map[Op]bool{ + "VPSCATTERDD": true, "VPSCATTERDQ": true, + "VPSCATTERQD": true, "VPSCATTERQQ": true, + "VSCATTERDPS": true, "VSCATTERDPD": true, + "VSCATTERQPS": true, "VSCATTERQPD": true, + } + for _, family := range x86ScatterOps { + for _, op := range family { + if !expected[op] { + t.Fatalf("unexpected or repeated scatter opcode %s", op) + } + delete(expected, op) + } + } + if len(expected) != 0 { + t.Fatalf("missing Go scatter opcodes: %v", expected) + } + for _, goarch := range []string{"amd64", "386"} { + t.Run(goarch, func(t *testing.T) { + source := x86RawScatterGoForms(goarch) + code := assembleX87ControlBytes(t, goarch, source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), goarch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } + }) + } +} + +func TestDecodeX86RawScatterRejectsMalformedForms(t *testing.T) { + valid := []byte{0x62, 0xf2, 0x7d, 0x49, 0xa3, 0x14, 0x8f} + for _, variant := range []struct { + name string + mutate func([]byte) []byte + }{ + {"missing mask", func(code []byte) []byte { code[3] &^= 7; return code }}, + {"zeroing", func(code []byte) []byte { code[3] |= 0x80; return code }}, + {"broadcast", func(code []byte) []byte { code[3] |= 0x10; return code }}, + {"reserved length", func(code []byte) []byte { code[3] |= 0x60; return code }}, + {"register destination", func(code []byte) []byte { code[5] |= 0xc0; return code }}, + {"missing SIB", func(code []byte) []byte { return code[:6] }}, + {"reserved vvvv", func(code []byte) []byte { code[2] &^= 0x08; return code }}, + } { + t.Run(variant.name, func(t *testing.T) { + code := variant.mutate(append([]byte(nil), valid...)) + if _, _, matched, err := decodedX86RawScatterInstruction(code, 64); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", code, matched, err) + } + }) + } +} + +func TestTranslateX86RawScatterLLVM22AllTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + source := x86RawScatterGoForms(target.goarch) + code := assembleX87ControlBytes(t, target.goarch, source) + file := x86RawUnprovedReturnFormProbeFile(t, target.goarch, "rawScatterForms", code) + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawScatterForms": {Name: "rawScatterForms", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-scatter.ll", "raw-scatter.o", ir) + }) + } +} diff --git a/amd64_raw_segment_absolute_test.go b/amd64_raw_segment_absolute_test.go new file mode 100644 index 000000000..3a824c697 --- /dev/null +++ b/amd64_raw_segment_absolute_test.go @@ -0,0 +1,59 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawSegmentAbsoluteMemory(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, tc := range []struct { + name string + code []byte + want string + }{ + {name: "gs-absolute", code: []byte{0x65, 0x48, 0x8b, 0x04, 0x25, 0x30, 0, 0, 0}, want: "48(GS)"}, + {name: "fs-absolute", code: []byte{0x64, 0x48, 0x8b, 0x04, 0x25, 0x30, 0, 0, 0}, want: "48(FS)"}, + {name: "gs-base", code: []byte{0x65, 0x48, 0x8b, 0x40, 0x30}, want: "48(AX)(GS)"}, + {name: "fs-base", code: []byte{0x64, 0x48, 0x8b, 0x40, 0x30}, want: "48(AX)(FS)"}, + } { + t.Run(tc.name, func(t *testing.T) { + inst, err := x86asm.Decode(tc.code, 64) + if err != nil { + t.Fatal(err) + } + syntax, err := decodedX86GoSyntax(inst, tc.code) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(syntax, tc.want) { + t.Fatalf("raw %x decoded as %q, want memory %s", tc.code, syntax, tc.want) + } + + var source strings.Builder + source.WriteString("TEXT rawSegmentMemory(SB),$0-0\n") + for _, value := range tc.code { + fmt.Fprintf(&source, "BYTE $%#02x\n", value) + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawSegmentMemory": {Name: "rawSegmentMemory", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "raw-segment-memory.ll", "raw-segment-memory.o", ir) + }) + } +} diff --git a/amd64_raw_segment_base_test.go b/amd64_raw_segment_base_test.go new file mode 100644 index 000000000..5a36ecaaa --- /dev/null +++ b/amd64_raw_segment_base_test.go @@ -0,0 +1,187 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +// Intel's F3 0F AE /0..3 register encodings correspond to all eight rows in +// Go's asm6.go: RDFSBASE, RDGSBASE, WRFSBASE and WRGSBASE, each with L/Q width. +var rawSegmentBaseTestOps = [...]string{"RDFSBASE", "RDGSBASE", "WRFSBASE", "WRGSBASE"} + +func rawSegmentBaseTestEncoding(operation, bits, register int) []byte { + code := []byte{0xf3} + rex := byte(0x40) + if bits == 64 { + rex |= 8 + } + if register >= 8 { + rex |= 1 + } + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, 0xae, byte(0xc0|operation<<3|register&7)) +} + +func TestX86RawSegmentBaseCompleteRegisterFamily(t *testing.T) { + if len(amd64SegmentBaseSpecs) != 2*len(rawSegmentBaseTestOps) { + t.Fatal("raw encoding tests must cover the complete named segment-base grammar") + } + registers := [...]Reg{AX, CX, DX, BX, SP, BP, SI, DI, "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15"} + for operation, name := range rawSegmentBaseTestOps { + for _, width := range []struct { + bits int + suffix string + }{{32, "L"}, {64, "Q"}} { + for register, wantRegister := range registers { + wantOp := Op(name + width.suffix) + t.Run(string(wantOp)+"/"+string(wantRegister), func(t *testing.T) { + code := rawSegmentBaseTestEncoding(operation, width.bits, register) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segment-base bytes", nil) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != wantOp || len(decoded[0].Args) != 1 || + decoded[0].Args[0].Kind != OpReg || decoded[0].Args[0].Reg != wantRegister { + t.Fatalf("%x decoded as %+v, want %s %s", code, decoded, wantOp, wantRegister) + } + if _, ok := amd64SegmentBaseSpecs[wantOp]; !ok { + t.Fatalf("raw opcode %s has no typed named grammar", wantOp) + } + }) + } + } + } +} + +func TestX86RawSegmentBaseIgnoredPrefixesPreserveRegisterWidth(t *testing.T) { + for operation, name := range rawSegmentBaseTestOps { + for _, bits := range []int{32, 64} { + for _, prefix := range []byte{0x66, 0x67, 0x64, 0x65} { + for _, afterREP := range []bool{false, true} { + code := rawSegmentBaseTestEncoding(operation, bits, 11) + if afterREP { + code = append([]byte{0xf3, prefix}, code[1:]...) + } else { + code = append([]byte{prefix}, code...) + } + inst, err := x86asm.Decode(code, 64) + if err != nil || inst.Len != len(code) { + t.Fatalf("decode %x: %+v, %v", code, inst, err) + } + got, err := decodedX86GoSyntax(inst, code) + width := "L" + if bits == 64 { + width = "Q" + } + want := name + width + " R11" + if err != nil || got != want { + t.Errorf("%x (DataSize=%d): got %q, %v; want %q", code, inst.DataSize, got, err, want) + } + } + } + } + } +} + +func TestX86RawSegmentBaseRejectsInvalidDecodedForms(t *testing.T) { + for _, op := range []x86asm.Op{x86asm.RDFSBASE, x86asm.RDGSBASE, x86asm.WRFSBASE, x86asm.WRGSBASE} { + for _, args := range []x86asm.Args{ + {}, {x86asm.Imm(1)}, {x86asm.Mem{Base: x86asm.RAX}}, + {x86asm.AL}, {x86asm.AX}, {x86asm.X0}, + {x86asm.RAX, x86asm.RBX}, {x86asm.RAX, nil, nil, x86asm.RBX}, + } { + inst := x86asm.Inst{Op: op, Mode: 64, DataSize: 64, Args: args} + if got, err := decodedX86GoSyntax(inst, nil); err == nil { + t.Errorf("accepted invalid %s operands %v as %q", op, args, got) + } + } + } + for operation := range rawSegmentBaseTestOps { + code := rawSegmentBaseTestEncoding(operation, 64, 3) + for _, invalid := range []struct { + name string + code []byte + mode int + }{ + {"lock", append([]byte{0xf0}, code...), 64}, + {"memory", []byte{0xf3, 0x48, 0x0f, 0xae, byte(operation << 3)}, 64}, + {"truncated", code[:len(code)-1], 64}, + {"32-bit-mode", rawSegmentBaseTestEncoding(operation, 32, 3), 32}, + } { + t.Run(rawSegmentBaseTestOps[operation]+"/"+invalid.name, func(t *testing.T) { + if got, err := decodeX86RawDirectiveGroup(invalid.code, invalid.mode, 0, "invalid segment-base bytes", nil); err == nil { + t.Fatalf("accepted invalid %x as %+v", invalid.code, got) + } + }) + } + } +} + +func TestTranslateX86RawSegmentBaseMatchesNamedFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + var raw, named strings.Builder + for _, source := range []*strings.Builder{&raw, &named} { + source.WriteString("TEXT segmentbase(SB),$0-0\nMOVQ $-1, BX\n") + } + // Keep the whole reported gVisor sequence, including SWAPGS before + // RDGSBASEQ, as well as every sibling operation and operand width. + for _, value := range []byte{0x0f, 0x01, 0xf8, 0xf3, 0x48, 0x0f, 0xae, 0xcb} { + fmt.Fprintf(&raw, "BYTE $%#x\n", value) + } + named.WriteString("SWAPGS\nRDGSBASEQ BX\n") + for operation, name := range rawSegmentBaseTestOps { + for _, bits := range []int{32, 64} { + for _, value := range rawSegmentBaseTestEncoding(operation, bits, 3) { + fmt.Fprintf(&raw, "BYTE $%#x\n", value) + } + width := "L" + if bits == 64 { + width = "Q" + } + fmt.Fprintf(&named, "%s%s BX\n", name, width) + } + } + var irs []string + for _, source := range []*strings.Builder{&raw, &named} { + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"segmentbase": {Name: "segmentbase", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + // LLVM assigns a different temporary filename on each parse. + // Compare every semantic line, excluding only that metadata. + lines := strings.Split(ir, "\n") + for index, line := range lines { + if strings.HasPrefix(line, "; ModuleID = ") || strings.HasPrefix(line, "source_filename = ") { + lines[index] = "" + } + } + irs = append(irs, strings.Join(lines, "\n")) + } + if irs[0] != irs[1] { + t.Fatalf("raw and named segment-base semantics differ:\nraw:\n%s\nnamed:\n%s", irs[0], irs[1]) + } + compileLLVMToObject(t, llc, triple, "raw-segment-base.ll", "raw-segment-base.o", irs[0]) + }) + } +} diff --git a/amd64_raw_segment_move_test.go b/amd64_raw_segment_move_test.go new file mode 100644 index 000000000..63309a2bc --- /dev/null +++ b/amd64_raw_segment_move_test.go @@ -0,0 +1,116 @@ +package plan9asm + +import ( + "fmt" + "testing" +) + +func TestDecodedX86RawSegmentMoveCompleteRegisterFamily(t *testing.T) { + segments := []string{"ES", "CS", "SS", "DS", "FS", "GS"} + count := 0 + for _, mode := range []int{32, 64} { + registers := 8 + if mode == 64 { + registers = 16 + } + for segmentNumber, segment := range segments { + for general := 0; general < registers; general++ { + rex := byte(0) + if mode == 64 && general >= 8 { + rex = 0x41 + } + for _, direction := range []struct { + opcode byte + want string + }{ + {opcode: 0x8e, want: fmt.Sprintf("MOVW %s, %s", decodedX86RegisterNameForTest(general), segment)}, + {opcode: 0x8c, want: fmt.Sprintf("MOVW %s, %s", segment, decodedX86RegisterNameForTest(general))}, + } { + code := make([]byte, 0, 3) + if rex != 0 { + code = append(code, rex) + } + code = append(code, direction.opcode, byte(0xc0|segmentNumber<<3|general&7)) + got, length, ok, err := decodedX86SegmentMoveInstruction(code, mode) + if err != nil || !ok || length != len(code) || got.Raw != direction.want { + t.Fatalf("decode mode=%d %x = %+v, length=%d, ok=%v, err=%v; want %q", mode, code, got, length, ok, err, direction.want) + } + count++ + } + } + } + } + if count != 288 { + t.Fatalf("covered %d raw segment-register moves, want 288", count) + } +} + +func decodedX86RegisterNameForTest(number int) string { + reg, ok := decodedX86GeneralRegister(number) + if !ok { + panic(number) + } + return string(reg) +} + +func TestDecodedX86RawSegmentMoveMemoryAndInvalidForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + code []byte + want string + }{ + { + name: "load FS from extended SIB with FS override", + mode: 64, + code: []byte{0x64, 0x43, 0x8e, 0x64, 0x8b, 0x20}, + want: "MOVW 32(R11)(R9*4)(FS), FS", + }, + { + name: "store GS to extended SIB with GS override", + mode: 64, + code: []byte{0x65, 0x43, 0x8c, 0x6c, 0x8b, 0x20}, + want: "MOVW GS, 32(R11)(R9*4)(GS)", + }, + { + name: "386 absolute memory", + mode: 32, + code: []byte{0x66, 0x8e, 0x1d, 0x78, 0x56, 0x34, 0x12}, + want: "MOVW 305419896(), DS", + }, + } { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86SegmentMoveInstruction(test.code, test.mode) + if err != nil || !ok || length != len(test.code) || got.Raw != test.want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", test.code, got, length, ok, err, test.want) + } + }) + } + + for _, test := range []struct { + name string + encoding []byte + wantOK bool + }{ + {name: "reserved segment six", encoding: []byte{0x8e, 0xf0}, wantOK: true}, + {name: "reserved segment seven", encoding: []byte{0x8c, 0xf8}, wantOK: true}, + {name: "REX.R reserved segment", encoding: []byte{0x44, 0x8e, 0xc0}, wantOK: true}, + {name: "RIP relative", encoding: []byte{0x8e, 0x1d, 0, 0, 0, 0}, wantOK: true}, + {name: "address override", encoding: []byte{0x67, 0x8e, 0xd8}, wantOK: true}, + {name: "truncated", encoding: []byte{0x8e}, wantOK: false}, + {name: "unrelated opcode", encoding: []byte{0x8f, 0xd8}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86SegmentMoveInstruction(test.encoding, 64) + if ok != test.wantOK { + t.Fatalf("encoding %x returned ok=%v err=%v, want ok=%v", test.encoding, ok, err, test.wantOK) + } + if test.wantOK && err == nil { + t.Fatalf("invalid encoding %x was accepted", test.encoding) + } + if !test.wantOK && err != nil { + t.Fatalf("unrelated encoding %x returned err=%v", test.encoding, err) + } + }) + } +} diff --git a/amd64_raw_sha_test.go b/amd64_raw_sha_test.go new file mode 100644 index 000000000..7875b2b32 --- /dev/null +++ b/amd64_raw_sha_test.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86SHACompleteGo127Family(t *testing.T) { + // Go 1.27 exposes every Intel SHA extension opcode. Cover register and + // memory sources, REX-extended X registers, the SHA1 immediate, and the + // implicit X0 operand used by SHA256RNDS2. + tests := []struct { + name string + code []byte + want string + }{ + {name: "sha1 next e", code: []byte{0x0f, 0x38, 0xc8, 0xca}, want: "SHA1NEXTE X2, X1"}, + {name: "sha1 message 1 high destination", code: []byte{0x44, 0x0f, 0x38, 0xc9, 0xf4}, want: "SHA1MSG1 X4, X14"}, + {name: "sha1 message 2 memory", code: []byte{0x41, 0x0f, 0x38, 0xca, 0x54, 0x8c, 0x20}, want: "SHA1MSG2 32(R12)(CX*4), X2"}, + {name: "sha256 message 1", code: []byte{0x0f, 0x38, 0xcc, 0xe5}, want: "SHA256MSG1 X5, X4"}, + {name: "sha256 message 2", code: []byte{0x0f, 0x38, 0xcd, 0xf5}, want: "SHA256MSG2 X5, X6"}, + {name: "sha1 rounds 4", code: []byte{0x0f, 0x3a, 0xcc, 0xd1, 0x03}, want: "SHA1RNDS4 $3, X1, X2"}, + {name: "sha256 rounds 2 implicit x0", code: []byte{0x0f, 0x38, 0xcb, 0xda}, want: "SHA256RNDS2 X0, X2, X3"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86SHAInstruction(test.code, 64) + if !ok { + t.Fatal("SHA encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioSHA256RNDS2(t *testing.T) { + code := []byte{0x0f, 0x38, 0xcb, 0xda} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio SHA256RNDS2", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "SHA256RNDS2" || !strings.HasPrefix(decoded[0].Raw, "SHA256RNDS2 X0, X2, X3 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_shiftrotate_test.go b/amd64_raw_shiftrotate_test.go new file mode 100644 index 000000000..28987c5e0 --- /dev/null +++ b/amd64_raw_shiftrotate_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawShiftRotateCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + var lines []string + widths := []string{"B", "W", "L"} + mode := 32 + if arch == "amd64" { + mode = 64 + widths = append(widths, "Q") + } + for _, stem := range []string{"ROL", "ROR", "RCL", "RCR", "SHL", "SHR", "SAR"} { + for _, width := range widths { + destinations := []string{"AX", "-7(BX)(CX*4)"} + if width == "B" { + destinations = []string{"AL", "AH", "-7(BX)(CX*4)"} + } + if arch == "amd64" { + destinations = append(destinations, "R11") + } + for _, dst := range destinations { + for _, count := range []string{"$1", "$255", "CL"} { + lines = append(lines, fmt.Sprintf("%s%s %s, %s", stem, width, count, dst)) + } + } + } + } + for _, stem := range []string{"SHL", "SHR"} { + for _, width := range widths[1:] { + for _, dst := range []string{"BX", "-7(BX)(CX*4)"} { + for _, count := range []string{"$-128", "$127", "CL"} { + lines = append(lines, fmt.Sprintf("%s%s %s, AX, %s", stem, width, count, dst)) + } + } + } + } + named := "TEXT shifts(SB),4,$0-0\n" + strings.Join(lines, "\n") + "\nRET\n" + code := assembleX87ControlBytes(t, arch, named) + offset := 0 + var raw strings.Builder + raw.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, line := range lines { + inst, err := x86asm.Decode(code[offset:], mode) + if err != nil { + t.Fatalf("Go encoded %s: %v", line, err) + } + encoded := code[offset : offset+inst.Len] + decoded, err := decodeX86RawDirectives(rawX86Function(encoded), arch) + if err != nil { + t.Fatal(err) + } + want, err := Parse(ArchAMD64, "TEXT f(SB),4,$0-0\n"+line+"\n") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != want.Funcs[0].Instrs[1].Op || !reflect.DeepEqual(decoded.Instrs[0].Args, want.Funcs[0].Instrs[1].Args) { + t.Fatalf("decoded %x as %+v, want %s", encoded, decoded.Instrs, line) + } + for _, b := range encoded { + fmt.Fprintf(&raw, "BYTE $%#02x\n", b) + } + offset += inst.Len + } + raw.WriteString("RET\n") + file, err := Parse(ArchAMD64, raw.String()) + if err != nil { + t.Fatal(err) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-shifts.ll", "raw-shifts.o", ir) + } + }) + } +} diff --git a/amd64_raw_static_return_runtime_test.go b/amd64_raw_static_return_runtime_test.go new file mode 100644 index 000000000..cbf301171 --- /dev/null +++ b/amd64_raw_static_return_runtime_test.go @@ -0,0 +1,196 @@ +package plan9asm + +import ( + "encoding/binary" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// The Go oracle uses an ordinary declared result slot. It does not pretend +// that Go ABI0 returns a scalar in AX. The translated raw-RIP counterpart has +// the same fixed-width read and data, transported through its typed C result. +func runX86StaticReadNumericOracle(t *testing.T, arch, triple string, compiler, runner []string) { + t.Helper() + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + values := []uint64{0, 1, 0x80000001, 0xffffffff, 0x89abcdef01234567, 0xfedcba9876543210} + var raw, native, declarations, goMain, cMain strings.Builder + goMain.WriteString("package main\nfunc main(){\n") + cMain.WriteString("typedef unsigned int u32; typedef unsigned long long u64;\n") + var cChecks strings.Builder + cChecks.WriteString("int main(void){\n") + sigs := make(map[string]FuncSig) + count := 0 + for _, width := range []int{4, 8} { + if arch == "386" && width == 8 { + continue // The 386 scalar MOVQ spelling is not an integer GP read. + } + for index, value := range values { + name := fmt.Sprintf("Read%d_%d", width, index) + typ, goType, cType, move := I32, "uint32", "u32", "MOVL" + if width == 8 { + typ, goType, cType, move = I64, "uint64", "u64", "MOVQ" + } else { + value = uint64(uint32(value)) + } + fmt.Fprintf(&declarations, "func %s()(r %s)\n", name, goType) + fmt.Fprintf(&goMain, "if got:=%s(); got!=%s(0x%x){panic(\"%s\")}\n", name, goType, value, name) + fmt.Fprintf(&cMain, "extern %s %s(void);\n", cType, name) + fmt.Fprintf(&cChecks, "if(%s()!=0x%xULL)return %d;\n", name, value, count+1) + fmt.Fprintf(&native, "DATA pool%d_%d<>+0(SB)/%d,$0x%x\nGLOBL pool%d_%d<>(SB),8,$%d\nTEXT ·%s(SB),4,$0-%d\n%s pool%d_%d<>(SB),AX\n%s AX,r+0(FP)\nBYTE $0xc3\n", + width, index, width, value, width, index, width, name, width, move, width, index, move) + if arch == "amd64" { + code := []byte{0x8b, 0x05, 1, 0, 0, 0, 0xc3} + if width == 8 { + code = append([]byte{0x48}, code...) + } + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, value) + raw.WriteString(x86StaticReturnSource(name, append(code, data[:width]...))) + } else { + // Same-file DATA allocations are architecture-neutral. In 386 + // mode these are static absolute references, never RIP reads. + fmt.Fprintf(&raw, "DATA pool%d_%d<>+0(SB)/%d,$0x%x\nTEXT %s(SB),4,$0-0\n%s pool%d_%d<>(SB),AX\nBYTE $0xc3\n", + width, index, width, value, name, move, width, index) + } + sigs[name] = FuncSig{Name: name, Ret: typ} + count++ + } + } + if arch == "amd64" { + // A source-local LEA makes the pool mutable through its returned + // address. The shared read proof must not resurrect specialization. + declarations.WriteString("func Mutable()(p *uint64,v uint64)\n") + native.WriteString("DATA mutable<>+0(SB)/8,$0x123456789abcdef\nGLOBL mutable<>(SB),16,$8\nTEXT ·Mutable(SB),4,$0-16\nLEAQ mutable<>(SB),AX\nMOVQ AX,p+0(FP)\nMOVQ mutable<>(SB),BX\nMOVQ BX,v+8(FP)\nBYTE $0xc3\n") + goMain.WriteString("p,v:=Mutable();if p==nil||v!=0x123456789abcdef{panic(\"initial mutable\")}\n*p=0xfedcba9876543210\nq,v:=Mutable();if p!=q||v!=0xfedcba9876543210{panic(\"changed mutable\")}\n") + code := []byte{ + 0x48, 0x8d, 0x1d, 0x08, 0, 0, 0, // LEAQ pool,BX. + 0x48, 0x8b, 0x05, 0x01, 0, 0, 0, // MOVQ pool,AX. + 0xc3, + } + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, 0x123456789abcdef) + raw.WriteString(x86StaticReturnSource("Mutable", append(code, data...))) + sigs["Mutable"] = FuncSig{Name: "Mutable", Ret: I64} + cMain.WriteString("extern u64 Mutable(void); extern u64 MutablePool[];\n") + cChecks.WriteString("if(Mutable()!=0x123456789abcdefULL)return 98;\nMutablePool[0]=0xfedcba9876543210ULL;\nif(Mutable()!=0xfedcba9876543210ULL)return 99;\n") + } + goMain.WriteString("}\n") + cChecks.WriteString("return 0;}\n") + cMain.WriteString(cChecks.String()) + goSource := strings.Replace(goMain.String(), "func main", declarations.String()+"func main", 1) + project := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module example.com/static-read-oracle\ngo 1.20\n", + "main.go": goSource, "read_" + arch + ".s": native.String(), + } { + if err := os.WriteFile(filepath.Join(project, name), []byte(content), 0600); err != nil { + t.Fatal(err) + } + } + executable := filepath.Join(project, "oracle") + cmd := exec.Command("go", "build", "-o", executable, ".") + cmd.Dir = project + goos := runtime.GOOS + if len(runner) != 0 { + goos = "linux" + } + cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+arch, "CGO_ENABLED=0", "GOWORK=off", "GOTOOLCHAIN=local") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("actual Go static-read oracle build: %v\n%s", err, output) + } + run := append(append([]string(nil), runner...), executable) + if output, err := exec.Command(run[0], run[1:]...).CombinedOutput(); err != nil { + t.Fatalf("actual Go static-read oracle run: %v\n%s", err, output) + } + t.Logf("actual Go %s: %d ordinary static scalar result cases", arch, count) + file := mustParseX86RawReturn(t, raw.String()) + opt := Options{Goarch: arch, TargetTriple: triple, Sigs: sigs, + ResolveSym: func(symbol string) string { + if strings.HasPrefix(symbol, "·__plan9asm_raw_address_") { + return "MutablePool" + } + return symbol + }} + for _, api := range []string{"text", "module", "owned-module"} { + var ir string + switch api { + case "text": + var err error + ir, err = Translate(file, opt) + if err != nil { + t.Fatal(err) + } + case "module": + module, err := TranslateModule(file, opt) + if err != nil { + t.Fatal(err) + } + ir = module.String() + module.Dispose() + default: + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err != nil { + ctx.Dispose() + t.Fatal(err) + } + ir = module.String() + module.Dispose() + ctx.Dispose() + } + compileAndRunRuntimeTestWithCompiler(t, llc, compiler, "static-read-"+api, triple, ir, cMain.String(), runner) + t.Logf("LLVM 22 %s/%s: %d scalar cases", arch, api, count) + } +} + +func TestX86RawReturnStaticReadNumericOracle(t *testing.T) { + if runtime.GOARCH != "amd64" && !(runtime.GOOS == "darwin" && runtime.GOARCH == "arm64") { + t.Skip("host-inapplicable: x86 scalar oracle runs in the required Linux/amd64 cross-runtime gate") + } + clang := findLLVM22Tool("clang") + if clang == "" { + t.Fatal("LLVM 22 clang not found") + } + triple := testTargetTriple(runtime.GOOS, "amd64") + compiler := []string{clang, "-target", triple} + if runtime.GOOS == "darwin" { + output, err := exec.Command("xcrun", "--show-sdk-path").CombinedOutput() + if err != nil { + t.Fatalf("required Darwin SDK: %v\n%s", err, output) + } + compiler = append(compiler, "-isysroot", strings.TrimSpace(string(output))) + } + runX86StaticReadNumericOracle(t, "amd64", triple, compiler, nil) +} + +func TestCrossLinuxRuntimeMatrixX86RawStaticRead(t *testing.T) { + if os.Getenv("PLAN9ASM_CROSS_EXEC") != "1" { + return // The explicit cross job runs this required counterpart. + } + if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" { + t.Fatal("cross static-read oracle requires Linux/amd64 runner") + } + qemu, err := exec.LookPath("qemu-i386") + if err != nil { + t.Fatal("required pinned qemu-i386:", err) + } + output, err := exec.Command(qemu, "--version").CombinedOutput() + if err != nil || !strings.Contains(string(output), "version 10.2.3") { + t.Fatalf("required QEMU 10.2.3: %v\n%s", err, output) + } + compiler, err := exec.LookPath("i686-linux-gnu-gcc") + if err != nil { + t.Fatal("required 386 cross compiler:", err) + } + runX86StaticReadNumericOracle(t, "386", "i386-unknown-linux-gnu", []string{compiler, "-no-pie"}, []string{qemu, "-L", "/usr/i686-linux-gnu"}) +} diff --git a/amd64_raw_static_return_test.go b/amd64_raw_static_return_test.go new file mode 100644 index 000000000..d0d4061db --- /dev/null +++ b/amd64_raw_static_return_test.go @@ -0,0 +1,249 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86StaticReturnSource(name string, code []byte) string { + var source strings.Builder + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + return source.String() +} + +func TestX86RawReturnSharedRIPReadContract(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + cases := []struct { + name string + code []byte + width int + }{ + {"legacy-i32", []byte{0x8b, 0x05, 1, 0, 0, 0, 0xc3, 1, 2, 3, 4}, 4}, + {"legacy-i64", []byte{0x48, 0x8b, 0x05, 1, 0, 0, 0, 0xc3, 1, 2, 3, 4, 5, 6, 7, 8}, 8}, + {"legacy-vector", x86RawLegacyPackedMoveConstant(0xf3, 0x6f), 16}, + {"vex-vector", x86RawPackedMoveLiteral(false, 32), 32}, + {"evex-vector", x86RawPackedMoveLiteral(true, 64), 64}, + {"broadcast-f32", x86RawBinaryFloatLiteral(0x58, 0, true, 64, true), 4}, + {"broadcast-f64", x86RawBinaryFloatLiteral(0x58, 1, true, 64, true), 8}, + {"scalar-f32", x86RawBinaryFloatLiteral(0x58, 2, false, 16, false), 4}, + {"scalar-f64", x86RawBinaryFloatLiteral(0x58, 3, false, 16, false), 8}, + {"nonfirst-source", x86RawCLMULLiteral("legacy", 16), 16}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + source := x86StaticReturnSource("Bound", test.code) + requireX86GoAssemblerResult(t, "amd64", source, true) + prepared, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal("shared RIP read rejected before materialization:", err) + } + if len(prepared.Data) != 1 || prepared.Data[0].Width != int64(test.width) { + t.Fatalf("materialized data: %+v, want exact width %d", prepared.Data, test.width) + } + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue // RIP-relative addressing exists only in 64-bit mode. + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(prepared, opt) + if err != nil { + t.Fatal("prepartition/final validation disagreed:", err) + } + compileLLVMToObject(t, llc, target.triple, "bounded-rip.ll", "bounded-rip.o", ir) + } + }) + } +} + +func TestX86RawReturnStaticDATAExtents(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + for _, offset := range []int{0, 4, 5} { + source := fmt.Sprintf("DATA pool<>+4(SB)/4,$0x76543210\nTEXT Bound(SB),4,$0-0\nMOVL pool<>+%d(SB),AX\nBYTE $0xc3\n", offset) + requireX86GoAssemblerResult(t, target.arch, source, true) + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: I32}}} + x86RawReturnPointerAPIs(t, source, opt, offset == 5) + if offset != 5 { + ir, err := Translate(mustParseX86RawReturn(t, source), opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "bounded-data.ll", "bounded-data.o", ir) + } + } + } +} + +func TestX86RawReturnMaterializedReadCannotAuthorizeChangedSource(t *testing.T) { + source := x86StaticReturnSource("Bound", x86RawPackedMoveLiteral(false, 32)) + mutations := map[string]func(*File){ + "opcode store": func(file *File) { + file.Funcs[0].Instrs[1].Op = "VMOVDQU" + args := file.Funcs[0].Instrs[1].Args + args[0], args[1] = args[1], args[0] + }, + "unrecognized opcode": func(file *File) { + file.Funcs[0].Instrs[1].Op = "UNRECOGNIZED" + }, + "pointer operand": func(file *File) { + file.Funcs[0].Instrs[1].Args[0] = Operand{Kind: OpMem, Mem: MemRef{Base: AX}} + }, + "shifted source": func(file *File) { + file.Funcs[0].Instrs[1].Args[0].Sym = file.Data[0].Sym + "+1(SB)" + }, + "missing data": func(file *File) { + file.Data = nil + }, + "short object": func(file *File) { + file.Data[0].Width-- + file.Data[0].Payload = file.Data[0].Payload[:len(file.Data[0].Payload)-1] + }, + "changed payload": func(file *File) { + payload := append([]byte(nil), file.Data[0].Payload...) + payload[0] ^= 0xff + file.Data[0].Payload = payload + }, + "extra DATA write": func(file *File) { + file.Data = append(file.Data, DataStmt{Sym: file.Data[0].Sym, Width: 1, Value: 0xff}) + }, + "wrong pending index": func(file *File) { + file.Funcs[0].Instrs[1].Args[1] = Operand{Kind: OpSym, Sym: "·__plan9asm_raw_literal_pending(SB)"} + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal(err) + } + mutate(file) + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + // Test every final API on this exact prepared/changed file. + if x86RawReturnMemoryBound(file.Funcs[0], file, opt, true) { + t.Fatal("a decoded read authorized changed source or missing backing") + } + for _, api := range []string{"text", "module", "owned-module"} { + var callErr error + switch api { + case "text": + _, callErr = Translate(file, opt) + case "module": + module, err := TranslateModule(file, opt) + if err == nil { + module.Dispose() + } + callErr = err + default: + ctx := llvm.NewContext() + module, err := TranslateModuleInContext(ctx, file, opt) + if err == nil { + module.Dispose() + } + ctx.Dispose() + callErr = err + } + if !errors.Is(callErr, ErrProbeNeedsContext) { + t.Errorf("%s changed prepared source bypassed the read contract: %v", api, callErr) + } + } + } + }) + } +} + +func TestX86RawReturnRIPReadRequires64BitDecodeContext(t *testing.T) { + source := x86StaticReturnSource("Bound", x86RawPackedMoveLiteral(false, 32)) + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal(err) + } + opt := Options{Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + _, err = Translate(file, opt) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("64-bit RIP proof was reused as a 32-bit absolute-address proof: %v", err) + } +} + +func TestX86RawReturnSharedAddressPoolRetainsExactReadRange(t *testing.T) { + code := []byte{ + 0x48, 0x8d, 0x05, 0x09, 0, 0, 0, // LEAQ pool,AX. + 0xf3, 0x0f, 0x6f, 0x05, 0x09, 0, 0, 0, // MOVOU pool+8,X0. + 0xc3, + } + code = append(code, make([]byte, 32)...) + source := x86StaticReturnSource("Bound", code) + requireX86GoAssemblerResult(t, "amd64", source, true) + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + if target.arch != "amd64" { + continue + } + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), target.arch) + if err != nil { + t.Fatal(err) + } + read := file.Funcs[0].Instrs[2] + if len(read.x86RIPLiteralData) != 0 || read.x86RIPMemoryRead.offset != 8 || + read.x86RIPMemoryRead.width != 16 || len(read.x86RIPMemoryRead.object) != 32 { + t.Fatalf("shared runtime read lost range or acquired specialization: %+v", read) + } + opt := Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + x86RawReturnPointerAPIs(t, source, opt, false) + ir, err := Translate(file, opt) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "shared-pool.ll", "shared-pool.o", ir) + file.Funcs[0].Instrs[2].Args[0].Sym = file.Data[0].Sym + "+17(SB)" + if x86RawReturnMemoryBound(file.Funcs[0], file, opt, true) { + t.Fatal("changed read extending outside the shared object retained decoder proof") + } + } +} + +func TestX86RawReturnReadProofBindsCompleteOperandShape(t *testing.T) { + source := x86StaticReturnSource("Bound", x86RawPackedMoveLiteral(true, 32)) + file, err := NormalizeRawFileForTranslation(mustParseX86RawReturn(t, source), "amd64") + if err != nil { + t.Fatal(err) + } + // The same valid EVEX opcode now requests twice the memory width. A + // private proof tied only to Op/source-index would incorrectly survive. + read := &file.Funcs[0].Instrs[1] + read.Args[len(read.Args)-1].Reg = "Z0" + opt := Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"Bound": {Name: "Bound", Ret: Void}}} + if x86RawReturnMemoryBound(file.Funcs[0], file, opt, true) { + t.Error("wider destination inherited a 32-byte decoded read proof") + } + _, err = Translate(file, opt) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("changed vector width reached final translation without Context: %v", err) + } +} diff --git a/amd64_raw_svm.go b/amd64_raw_svm.go new file mode 100644 index 000000000..8517e834b --- /dev/null +++ b/amd64_raw_svm.go @@ -0,0 +1,30 @@ +package plan9asm + +// decodedX86SystemManagementInstruction covers AMD's fixed 0F 01 D8-DF +// system-management family and Intel's matching KVM hypercall, VMCALL. +// golang.org/x/arch/x86asm does not decode +// these privileged instructions, so packages conventionally spell them as +// Plan 9 BYTE directives even though LLVM 22 can assemble their mnemonics. +func decodedX86SystemManagementInstruction(code []byte) (Instr, int, bool) { + if len(code) < 3 || code[0] != 0x0f || code[1] != 0x01 { + return Instr{}, 0, false + } + if code[2] == 0xc1 { + return Instr{Op: "VMCALL", Raw: "VMCALL"}, 3, true + } + if code[2] < 0xd8 || code[2] > 0xdf { + return Instr{}, 0, false + } + ops := [...]Op{ + "VMRUN", + "VMMCALL", + "VMLOAD", + "VMSAVE", + "STGI", + "CLGI", + "SKINIT", + "INVLPGA", + } + op := ops[code[2]-0xd8] + return Instr{Op: op, Raw: string(op)}, 3, true +} diff --git a/amd64_raw_svm_test.go b/amd64_raw_svm_test.go new file mode 100644 index 000000000..c8faba1a1 --- /dev/null +++ b/amd64_raw_svm_test.go @@ -0,0 +1,107 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86RawDecoderCompleteAMDSystemManagementFamily(t *testing.T) { + family := []struct { + opcode byte + op Op + }{ + {opcode: 0xd8, op: "VMRUN"}, + {opcode: 0xd9, op: "VMMCALL"}, + {opcode: 0xda, op: "VMLOAD"}, + {opcode: 0xdb, op: "VMSAVE"}, + {opcode: 0xdc, op: "STGI"}, + {opcode: 0xdd, op: "CLGI"}, + {opcode: 0xde, op: "SKINIT"}, + {opcode: 0xdf, op: "INVLPGA"}, + } + for _, goarch := range []string{"386", "amd64"} { + for _, test := range family { + t.Run(goarch+"/"+string(test.op), func(t *testing.T) { + fn := Func{Instrs: []Instr{ + {Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: 0x0f}}}, + {Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: 0x01}}}, + {Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(test.opcode)}}}, + }} + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != test.op || len(got.Instrs[0].Args) != 0 { + t.Fatalf("decoded 0f 01 %02x as %#v, want one operand-free %s", test.opcode, got.Instrs, test.op) + } + }) + } + } +} + +func TestTranslateX86RawAMDSystemManagementFamilyAllTargets(t *testing.T) { + const source = `TEXT svmfamily(SB),$0-0 + BYTE $0x0f; BYTE $0x01; BYTE $0xd8 // VMRUN + BYTE $0x0f; BYTE $0x01; BYTE $0xd9 // VMMCALL + BYTE $0x0f; BYTE $0x01; BYTE $0xda // VMLOAD + BYTE $0x0f; BYTE $0x01; BYTE $0xdb // VMSAVE + BYTE $0x0f; BYTE $0x01; BYTE $0xdc // STGI + BYTE $0x0f; BYTE $0x01; BYTE $0xdd // CLGI + BYTE $0x0f; BYTE $0x01; BYTE $0xde // SKINIT + BYTE $0x0f; BYTE $0x01; BYTE $0xdf // INVLPGA + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{"svmfamily": {Name: "svmfamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + for _, op := range []string{"vmrun", "vmmcall", "vmload", "vmsave", "stgi", "clgi", "skinit", "invlpga"} { + if !strings.Contains(ll, `asm sideeffect "`+op+`"`) { + t.Fatalf("translated IR does not retain %s:\n%s", op, ll) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-svm-family-"+target.name+".ll", "raw-svm-family-"+target.name+".o", ll) + }) + } +} + +func TestTranslateX86AMDSystemManagementFamilyRejectsOperands(t *testing.T) { + for _, op := range []string{"VMRUN", "VMMCALL", "VMLOAD", "VMSAVE", "STGI", "CLGI", "SKINIT", "INVLPGA"} { + file, err := Parse(ArchAMD64, "TEXT bad(SB),NOSPLIT,$0-0\n\t"+op+" AX\n") + if err != nil { + t.Fatalf("Parse(%s AX) error = %v", op, err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}, + }); err == nil { + t.Fatalf("Translate accepted operand form for fixed no-operand %s", op) + } + } +} diff --git a/amd64_raw_text_blob_test.go b/amd64_raw_text_blob_test.go new file mode 100644 index 000000000..cbfe80bcf --- /dev/null +++ b/amd64_raw_text_blob_test.go @@ -0,0 +1,219 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestX86AddressSensitiveRawTextBlobReportedURootFarJump(t *testing.T) { + const source = `TEXT setup(SB),NOSPLIT,$0-0 + MOVL AX, farjump64+6(SB) + JMP farjump64(SB) +TEXT farjump64(SB),NOSPLIT,$0-0 + BYTE $0xff; BYTE $0x2d; LONG $0 + LONG $0 + LONG $8 +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "setup": {Name: "setup", Ret: Void}, + "farjump64": {Name: "farjump64", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "define void @farjump64()") || + !strings.Contains(ir, "naked noinline") || + !strings.Contains(ir, `.byte 255, 45, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0`) { + t.Fatalf("byte-exact naked raw TEXT body missing:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "raw-text-"+target.name+".ll", "raw-text-"+target.name+".o", ir) + assertX86RawTextAssemblyBytes(t, llc, target.triple, ir, []byte{0xff, 0x2d, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0}) + }) + } +} + +func TestX86AddressSensitiveRawTextLeadingPCALIGNFormats(t *testing.T) { + for _, alignment := range []int{8, 16, 32, 64, 128, 256, 512, 1024, 2048} { + t.Run(strconv.Itoa(alignment), func(t *testing.T) { + source := fmt.Sprintf(`TEXT refer(SB),$0-0 + LEAQ data(SB), AX + RET +TEXT data(SB),$0 + PCALIGN $%d + LONG $0xdeadbeef +`, alignment) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + for pass := 0; pass < 2; pass++ { + file, err = normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if got := file.Funcs[1].X86RawAlign; got != int64(alignment) { + t.Fatalf("alignment after normalization = %d, want %d", got, alignment) + } + } + }) + } +} + +func assertX86RawTextAssemblyBytes(t *testing.T, llc, triple, ir string, want []byte) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "raw-text.ll") + if err := os.WriteFile(path, []byte(ir), 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(llc, "-mtriple="+triple, "-filetype=asm", "-o", "-", path) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("LLVM 22 assembly emission failed: %v\n%s", err, out) + } + var got []byte + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) != 2 || fields[0] != ".byte" { + continue + } + value, err := strconv.ParseUint(fields[1], 0, 8) + if err != nil { + t.Fatalf("parse emitted byte line %q: %v", line, err) + } + got = append(got, byte(value)) + } + if string(got) != string(want) { + t.Fatalf("LLVM 22 emitted raw TEXT bytes %v, want %v\n%s", got, want, out) + } +} + +func TestX86AddressSensitiveRawTextBlobSelection(t *testing.T) { + const source = `TEXT refs(SB),NOSPLIT,$0-0 + MOVL AX, farjump32+1(SB) + LEAQ gdt(SB), CX + MOVL info(SB), BX + RET +TEXT farjump32(SB),NOSPLIT,$0-0 + BYTE $0xea; LONG $0; WORD $0x18 +TEXT gdt(SB),NOSPLIT,$0-0 + QUAD $0; QUAD $0x00cf9a000000ffff +TEXT info(SB),NOSPLIT,$0-0 + LONG $0 +TEXT semantic(SB),NOSPLIT,$0-0 + BYTE $0x90; BYTE $0xc3 +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + wants := map[string][]byte{ + "farjump32": {0xea, 0, 0, 0, 0, 0x18, 0}, + "gdt": {0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0, 0x9a, 0xcf, 0}, + "info": {0, 0, 0, 0}, + } + for _, fn := range normalized.Funcs { + want, selected := wants[fn.Sym] + if selected && string(fn.X86RawText) != string(want) { + t.Fatalf("%s raw TEXT bytes %v, want %v", fn.Sym, fn.X86RawText, want) + } + if fn.Sym == "semantic" && fn.X86RawText != nil { + t.Fatal("unaddressed raw instruction body bypassed semantic decoding") + } + } +} + +func TestX86AddressSensitiveAlignedRawTextBlobReportedURoot(t *testing.T) { + const source = `TEXT trampoline_start(SB),$0-0 + LEAQ stack_top(SB), AX + MOVQ (AX), AX + RET +TEXT stack_top(SB),$0 + PCALIGN $32 + LONG $0xdeadbeef +` + requireX86GoAssemblerResult(t, "amd64", source, true) + + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + var blob *Func + for index := range normalized.Funcs { + if normalized.Funcs[index].Sym == "stack_top" { + blob = &normalized.Funcs[index] + } + } + if blob == nil || string(blob.X86RawText) != string([]byte{0xef, 0xbe, 0xad, 0xde}) || blob.X86RawAlign != 32 { + t.Fatalf("aligned raw TEXT was not retained: %+v", blob) + } + + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + triple string + }{ + {name: "linux", triple: "x86_64-unknown-linux-gnu"}, + {name: "darwin", triple: "x86_64-apple-darwin"}, + {name: "windows", triple: "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: target.triple, + Sigs: map[string]FuncSig{ + "trampoline_start": {Name: "trampoline_start", Ret: Void}, + "stack_top": {Name: "stack_top", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "define void @stack_top()") || !strings.Contains(ir, " align 32 {") { + t.Fatal("aligned raw TEXT declaration missing") + } + compileLLVMToObject(t, llc, target.triple, "aligned-raw-text.ll", "aligned-raw-text.o", ir) + assertX86RawTextAssemblyBytes(t, llc, target.triple, ir, []byte{0xef, 0xbe, 0xad, 0xde}) + }) + } +} diff --git a/amd64_raw_text_frame.go b/amd64_raw_text_frame.go new file mode 100644 index 000000000..164368807 --- /dev/null +++ b/amd64_raw_text_frame.go @@ -0,0 +1,51 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Naked emission preserves the raw directive bytes, not obj6's additional +// Go frame, wrapper or stack-split prologue. Keep this source boundary explicit +// even when the body's bytes are data rather than decodable instructions. +func validateX86AddressSensitiveRawText(fn Func, goarch string) error { + if fn.FrameSize != 0 || fn.ArgSize != 0 { + return fmt.Errorf("%w: address-sensitive raw TEXT needs its source Go frame/argument transport contract", ErrProbeNeedsContext) + } + if len(fn.Instrs) == 0 || fn.Instrs[0].Op != OpTEXT { + return fmt.Errorf("%w: address-sensitive raw TEXT needs an actual source TEXT header", ErrProbeNeedsContext) + } + _, rest := splitOpcode(fn.Instrs[0].Raw) + parts := strings.Split(rest, ",") + if len(parts) != 2 && len(parts) != 3 { + return fmt.Errorf("%w: address-sensitive raw TEXT needs a resolved source header", ErrProbeNeedsContext) + } + if !strings.HasPrefix(strings.TrimSpace(parts[len(parts)-1]), "$") { + return fmt.Errorf("%w: address-sensitive raw TEXT needs a known source frame", ErrProbeNeedsContext) + } + frame, args, err := parseTEXTFrame(parts) + if err != nil || frame != fn.FrameSize || args != fn.ArgSize { + return fmt.Errorf("%w: address-sensitive raw TEXT source frame differs from its retained metadata", ErrProbeNeedsContext) + } + var flags uint64 + if len(parts) == 3 { + expression := globlFlagName.ReplaceAllStringFunc(parts[1], func(name string) string { + if value, known := goTextFlagValues[name]; known { + return value + } + return name + }) + var known bool + flags, known = parseImmExpr(strings.TrimSpace(expression)) + if !known { + return fmt.Errorf("%w: address-sensitive raw TEXT flags need actual Go source context", ErrProbeNeedsContext) + } + } + if flags & ^uint64(2|4|512) != 0 { + return fmt.Errorf("%w: address-sensitive raw TEXT requires unmodeled Go prologue/entry flags", ErrProbeNeedsContext) + } + if goarch == "386" && flags&4 == 0 { + return fmt.Errorf("%w: address-sensitive 386 raw TEXT cannot omit Go's stack-split prologue", ErrProbeNeedsContext) + } + return nil +} diff --git a/amd64_raw_text_frame_test.go b/amd64_raw_text_frame_test.go new file mode 100644 index 000000000..5ebdbda45 --- /dev/null +++ b/amd64_raw_text_frame_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "errors" + "fmt" + "testing" + + "github.com/xgo-dev/llvm" +) + +func x86AddressSensitiveFrameSource(arch, text string) string { + address := "LEAQ" + if arch == "386" { + address = "LEAL" + } + return fmt.Sprintf("TEXT refer(SB),4,$0-0\n%s blob(SB),AX\nRET\nTEXT blob(SB),%s\nBYTE $0xc3\n", address, text) +} + +func requireX86RawTextContextAPIs(t *testing.T, file *File, opt Options) { + t.Helper() + for _, api := range []string{"text", "module", "owned-module", "annotated"} { + var callErr error + switch api { + case "text": + _, callErr = Translate(file, opt) + case "module": + module, err := TranslateModule(file, opt) + if err == nil { + module.Dispose() + } + callErr = err + default: + ctx := llvm.NewContext() + options := opt + options.AnnotateSource = api == "annotated" + module, err := TranslateModuleInContext(ctx, file, options) + if err == nil { + module.Dispose() + } + ctx.Dispose() + callErr = err + } + if !errors.Is(callErr, ErrProbeNeedsContext) { + t.Errorf("%s omitted raw TEXT's source prologue/Go-frame contract: %v", api, callErr) + } + } +} + +func TestX86AddressSensitiveRawTextNeedsSourceFrameContract(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, text := range []string{"4,$8-0", "4,$16-0", "4,$0-8", "32,$0-0"} { + t.Run(target.triple+"/"+text, func(t *testing.T) { + source := x86AddressSensitiveFrameSource(target.arch, text) + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + requireX86RawTextContextAPIs(t, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": {Name: "blob", Ret: Void}}, + }) + }) + } + } +} + +func TestX86AddressSensitiveRawTextCannotBypassGoFrameTransport(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, sig := range []FuncSig{ + {Name: "blob", Ret: Void, Args: []LLVMType{I32}, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: I32, Field: -1}}}}, + {Name: "blob", Ret: I32, Frame: FrameLayout{Results: []FrameSlot{{Offset: 0, Type: I32, Field: -1}}}}, + } { + t.Run(target.triple+"/"+string(sig.Ret), func(t *testing.T) { + source := x86AddressSensitiveFrameSource(target.arch, "4,$0-0") + requireX86GoAssemblerResult(t, target.arch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + requireX86RawTextContextAPIs(t, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": sig}, + }) + }) + } + } +} + +func TestX86AddressSensitiveRawText386CannotOmitStackSplit(t *testing.T) { + // obj6's automatic NOSPLIT leaf promotion is amd64-only. An unflagged + // 386 TEXT has real TLS/stack-check bytes before the raw source body. + plain := assembleX87ControlBytes(t, "386", "TEXT blob(SB),$0-0\nBYTE $0xc3\n") + nosplit := assembleX87ControlBytes(t, "386", "TEXT blob(SB),4,$0-0\nBYTE $0xc3\n") + if len(nosplit) != 1 || nosplit[0] != 0xc3 || len(plain) <= len(nosplit) { + t.Fatalf("Go source prologue witness is not independent: plain=%x NOSPLIT=%x", plain, nosplit) + } + t.Logf("actual Go 386 source TEXT bytes: unflagged=%x NOSPLIT=%x", plain, nosplit) + file, err := Parse(ArchAMD64, x86AddressSensitiveFrameSource("386", "$0-0")) + if err != nil { + t.Fatal(err) + } + requireX86RawTextContextAPIs(t, file, Options{ + Goarch: "386", TargetTriple: "i386-unknown-linux-gnu", + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": {Name: "blob", Ret: Void}}, + }) +} + +func TestX86AddressSensitiveRawTextFramelessBytesRemainExact(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range x86RawReturnTargets { + flags := []string{"4,$0-0", "6,$0-0", "516,$0-0"} + if target.arch == "amd64" { + // obj6 promotes this raw-only, zero-frame amd64 leaf to NOSPLIT. + flags = append(flags, "$0-0") + } + for _, text := range flags { + t.Run(target.triple+"/"+text, func(t *testing.T) { + original := assembleX87ControlBytes(t, target.arch, "TEXT blob(SB),"+text+"\nBYTE $0xc3\n") + if len(original) != 1 || original[0] != 0xc3 { + t.Fatalf("source Go TEXT unexpectedly adds a native prologue: %x", original) + } + file, err := Parse(ArchAMD64, x86AddressSensitiveFrameSource(target.arch, text)) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := TranslateModuleInContext(ctx, file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"refer": {Name: "refer", Ret: Void}, "blob": {Name: "blob", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + defer module.Dispose() + compileLLVMToObject(t, llc, target.triple, "frameless-raw-text.ll", "frameless-raw-text.o", module.String()) + assertX86RawTextAssemblyBytes(t, llc, target.triple, module.String(), original) + }) + } + } +} diff --git a/amd64_raw_unpack_float_test.go b/amd64_raw_unpack_float_test.go new file mode 100644 index 000000000..01f163aa4 --- /dev/null +++ b/amd64_raw_unpack_float_test.go @@ -0,0 +1,269 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +type rawFloatingUnpackSpec struct { + opcode byte + double bool + high bool +} + +var rawFloatingUnpackSpecs = []rawFloatingUnpackSpec{ + {opcode: 0x14}, + {opcode: 0x15, high: true}, + {opcode: 0x14, double: true}, + {opcode: 0x15, double: true, high: true}, +} + +func (spec rawFloatingUnpackSpec) op(vector bool) Op { + name := "UNPCKL" + if spec.high { + name = "UNPCKH" + } + if vector { + name = "V" + name + } + if spec.double { + name += "PD" + } else { + name += "PS" + } + return Op(name) +} + +func encodeX86RawLegacyFloatingUnpack(spec rawFloatingUnpackSpec, source, destination int) []byte { + code := []byte{} + if spec.double { + code = append(code, 0x66) + } + rex := byte(0x40 | source>>3&1 | destination>>3&1<<2) + if rex != 0x40 { + code = append(code, rex) + } + return append(code, 0x0f, spec.opcode, byte(0xc0|destination&7<<3|source&7)) +} + +func encodeX86RawVEXFloatingUnpack(spec rawFloatingUnpackSpec, vectorBits byte, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | 0x40 | byte((^first>>3)&1)<<5 | 1 + p1 := byte(^second&15)<<3 | vectorBits<<2 + if spec.double { + p1 |= 1 + } + return []byte{0xc4, p0, p1, spec.opcode, byte(0xc0 | destination&7<<3 | first&7)} +} + +func encodeX86RawEVEXFloatingUnpack(spec rawFloatingUnpackSpec, vectorBits byte, mask int, zeroing, broadcast bool, first, second, destination int) []byte { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 1 + p1 := byte(^second&15)<<3 | 4 + if spec.double { + p1 |= 0x81 + } + p2 := vectorBits<<5 | byte((^second>>4)&1)<<3 | byte(mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + return []byte{0x62, p0, p1, p2, spec.opcode, byte(0xc0 | destination&7<<3 | first&7)} +} + +func TestDecodeX86RawDirectiveGroupReportedSegDSPVUNPCKLPS(t *testing.T) { + // vunpcklps %xmm3, %xmm5, %xmm0; + // vinsertf128 $1, %xmm0, %ymm0, %ymm0 + code := []byte{0xc5, 0xd0, 0x14, 0xc3, 0xc4, 0xe3, 0x7d, 0x18, 0xc0, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "segdsp VUNPCKLPS sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !strings.HasPrefix(decoded[0].Raw, "VUNPCKLPS X3, X5, X0 ") { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodedX86RawFloatingUnpackCompleteRegisterFamily(t *testing.T) { + count := 0 + for _, spec := range rawFloatingUnpackSpecs { + for first := 0; first < 16; first++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawLegacyFloatingUnpack(spec, first, destination) + got, length, ok, err := decodedX86FloatingUnpackInstruction(code, 64) + want := fmt.Sprintf("%s X%d, X%d", spec.op(false), first, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + for vectorBits := byte(0); vectorBits < 2; vectorBits++ { + vectorName := [...]string{"X", "Y"}[vectorBits] + for first := 0; first < 16; first++ { + for second := 0; second < 16; second++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86RawVEXFloatingUnpack(spec, vectorBits, first, second, destination) + got, length, ok, err := decodedX86FloatingUnpackInstruction(code, 64) + want := fmt.Sprintf("%s %s%d, %s%d, %s%d", spec.op(true), vectorName, first, vectorName, second, vectorName, destination) + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + vectorName := [...]string{"X", "Y", "Z"}[vectorBits] + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{{}, {mask: 3}, {mask: 7, zeroing: true, suffix: ".Z"}} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + code := encodeX86RawEVEXFloatingUnpack(spec, vectorBits, masking.mask, masking.zeroing, false, first, second, destination) + got, length, ok, err := decodedX86FloatingUnpackInstruction(code, 64) + args := []string{fmt.Sprintf("%s%d", vectorName, first), fmt.Sprintf("%s%d", vectorName, second)} + if masking.mask != 0 { + args = append(args, fmt.Sprintf("K%d", masking.mask)) + } + args = append(args, fmt.Sprintf("%s%d", vectorName, destination)) + want := string(spec.op(true)) + masking.suffix + " " + strings.Join(args, ", ") + if err != nil || !ok || length != len(code) || got.Raw != want { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v; want %q", code, got, length, ok, err, want) + } + count++ + } + } + } + } + } + } + if count != 1213440 { + t.Fatalf("covered %d floating-unpack register encodings, want 1213440", count) + } +} + +func TestDecodedX86RawFloatingUnpackMemoryAndInvalidForms(t *testing.T) { + legacyMemory := []byte{0x66, 0x47, 0x0f, 0x15, 0x7c, 0x8b, 0x01} + got, length, ok, err := decodedX86FloatingUnpackInstruction(legacyMemory, 64) + wantMemory := MemRef{Base: "R11", Index: "R9", Scale: 4, Off: 1} + if err != nil || !ok || length != len(legacyMemory) || got.Op != "UNPCKHPD" || got.Args[1].String() != "X15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", legacyMemory, got, length, ok, err) + } + vexMemory := []byte{0xc4, 0x01, 0x75, 0x14, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86FloatingUnpackInstruction(vexMemory, 64) + wantMemory.Off = 2 + if err != nil || !ok || length != len(vexMemory) || got.Op != "VUNPCKLPD" || got.Args[1].String() != "Y1" || got.Args[2].String() != "Y15" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", vexMemory, got, length, ok, err) + } + evexMemory := []byte{0x62, 0x01, 0x85, 0xdf, 0x15, 0x7c, 0x8b, 0x02} + got, length, ok, err = decodedX86FloatingUnpackInstruction(evexMemory, 64) + wantMemory.Off = 16 + if err != nil || !ok || length != len(evexMemory) || got.Op != "VUNPCKHPD.BCST.Z" || got.Args[1].String() != "Z15" || got.Args[2].String() != "K7" || got.Args[3].String() != "Z31" || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", evexMemory, got, length, ok, err) + } + + validVEX := encodeX86RawVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 0, 7, 7, 7) + validEVEX := encodeX86RawEVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 2, 0, false, false, 7, 7, 7) + for name, invalid := range map[string][]byte{ + "vex W": {validVEX[0], validVEX[1], validVEX[2] | 0x80, validVEX[3], validVEX[4]}, + "vex prefix": {validVEX[0], validVEX[1], validVEX[2] | 0x02, validVEX[3], validVEX[4]}, + "evex missing fixed bit": {validEVEX[0], validEVEX[1], validEVEX[2] &^ 0x04, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex prefix": {validEVEX[0], validEVEX[1], validEVEX[2] | 0x02, validEVEX[3], validEVEX[4], validEVEX[5]}, + "evex reserved length": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x60, validEVEX[4], validEVEX[5]}, + "zero without mask": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x80, validEVEX[4], validEVEX[5]}, + "register broadcast": {validEVEX[0], validEVEX[1], validEVEX[2], validEVEX[3] | 0x10, validEVEX[4], validEVEX[5]}, + "address override": append([]byte{0x67}, validVEX...), + } { + if instruction, _, matched, decodeErr := decodedX86FloatingUnpackInstruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, invalid, instruction, matched, decodeErr) + } + } + masked386 := encodeX86RawEVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 2, 1, false, false, 0, 1, 2) + if instruction, _, matched, decodeErr := decodedX86FloatingUnpackInstruction(masked386, 32); !matched || decodeErr == nil { + t.Fatalf("386 masked encoding %x decoded as %+v, ok=%v, err=%v", masked386, instruction, matched, decodeErr) + } + highZ386 := encodeX86RawEVEXFloatingUnpack(rawFloatingUnpackSpecs[0], 2, 0, false, false, 8, 1, 2) + if instruction, _, matched, decodeErr := decodedX86FloatingUnpackInstruction(highZ386, 32); !matched || decodeErr == nil { + t.Fatalf("386 high-Z encoding %x decoded as %+v, ok=%v, err=%v", highZ386, instruction, matched, decodeErr) + } +} + +func TestX86FloatingUnpackGo127ArchitectureOracle(t *testing.T) { + const accepted386 = `TEXT ok(SB),$0-0 + VUNPCKLPS X20, X21, X22 + VUNPCKLPD Y20, Y21, Y22 + VUNPCKHPS Z5, Z6, Z7 + VUNPCKHPD.BCST 8(AX), X20, X21 + RET +` + requireX86GoAssemblerResult(t, "386", accepted386, true) + for _, instruction := range []string{ + "VUNPCKLPS Z8, Z1, Z2", + "VUNPCKHPS Z0, Z1, K1, Z2", + } { + requireX86GoAssemblerResult(t, "386", "TEXT bad(SB),$0-0\n\t"+instruction+"\n\tRET\n", false) + } +} + +func TestTranslateX86RawFloatingUnpackTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + registerMax int + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin", registerMax: 15}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu", registerMax: 15}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc", registerMax: 15}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu", registerMax: 7}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc", registerMax: 7}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT raw_floating_unpack(SB),NOSPLIT,$0-0\n") + for _, spec := range rawFloatingUnpackSpecs { + code := [][]byte{ + encodeX86RawLegacyFloatingUnpack(spec, target.registerMax, target.registerMax), + encodeX86RawVEXFloatingUnpack(spec, 1, target.registerMax, target.registerMax, target.registerMax), + } + if target.goarch == "amd64" { + code = append(code, encodeX86RawEVEXFloatingUnpack(spec, 2, 7, true, false, 31, 30, 29)) + } else { + code = append(code, encodeX86RawEVEXFloatingUnpack(spec, 2, 0, false, false, 7, 6, 5)) + } + for _, instruction := range code { + for _, value := range instruction { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"raw_floating_unpack": {Name: "raw_floating_unpack", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-floating-unpack-"+target.name+".ll", "raw-floating-unpack-"+target.name+".o", ir) + }) + } +} diff --git a/amd64_raw_unreachable_tail_test.go b/amd64_raw_unreachable_tail_test.go new file mode 100644 index 000000000..1096e2d11 --- /dev/null +++ b/amd64_raw_unreachable_tail_test.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestTranslateX86UnreachableRawTailAfterReturn(t *testing.T) { + const source = ` +TEXT rawtail(SB),$0-0 + RET +GLOBL structtbl(SB),$8 + BYTE $0x7b; BYTE $0x7d; BYTE $0x5b; BYTE $0x5d + BYTE $0x22; BYTE $0x3a; BYTE $0x2c; BYTE $0x27 +GLOBL wordtbl(SB),$4 + BYTE $0x74; BYTE $0x68; BYTE $0x6e; BYTE $0x6b +` + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "rawtail": {Name: "rawtail", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "structtbl") || !strings.Contains(ll, "wordtbl") { + t.Fatal("GLOBL declarations disappeared from IR") + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, target.triple, "x86-raw-unreachable-tail.ll", "x86-raw-unreachable-tail.o", ll) + }) + } +} + +func TestTranslateX86LabeledRawTailRemainsReachable(t *testing.T) { + const source = ` +TEXT rawtail(SB),$0-0 + JMP tail + RET +tail: + BYTE $0x7b; BYTE $0x7d + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawtail": {Name: "rawtail", Ret: Void}, + }, + }); err == nil || !strings.Contains(err.Error(), "outside directive group") { + t.Fatalf("reachable malformed raw tail was not rejected: %v", err) + } +} + +func TestTranslateX86AddressObservedRawTailFailsClosed(t *testing.T) { + const source = ` +TEXT rawtail(SB),$0-0 + RET + BYTE $0x7b; BYTE $0x7d +TEXT address(SB),$0-0 + MOVQ $rawtail(SB), AX + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawtail": {Name: "rawtail", Ret: Void}, + "address": {Name: "address", Ret: Void}, + }, + }); err == nil || !strings.Contains(err.Error(), "address-observed raw tail") { + t.Fatalf("accepted layout-sensitive raw tail: %v", err) + } +} diff --git a/amd64_raw_valign_test.go b/amd64_raw_valign_test.go new file mode 100644 index 000000000..c86661631 --- /dev/null +++ b/amd64_raw_valign_test.go @@ -0,0 +1,95 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodeX86RawVectorAlignCompleteGo127Family(t *testing.T) { + tests := []struct { + name string + code []byte + want string + }{ + {name: "dword x high masked", code: []byte{0x62, 0xa3, 0x55, 0x01, 0x03, 0xf4, 0xca}, want: "VALIGND $202, X20, X21, K1, X22"}, + {name: "qword y memory zeroing", code: []byte{0x62, 0xd3, 0xf5, 0xaa, 0x03, 0x50, 0x03, 0x96}, want: "VALIGNQ.Z $150, 96(R8), Y1, K2, Y2"}, + {name: "dword z register", code: []byte{0x62, 0x73, 0x55, 0x48, 0x03, 0xce, 0xca}, want: "VALIGND $202, Z6, Z5, Z9"}, + {name: "dword z broadcast high masked", code: []byte{0x62, 0x53, 0x7d, 0x54, 0x03, 0x4c, 0x24, 0x7f, 0x44}, want: "VALIGND.BCST $68, 508(R12), Z16, K4, Z9"}, + {name: "qword z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x03, 0x50, 0x7f, 0x09}, want: "VALIGNQ.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + decoded, err := decodeX86RawDirectiveGroup(test.code, 64, 0, "VALIGN", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || !strings.HasPrefix(decoded[0].Raw, test.want+" ") { + t.Fatalf("decoded %x as %#v, want %q", test.code, decoded, test.want) + } + }) + } +} + +func TestDecodedX86RawVectorAlignAllRegisterFields(t *testing.T) { + for _, lane := range []struct { + w byte + op Op + }{ + {w: 0, op: "VALIGND"}, + {w: 1, op: "VALIGNQ"}, + } { + for vectorBits, vectorName := range []string{"X", "Y", "Z"} { + for first := 0; first < 32; first++ { + for second := 0; second < 32; second++ { + for destination := 0; destination < 32; destination++ { + p0 := byte((^destination>>3)&1)<<7 | + byte((^first>>4)&1)<<6 | + byte((^first>>3)&1)<<5 | + byte((^destination>>4)&1)<<4 | 3 + p1 := lane.w<<7 | byte(^second&15)<<3 | 5 + p2 := byte(vectorBits)<<5 | byte((^second>>4)&1)<<3 + modRM := byte(0xc0 | destination&7<<3 | first&7) + encoding := []byte{0x62, p0, p1, p2, 0x03, modRM, 0xa5} + instruction, length, ok, err := decodedX86VectorAlignInstruction(encoding, 64) + if !ok || err != nil || length != len(encoding) { + t.Fatalf("op=%s width=%s first=%d second=%d destination=%d: ok=%v length=%d err=%v", lane.op, vectorName, first, second, destination, ok, length, err) + } + want := fmt.Sprintf("%s $165, %s%d, %s%d, %s%d", lane.op, vectorName, first, vectorName, second, vectorName, destination) + if instruction.Raw != want { + t.Fatalf("encoding %x decoded as %q, want %q", encoding, instruction.Raw, want) + } + } + } + } + } + } +} + +func TestDecodedX86RawVectorAlignRejectsReservedAndUnsafeForms(t *testing.T) { + for _, test := range []struct { + name string + mode int + encoding []byte + wantOK bool + }{ + {name: "386 operand limit", mode: 32, encoding: []byte{0x62, 0xf3, 0x75, 0x08, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "reserved vector length", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x68, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "broadcast register", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x18, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "zero without mask", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x88, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "address override", mode: 64, encoding: []byte{0x67, 0x62, 0xf3, 0x75, 0x08, 0x03, 0xc2, 0x01}, wantOK: true}, + {name: "missing immediate", mode: 64, encoding: []byte{0x62, 0xf3, 0x75, 0x08, 0x03, 0xc2}, wantOK: false}, + {name: "wrong map", mode: 64, encoding: []byte{0x62, 0xf2, 0x75, 0x08, 0x03, 0xc2, 0x01}, wantOK: false}, + {name: "wrong prefix", mode: 64, encoding: []byte{0x62, 0xf3, 0x74, 0x08, 0x03, 0xc2, 0x01}, wantOK: false}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VectorAlignInstruction(test.encoding, test.mode) + if ok != test.wantOK { + t.Fatalf("encoding %x: ok=%v, want %v (err=%v)", test.encoding, ok, test.wantOK, err) + } + if test.wantOK && err == nil { + t.Fatalf("encoding %x was recognized without rejection", test.encoding) + } + }) + } +} diff --git a/amd64_raw_variable_shift.go b/amd64_raw_variable_shift.go new file mode 100644 index 000000000..5283e526f --- /dev/null +++ b/amd64_raw_variable_shift.go @@ -0,0 +1,42 @@ +package plan9asm + +import "fmt" + +// Go has five VEX+EVEX D/Q members and four EVEX-only W/Q members. +// All use per-element counts, not the scalar count of VPSLL/PSRL/PSRA. +func decodedX86PerLaneVariableShiftInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + // F3.0F38 opcodes 10-12 are single-source narrowing instructions, + // not a malformed 66.0F38 per-lane shift. Other wrong prefixes remain + // owned here so callers get an explicit invalid-form diagnostic. + if !matched || p.mapNumber != 2 || p.pp == 2 { + return Instr{}, 0, false, nil + } + var op Op + var spec amd64PerLaneVariableShiftSpec + owned := false + for name, candidate := range amd64PerLaneVariableShiftSpecs { + if int(candidate.opcode) != p.opcode { + continue + } + owned = true + if p.w == (candidate.laneBits != 32) { + op, spec = Op(name), candidate + break + } + } + if !owned { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("per-lane variable shift: %s", message) + } + if op == "" { + return fail("invalid element width bit") + } + if !p.evex && !spec.vex { + return fail("instruction requires EVEX encoding") + } + + return decodedX86BinaryVectorOperands(code, p, mode, op, spec.laneBits) +} diff --git a/amd64_raw_variable_shift_literal_test.go b/amd64_raw_variable_shift_literal_test.go new file mode 100644 index 000000000..7de2030c6 --- /dev/null +++ b/amd64_raw_variable_shift_literal_test.go @@ -0,0 +1,170 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVariableShiftLiteral(op string, evex bool, width int, broadcast bool) []byte { + spec := amd64PerLaneVariableShiftSpecs[op] + mask, zeroing := 0, false + if evex { + mask, zeroing = 7, true + } + code := encodeRawScalarMove(evex, 1, int(spec.opcode), 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | 2 + code[2] &^= 0x80 + if spec.laneBits != 32 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + if broadcast { + code[3] |= 0x10 + } + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*19+3)) + } + return code +} + +func TestDecodeX86RawVariableShiftRIPDataCompleteFamily(t *testing.T) { + for _, op := range x86VariableShiftTestOps { + spec := amd64PerLaneVariableShiftSpecs[op] + for _, evex := range []bool{false, true} { + if !evex && !spec.vex { + continue + } + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + broadcasts := []bool{false} + if evex && spec.laneBits >= 32 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/evex%t/%d/bcst%t", op, evex, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawVariableShiftLiteral(op, evex, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op(op) + if broadcast { + want += ".BCST" + } + if evex { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawVariableShiftRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawVariableShiftLiteral("VPSLLVQ", true, 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x64}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVariableShiftRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, op := range x86VariableShiftTestOps { + spec := amd64PerLaneVariableShiftSpecs[op] + evex := !spec.vex || index%2 == 0 + width := 32 + if evex { + width = 64 + } + broadcast := evex && spec.laneBits >= 32 + code := x86RawVariableShiftLiteral(op, evex, width, broadcast) + name := fmt.Sprintf("variableShiftLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if broadcast { + dataWidth = spec.laneBits / 8 + } + pools = append(pools, code[len(code)-dataWidth:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "variable-shift-literal.ll", "variable-shift-literal.o", ir) + }) + } +} diff --git a/amd64_raw_variable_shift_test.go b/amd64_raw_variable_shift_test.go new file mode 100644 index 000000000..2112e290b --- /dev/null +++ b/amd64_raw_variable_shift_test.go @@ -0,0 +1,449 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strings" + "testing" +) + +var x86VariableShiftTestOps = []string{"VPSLLVW", "VPSLLVD", "VPSLLVQ", "VPSRLVW", "VPSRLVD", "VPSRLVQ", "VPSRAVW", "VPSRAVD", "VPSRAVQ"} + +func TestX86VariableShiftGrammarMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/avx_optabs.go")) + if err != nil { + t.Fatal(err) + } + rows := regexp.MustCompile(`(?s)\{as: A(VPS(?:LL|RL|RA)V[WDQ]), ytab: (\w+), prefix: Pavx, op: opBytes\{(.*?)\}\}`).FindAllStringSubmatch(string(data), -1) + if len(rows) != 9 || len(rows) != len(amd64PerLaneVariableShiftSpecs) { + t.Fatalf("Go entries=%d, grammar=%d", len(rows), len(amd64PerLaneVariableShiftSpecs)) + } + for _, row := range rows { + spec, ok := amd64PerLaneVariableShiftSpecs[row[1]] + if !ok { + t.Fatalf("missing %s", row[1]) + } + bits := map[byte]int{'W': 16, 'D': 32, 'Q': 64}[row[1][len(row[1])-1]] + operation := "shl" + if strings.HasPrefix(row[1], "VPSRL") { + operation = "lshr" + } + if strings.HasPrefix(row[1], "VPSRA") { + operation = "ashr" + } + if spec.laneBits != bits || spec.operation != operation || spec.arithmetic != (operation == "ashr") { + t.Fatalf("Go semantics differ for %s: %+v", row[1], spec) + } + vex := bits != 16 && row[1] != "VPSRAVQ" + if spec.vex != vex || (row[2] == "_yvandnpd") != vex || strings.Contains(row[3], "| vex128") != vex || strings.Contains(row[3], "evexBcst") != (bits != 16) { + t.Fatalf("Go encoding surface changed for %s: %s", row[1], row[3]) + } + for _, opcode := range regexp.MustCompile(`0x[0-9A-F]+`).FindAllString(row[3], -1) { + if opcode != fmt.Sprintf("0x%X", spec.opcode) { + t.Fatalf("Go opcode %s != spec %+v", opcode, spec) + } + } + } +} + +func TestX86RawVariableShiftGoEncoderForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + last := 7 + if arch == "amd64" { + last = 31 + } + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, op := range x86VariableShiftTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, reg := range []int{0, last} { + for _, input := range []string{fmt.Sprintf("%s%d", width, last-reg), "(AX)", "-7(BX)(CX*4)", "64(SI)", "8192(DI)"} { + fmt.Fprintf(&source, "%s %s,%s%d,%s%d\n", op, input, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s %s,%s%d,K1,%s%d\n%s.Z %s,%s%d,K7,%s%d\n", op, input, width, reg, width, last-reg, op, input, width, reg, width, last-reg) + } + } + if !strings.HasSuffix(op, "W") { + for _, input := range []string{"-8(BX)(CX*4)", "64(SI)"} { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,%s%d\n", op, input, width, reg, width, last-reg) + if arch == "amd64" { + fmt.Fprintf(&source, "%s.BCST %s,%s%d,K1,%s%d\n%s.BCST.Z %s,%s%d,K7,%s%d\n", op, input, width, reg, width, last-reg, op, input, width, reg, width, last-reg) + } + } + } + } + // Low registers force the VEX row where available, even when the + // high-register rows above necessarily select EVEX on amd64. + if width != "Z" { + fmt.Fprintf(&source, "%s %s0,%s1,%s2\n%s (AX),%s1,%s2\n", op, width, width, width, op, width, width) + } + } + } + source.WriteString("RET\n") + code := assembleX87ControlBytes(t, arch, source.String()) + got, err := decodeX86RawDirectives(rawX86Function(code), arch) + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(got.Instrs) != len(want) { + t.Fatalf("got %d instructions, want %d", len(got.Instrs), len(want)) + } + for i, instruction := range got.Instrs { + if instruction.Op != want[i].Op || !reflect.DeepEqual(instruction.Args, want[i].Args) { + t.Fatalf("instruction %d: %+v, want %+v", i, instruction, want[i]) + } + } + file := x86RawUnprovedReturnFormProbeFile(t, arch, "shifts", code) + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (arch == "386") != strings.HasPrefix(triple, "i") { + continue + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "shifts.ll", "shifts.o", ir) + } + t.Logf("%s: %d actual Go encodings", arch, len(want)-1) + }) + } +} + +func TestX86VariableShiftGoRejectedForms(t *testing.T) { + for _, arch := range []string{"386", "amd64"} { + for _, op := range x86VariableShiftTestOps { + lines := []string{op + " X0,Y1,Y2", op + " X0,X1,(AX)", op + ".Z X0,X1,X2", op + ".SAE X0,X1,X2", op + " X0,X1,K0,X2", op + ".BCST X0,X1,X2", op + ".Z.BCST (AX),X1,K1,X2", op + " $1,X1,X2"} + if strings.HasSuffix(op, "W") { + lines = append(lines, op+".BCST (AX),X1,X2") + } + if arch == "386" { + lines = append(lines, op+" Z8,Z0,Z1", op+" X0,X1,K1,X2") + } + for _, line := range lines { + // Go 1.20 emitted reserved zeroing/K0 for maskless .Z. Preserve + // the strict named/raw rejection even for that historical encoder. + { + dir := t.TempDir() + path := filepath.Join(dir, "invalid.s") + if err := os.WriteFile(path, []byte("TEXT bad(SB),4,$0-0\n"+line+"\nRET\n"), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-o", filepath.Join(dir, "invalid.o"), path) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+arch) + out, err := cmd.CombinedOutput() + historicalZero := strings.HasPrefix(runtime.Version(), "go1.20") && strings.Contains(line, ".Z X0") + if historicalZero && err != nil || !historicalZero && err == nil { + t.Fatalf("Go %s unexpected acceptance for %s (historical zeroing=%v): %v %s", arch, line, historicalZero, err, out) + } + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + assertX86PerLaneVariableShiftRejected(t, arch, triple, line) + } + } + } +} + +func TestX86RawVariableShiftEncodingAxes(t *testing.T) { + count := 0 + for _, op := range x86VariableShiftTestOps { + bits := map[byte]int{'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + opcode := map[string]int{"VPSLLVW": 0x12, "VPSLLVD": 0x47, "VPSLLVQ": 0x47, "VPSRLVW": 0x10, "VPSRLVD": 0x45, "VPSRLVQ": 0x45, "VPSRAVW": 0x11, "VPSRAVD": 0x46, "VPSRAVQ": 0x46}[op] + for _, evex := range []bool{false, true} { + if !evex && (bits == 16 || op == "VPSRAVQ") { + continue + } + limit, widths := 16, 2 + if evex { + limit, widths = 32, 3 + } + for width := 0; width < widths; width++ { + for src1 := 0; src1 < limit; src1++ { + for src2 := 0; src2 < limit; src2++ { + for dst := 0; dst < limit; dst++ { + mask := 0 + zero := false + if evex { + mask = (dst + src1 + src2) % 8 + zero = mask != 0 && src2&1 != 0 + } + code := encodeRawScalarMove(evex, 1, 0, src2, src1, dst, mask, zero) + code[len(code)-2] = byte(opcode) + code[1] = code[1]&0xf0 | 2 + code[2] &^= 0x80 + if bits != 32 { + code[2] |= 0x80 + } + if evex { + code[3] |= byte(width) << 5 + } else { + code[2] |= byte(width) << 2 + } + got, size, matched, err := decodedX86PerLaneVariableShiftInstruction(code, 64) + prefix := []string{"X", "Y", "Z"}[width] + wantOp := Op(op) + if zero { + wantOp += ".Z" + } + args := []Operand{{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src2))}, {Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, src1))}} + if mask != 0 { + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("K%d", mask))}) + } + args = append(args, Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", prefix, dst))}) + if err != nil || !matched || size != len(code) || got.Op != wantOp || !reflect.DeepEqual(got.Args, args) { + t.Fatalf("decode %x: %+v %d %v %v; want %s %+v", code, got, size, matched, err, wantOp, args) + } + _, _, matched, err = decodedX86PerLaneVariableShiftInstruction(code, 32) + if !matched || (err != nil) != (src1 >= 8 || src2 >= 8 || dst >= 8) { + t.Fatalf("386 decode %x matched=%v err=%v", code, matched, err) + } + count++ + } + } + } + } + } + } + if count != 925696 { + t.Fatalf("covered %d encodings, want 925696", count) + } +} + +func TestX86RawVariableShiftInvalidAndMemoryForms(t *testing.T) { + valid := []byte{0x62, 0xf2, 0xf5, 0x0b, 0x47, 0xe0} + for name, mutate := range map[string]func([]byte){"fixed": func(b []byte) { b[2] &^= 4 }, "length": func(b []byte) { b[3] |= 0x60 }, "zero-mask": func(b []byte) { b[3] = 0x88 }, "broadcast-register": func(b []byte) { b[3] |= 0x10 }, "prefix": func(b []byte) { b[2] &^= 3 }} { + b := append([]byte(nil), valid...) + mutate(b) + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted %s %x", name, b) + } + } + for _, b := range [][]byte{valid[:5], {0x62, 0xf2, 0xf5, 0x0b, 0x47, 0x04}, {0x62, 0xf2, 0xf5, 0x0b, 0x47, 0x40}, append([]byte{0x67}, valid...), {0x62, 0xf2, 0xf5, 0x1b, 0x12, 0x00}, {0x62, 0xf2, 0x75, 0x0b, 0x12, 0xc0}, {0xc4, 0xe2, 0xf9, 0x46, 0xc0}, {0xc4, 0xe2, 0xf9, 0x12, 0xc0}, {0xc4, 0xe2, 0xf9, 0x47, 0x05, 0, 0, 0, 0}} { + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64); !matched || err == nil { + t.Fatalf("accepted invalid %x", b) + } + } + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(valid, 16); !matched || err == nil { + t.Fatal("accepted mode16") + } + for _, b := range [][]byte{nil, {0x62}, {0x90}, {0xc4, 0xe1, 0xf9, 0x47, 0xc0}, {0xc4, 0xe2, 0xf9, 0x48, 0xc0}} { + if _, _, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64); matched || err != nil { + t.Fatalf("claimed unrelated %x", b) + } + } + for _, segment := range []struct { + prefix byte + reg Reg + }{{0x64, FS}, {0x65, GS}} { + for _, broadcast := range []bool{false, true} { + b := []byte{segment.prefix, 0x62, 0x92, 0xf5, 0xcb, 0x47, 0x64, 0x88, 0xfe} + offset := int64(-128) + op := Op("VPSLLVQ.Z") + if broadcast { + b[4] |= 0x10 + offset = -16 + op = "VPSLLVQ.BCST.Z" + } + got, size, matched, err := decodedX86PerLaneVariableShiftInstruction(b, 64) + want := MemRef{Segment: segment.reg, Base: "R8", Index: "R9", Scale: 4, Off: offset} + if err != nil || !matched || size != len(b) || got.Op != op || !reflect.DeepEqual(got.Args[0].Mem, want) { + t.Fatalf("decode %x=%+v: %v", b, got, err) + } + } + } +} + +func TestX86RawVariableShift386MaskedObjects(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT shifts(SB),4,$0-0\n") + for _, op := range x86VariableShiftTestOps { + for _, width := range []string{"X", "Y", "Z"} { + for _, input := range []string{width + "0", "(AX)"} { + for _, suffix := range []string{"", ".Z"} { + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s%s %s,%s1,K7,%s2", op, suffix, input, width, width), true) + } + } + if !strings.HasSuffix(op, "W") { + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s.BCST.Z (AX),%s1,K7,%s2", op, width, width), true) + } + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{"i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + ir, err := Translate(file, Options{Goarch: "386", TargetTriple: triple, Sigs: map[string]FuncSig{"shifts": {Name: "shifts", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "masked-shifts-386.ll", "masked-shifts-386.o", ir) + } +} + +func TestX86VariableShiftMemoryAndViewsRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + cross := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !cross { + t.Skip("execution requires amd64 or Rosetta; five-target object tests remain required") + } + var source, decl strings.Builder + checks := map[string]*strings.Builder{"views": {}, "memory": {}} + sigs := map[string]FuncSig{} + index := 0 + for _, op := range x86VariableShiftTestOps { + bits := map[byte]int{'W': 16, 'D': 32, 'Q': 64}[op[len(op)-1]] + operation := 0 + if strings.HasPrefix(op, "VPSRL") { + operation = 1 + } + if strings.HasPrefix(op, "VPSRA") { + operation = 2 + } + for w, width := range []string{"X", "Y", "Z"} { + for mode := 0; mode < 5; mode++ { // reg/mem unmasked, reg/mem masked, broadcast masked. + if mode == 4 && bits == 16 { + continue + } + for _, zero := range []bool{false, true} { + if zero && mode < 2 { + continue + } + for _, raw := range []bool{false, true} { + name := fmt.Sprintf("shift%d", index) + index++ + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-32\nMOVQ data+0(FP),AX\nMOVQ counts+8(FP),BX\nMOVQ out+16(FP),DI\nVMOVUPS (AX),Z0\n", name) + input := "(BX)" + if mode == 0 || mode == 2 { + source.WriteString("VMOVUPS (BX),Z1\n") + input = width + "1" + } else if raw { + source.WriteString("VMOVUPS (AX),Z1\n") + } + suffix, mask := "", "" + if mode == 4 { + suffix = ".BCST" + } + if zero { + suffix += ".Z" + } + if mode >= 2 { + source.WriteString("KMOVQ mask+24(FP),K1\n") + mask = "K1," + } + dst := 0 + if raw { + dst = 1 + } + emitX86GoEncodedTestInstruction(t, &source, fmt.Sprintf("%s%s %s,%s0,%s%s%d", op, suffix, input, width, mask, width, dst), raw) + fmt.Fprintf(&source, "VMOVUPS Z%d,(DI)\nVMOVUPS Y%d,64(DI)\nVMOVUPS X%d,96(DI)\nRET\n", dst, dst, dst) + sigs[name] = FuncSig{Name: name, Args: []LLVMType{Ptr, Ptr, Ptr, I64}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, {Offset: 16, Type: Ptr, Index: 2, Field: -1}, {Offset: 24, Type: I64, Index: 3, Field: -1}}}} + fmt.Fprintf(&decl, "extern void %s(const void*,const void*,void*,uint64_t);\n", name) + kind := "views" + if mode >= 2 { + kind = "memory" + } + z := 0 + if zero { + z = 1 + } + oldCounts := 0 + if raw && (mode == 0 || mode == 2) { + oldCounts = 1 + } + fmt.Fprintf(checks[kind], "if(check(%s,%d,%d,%d,%d,%d,%d)){fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, 16< +#include +#include +` + x86TestGuardPagesC + decl.String() + ` +static uint64_t lane(const uint8_t *p,int bytes){uint64_t v=0;memcpy(&v,p,bytes);return v;} +static uint64_t shifted(uint64_t value,uint64_t count,int bits,int op){ + uint64_t limit=bits==64?UINT64_MAX:(UINT64_C(1)<=(uint64_t)bits)return op==2 && (value&sign)?limit:0; + if(op==0)return (value<>count; + if(op==2 && (value&sign) && count)result|=limit^(limit>>count); + return result; +} +static int check(void(*fn)(const void*,const void*,void*,uint64_t),int width,int bytes,int op,int mode,int zero,int oldCounts){ + for(int n=0;n<96;n++){ + uint8_t ab[65],bb[65],out[112],want[112]={0};uint8_t *a=ab+1,*b=bb+1; + uint64_t counts[]={0,1,(uint64_t)bytes*8-1,(uint64_t)bytes*8,(uint64_t)bytes*8+1,UINT64_MAX,UINT64_C(1)<<(bytes*8-1)}; + for(int i=0;i<64;i++)a[i]=(uint8_t)(i*31+n*17); + for(int i=0;i<64/bytes;i++){uint64_t count=counts[(n+i)%7];memcpy(b+i*bytes,&count,bytes);} + int lanes=width/bytes;uint64_t mask=n==0?0:n==1?UINT64_C(1)<<63:n==2?UINT64_MAX:UINT64_C(0x5a5a01239876cdef)>>(n%32); + uint64_t active=mask&((UINT64_C(1)<>i)&1)?shifted(value,count,bytes*8,op):zero?0:oldCounts?lane(b+i*bytes,bytes):value; + memcpy(want+i*bytes,&result,bytes); + } + memcpy(want+64,want,32);memcpy(want+96,want,16); + fn(a,mode>=3 && !active?NULL:b,out,mask); + if(memcmp(out,want,112)){fprintf(stderr,"width=%d bytes=%d op=%d mode=%d zero=%d n=%d\n",width,bytes,op,mode,zero,n);return 1;} + } + if(mode>=3){ + int lanes=width/bytes; + for(int enabled=1;enabled<=lanes;enabled++){ + uint8_t a[64],out[112],want[112]={0};memset(a,0xff,64); + int readable=mode==4?bytes:enabled*bytes;uint8_t *b=guard+page-readable; + memset(b,0,(size_t)readable); + uint64_t mask=(UINT64_C(1)< 1 { + return fail("invalid 66 prefix, reserved vvvv or vector length") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if mode == 32 && (p.r != 0 || p.x != 0 || p.b != 0) { + return fail("extended register in 32-bit mode") + } + width := "X" + if p.vectorLength == 1 { + width = "Y" + } + source, size, err := decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, width) + if err != nil { + return Instr{}, 0, true, err + } + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("%s%d", width, int(code[p.modRM]>>3&7)+p.r*8))} + return Instr{ + Op: op, + Args: []Operand{source, destination}, + Raw: fmt.Sprintf("%s %s, %s", op, source.String(), destination.String()), + }, p.modRM + size, true, nil +} diff --git a/amd64_raw_vecflags_test.go b/amd64_raw_vecflags_test.go new file mode 100644 index 000000000..d6bd96cb0 --- /dev/null +++ b/amd64_raw_vecflags_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func TestX86RawVectorTestCompleteGo127EncodingFamily(t *testing.T) { + if len(x86RawVectorTestOpcodes) != len(amd64VectorTestSpecs)-1 { + t.Fatalf("raw VEX vector-test opcode table has %d entries, named grammar has %d VEX entries", len(x86RawVectorTestOpcodes), len(amd64VectorTestSpecs)-1) + } + for _, op := range x86RawVectorTestOpcodes { + if spec, ok := amd64VectorTestSpecs[op]; !ok || !spec.vector { + t.Fatalf("raw opcode %s has no matching named VEX grammar", op) + } + } + + type form struct { + name string + op Op + code []byte + src Operand + dst Reg + } + var forms []form + for _, spec := range []struct { + name Op + opcode byte + legacy bool + }{ + {"PTEST", 0x17, true}, + {"VPTEST", 0x17, false}, + {"VTESTPS", 0x0e, false}, + {"VTESTPD", 0x0f, false}, + } { + for _, width := range []struct { + name string + vex3 byte + }{ + {"X", 0x79}, + {"Y", 0x7d}, + } { + if spec.legacy && width.name == "Y" { + continue + } + for _, source := range []struct { + name string + modRM byte + value Operand + }{ + {"register", 0xc1, Operand{Kind: OpReg, Reg: Reg(width.name + "1")}}, + {"memory", 0x08, Operand{Kind: OpMem, Mem: MemRef{Base: AX}}}, + } { + code := []byte{0xc4, 0xe2, width.vex3, spec.opcode, source.modRM} + if spec.legacy { + code = []byte{0x66, 0x0f, 0x38, spec.opcode, source.modRM} + } + dest := Reg(width.name + "0") + if source.name == "memory" { + dest = Reg(width.name + "1") + } + forms = append(forms, form{ + name: string(spec.name) + "/" + width.name + "/" + source.name, + op: spec.name, code: code, src: source.value, dst: dest, + }) + } + } + } + for _, goarch := range []string{"386", "amd64"} { + for _, form := range forms { + t.Run(goarch+"/"+form.name, func(t *testing.T) { + fn := Func{} + for _, value := range form.code { + fn.Instrs = append(fn.Instrs, Instr{Op: OpBYTE, Args: []Operand{{Kind: OpImm, Imm: int64(value)}}}) + } + got, err := decodeX86RawDirectives(fn, goarch) + if err != nil { + t.Fatal(err) + } + if len(got.Instrs) != 1 || got.Instrs[0].Op != form.op || len(got.Instrs[0].Args) != 2 { + t.Fatalf("decoded %#x as %#v, want %s", form.code, got.Instrs, form.op) + } + ins := got.Instrs[0] + if ins.Args[0].Kind != form.src.Kind || ins.Args[0].Reg != form.src.Reg || ins.Args[0].Mem != form.src.Mem || ins.Args[1].Reg != form.dst { + t.Fatalf("decoded %#x operands as %#v, want %#v, %s", form.code, ins.Args, form.src, form.dst) + } + }) + } + } +} + +func TestX86RawVectorTestRejectsReservedEncodings(t *testing.T) { + for _, tc := range []struct { + name string + code []byte + mode int + }{ + {"missing-66", []byte{0xc4, 0xe2, 0x78, 0x17, 0xc1}, 64}, + {"nonzero-vvvv", []byte{0xc4, 0xe2, 0x71, 0x17, 0xc1}, 64}, + {"address-override", []byte{0x67, 0xc4, 0xe2, 0x79, 0x17, 0x08}, 64}, + {"missing-modrm", []byte{0xc4, 0xe2, 0x79, 0x17}, 64}, + {"extended-386", []byte{0xc4, 0x62, 0x79, 0x17, 0xc1}, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86VEXVectorTestInstruction(tc.code, tc.mode); !recognized || err == nil { + t.Fatalf("reserved encoding %#x: recognized=%v error=%v", tc.code, recognized, err) + } + }) + } + for _, tc := range []struct { + name string + code []byte + want Operand + }{ + {"W-ignored", []byte{0xc4, 0xe2, 0xf9, 0x17, 0xc1}, Operand{Kind: OpReg, Reg: "X1"}}, + {"SIB-memory", []byte{0xc4, 0xe2, 0x7d, 0x0f, 0x44, 0x88, 0x20}, Operand{Kind: OpMem, Mem: MemRef{Base: AX, Index: CX, Scale: 4, Off: 32}}}, + } { + t.Run(tc.name, func(t *testing.T) { + got, length, recognized, err := decodedX86VEXVectorTestInstruction(tc.code, 64) + if !recognized || err != nil || length != len(tc.code) { + t.Fatalf("encoding %#x: instruction=%+v length=%d recognized=%v error=%v", tc.code, got, length, recognized, err) + } + if !reflect.DeepEqual(got.Args[0], tc.want) { + t.Fatalf("source = %+v, want %+v", got.Args[0], tc.want) + } + }) + } +} + +func TestX86RawVectorTestCompilesEveryForm(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawvectortest(SB),NOSPLIT,$0-0\n") + for _, spec := range []struct { + opcode byte + legacy bool + }{ + {0x17, true}, {0x17, false}, {0x0e, false}, {0x0f, false}, + } { + for _, width := range []byte{0x79, 0x7d} { + if spec.legacy && width == 0x7d { + continue + } + for _, modRM := range []byte{0xc1, 0x08} { + code := []byte{0xc4, 0xe2, width, spec.opcode, modRM} + if spec.legacy { + code = []byte{0x66, 0x0f, 0x38, spec.opcode, modRM} + } + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%d\n", value) + } + source.WriteString("\tNOP\n") + } + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawvectortest": {Name: "rawvectortest", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-vector-test.ll", "raw-vector-test.o", ir) + }) + } +} diff --git a/amd64_raw_vector_float_compare_literal_test.go b/amd64_raw_vector_float_compare_literal_test.go new file mode 100644 index 000000000..8c4f4a4d1 --- /dev/null +++ b/amd64_raw_vector_float_compare_literal_test.go @@ -0,0 +1,177 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVectorFloatCompareLiteral(pp byte, encoding string, width int, broadcast bool) []byte { + var code []byte + switch encoding { + case "vex2": + code = encodeX86VEX2VectorFloatCompare(pp, width == 32, 0, 0, 0, 12) + case "vex3": + code = encodeX86VEXVectorFloatCompare(pp, width == 32, 0, 0, 0, 12) + case "evex": + code = encodeX86EVEXVectorFloatCompare( + pp, byte(map[int]int{16: 0, 32: 1, 64: 2}[width]), + broadcast, 1, 1, 0, 0, 12, + ) + } + modRMIndex := len(code) - 2 + code[modRMIndex] = 0x05 + immediate := code[len(code)-1] + code = append(code[:modRMIndex+1], 1, 0, 0, 0, immediate, 0xc3) + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + for index := 0; index < dataWidth; index++ { + code = append(code, byte(index*41+15)) + } + return code +} + +func TestDecodeX86RawVectorFloatCompareRIPDataCompleteFamily(t *testing.T) { + for pp := byte(0); pp < 4; pp++ { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + widths := []int{16, 32} + if encoding == "evex" { + widths = append(widths, 64) + } + if pp >= 2 { + widths = []int{16} + } + for _, width := range widths { + broadcasts := []bool{false} + if encoding == "evex" && pp < 2 { + broadcasts = append(broadcasts, true) + } + for _, broadcast := range broadcasts { + name := fmt.Sprintf("%s/%s/%d/bcst%t", vectorFloatCompareRawOp(pp), encoding, width, broadcast) + t.Run(name, func(t *testing.T) { + code := x86RawVectorFloatCompareLiteral(pp, encoding, width, broadcast) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := vectorFloatCompareRawOp(pp) + if broadcast { + want += ".BCST" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpImm || decoded[0].Args[0].Imm != 12 || + decoded[0].Args[1].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } + } +} + +func TestDecodeX86RawVectorFloatCompareRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawVectorFloatCompareLiteral(1, "evex", 64, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + "EVEX zeroing": func(code []byte) []byte { + code[3] |= 0x80 + return code + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVectorFloatCompareRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + index := 0 + for pp := byte(0); pp < 4; pp++ { + for _, encoding := range []string{"vex2", "vex3", "evex"} { + width := 16 + if pp < 2 { + width = 32 + if encoding == "evex" { + width = 64 + } + } + broadcast := encoding == "evex" && pp < 2 + code := x86RawVectorFloatCompareLiteral(pp, encoding, width, broadcast) + name := fmt.Sprintf("vectorCompareLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + dataWidth := width + if pp >= 2 || broadcast { + dataWidth = 4 + if pp == 1 || pp == 3 { + dataWidth = 8 + } + } + pools = append(pools, code[len(code)-dataWidth:]) + index++ + } + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "vector-compare-literal.ll", "vector-compare-literal.o", ir) + }) + } +} diff --git a/amd64_raw_vector_float_compare_test.go b/amd64_raw_vector_float_compare_test.go new file mode 100644 index 000000000..34a374e92 --- /dev/null +++ b/amd64_raw_vector_float_compare_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXVectorFloatCompare(pp byte, width256 bool, destination, secondSource, firstSource int, immediate byte) []byte { + l := byte(0) + if width256 { + l = 1 + } + vex0 := byte((1-(destination>>3)&1)<<7 | 1<<6 | (1-(firstSource>>3)&1)<<5 | 1) + vex1 := byte((^secondSource&15)<<3 | int(l)<<2 | int(pp)) + modRM := byte(0xc0 | (destination&7)<<3 | firstSource&7) + return []byte{0xc4, vex0, vex1, 0xc2, modRM, immediate} +} + +func encodeX86VEX2VectorFloatCompare(pp byte, width256 bool, destination, secondSource, firstSource int, immediate byte) []byte { + l := byte(0) + if width256 { + l = 1 + } + vex1 := byte((1-(destination>>3)&1)<<7 | (^secondSource&15)<<3 | int(l)<<2 | int(pp)) + modRM := byte(0xc0 | (destination&7)<<3 | firstSource&7) + return []byte{0xc5, vex1, 0xc2, modRM, immediate} +} + +func encodeX86EVEXVectorFloatCompare(pp, vectorBits byte, broadcastOrSAE bool, writeMask, destinationMask, secondSource, firstSource int, immediate byte) []byte { + p0 := byte(1<<7 | (1-(firstSource>>4)&1)<<6 | (1-(firstSource>>3)&1)<<5 | 1<<4 | 1) + p1 := byte((^secondSource&15)<<3 | 1<<2 | int(pp)) + if pp == 1 || pp == 3 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^secondSource>>4&1)<<3 | writeMask) + if broadcastOrSAE { + p2 |= 0x10 + } + modRM := byte(0xc0 | (destinationMask&7)<<3 | firstSource&7) + return []byte{0x62, p0, p1, p2, 0xc2, modRM, immediate} +} + +func vectorFloatCompareRawOp(pp byte) Op { + return [...]Op{"VCMPPS", "VCMPPD", "VCMPSS", "VCMPSD"}[pp] +} + +func TestTranslateRawEVEXVectorFloatCompareWeaviateRegression(t *testing.T) { + const source = `TEXT rawVectorFloatCompare(SB), $0-0 + LONG $0x483cf162; WORD $0x06c2; BYTE $0x0c + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawVectorFloatCompare": {Name: "rawVectorFloatCompare", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, `asm sideeffect "vcmpps $$12,`) { + t.Fatalf("raw EVEX VCMPPS lowering omitted floating compare:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vector-float-compare.ll", "amd64-raw-vector-float-compare.o", ir) +} + +func TestDecodedX86VEXVectorFloatCompareCompleteRegisterFamily(t *testing.T) { + count := 0 + for pp := byte(0); pp < 4; pp++ { + widths := []bool{false, true} + if pp >= 2 { + widths = []bool{false} + } + for _, width256 := range widths { + prefix := "X" + if width256 { + prefix = "Y" + } + for destination := 0; destination < 16; destination++ { + for secondSource := 0; secondSource < 16; secondSource++ { + for firstSource := 0; firstSource < 16; firstSource++ { + code := encodeX86VEXVectorFloatCompare(pp, width256, destination, secondSource, firstSource, 0xa5) + got, length, ok, err := decodedX86VectorFloatCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{"$165", fmt.Sprintf("%s%d", prefix, firstSource), fmt.Sprintf("%s%d", prefix, secondSource), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != vectorFloatCompareRawOp(pp) || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, vectorFloatCompareRawOp(pp), strings.Join(wantArgs, ", ")) + } + count++ + } + for firstSource := 0; firstSource < 8; firstSource++ { + code := encodeX86VEX2VectorFloatCompare(pp, width256, destination, secondSource, firstSource, 0xa5) + got, length, ok, err := decodedX86VectorFloatCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{"$165", fmt.Sprintf("%s%d", prefix, firstSource), fmt.Sprintf("%s%d", prefix, secondSource), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != vectorFloatCompareRawOp(pp) || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, vectorFloatCompareRawOp(pp), strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 36864 { + t.Fatalf("covered %d VEX vector floating compare encodings, want 36864", count) + } +} + +func TestDecodedX86EVEXVectorFloatCompareCompleteRegisterFamily(t *testing.T) { + count := 0 + for pp := byte(0); pp < 4; pp++ { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + if pp >= 2 { + prefix = "X" + } + for destinationMask := 0; destinationMask < 8; destinationMask++ { + for secondSource := 0; secondSource < 32; secondSource++ { + for firstSource := 0; firstSource < 32; firstSource++ { + code := encodeX86EVEXVectorFloatCompare(pp, vectorBits, false, 1, destinationMask, secondSource, firstSource, 0x5a) + got, length, ok, err := decodedX86VectorFloatCompareInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{"$90", fmt.Sprintf("%s%d", prefix, firstSource), fmt.Sprintf("%s%d", prefix, secondSource), "K1", fmt.Sprintf("K%d", destinationMask)} + if got.Op != vectorFloatCompareRawOp(pp) || len(got.Args) != 5 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] || got.Args[4].String() != wantArgs[4] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, vectorFloatCompareRawOp(pp), strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 98304 { + t.Fatalf("covered %d EVEX vector floating compare encodings, want 98304", count) + } +} + +func TestDecodedX86EVEXVectorFloatCompareModifiersMemoryAndInvalid(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "weaviate memory", code: []byte{0x62, 0xf1, 0x3c, 0x48, 0xc2, 0x06, 0x0c}, op: "VCMPPS", args: []string{"$12", "0(SI)", "Z8", "K0"}}, + {name: "packed broadcast", code: []byte{0x62, 0xf1, 0xfd, 0x59, 0xc2, 0x48, 0x02, 0x7f}, op: "VCMPPD.BCST", args: []string{"$127", "16(AX)", "Z0", "K1", "K1"}}, + {name: "packed sae", code: encodeX86EVEXVectorFloatCompare(0, 2, true, 2, 7, 20, 21, 31), op: "VCMPPS.SAE", args: []string{"$31", "Z21", "Z20", "K2", "K7"}}, + {name: "scalar lig", code: encodeX86EVEXVectorFloatCompare(3, 1, false, 0, 4, 3, 21, 27), op: "VCMPSD", args: []string{"$27", "X21", "X3", "K4"}}, + {name: "scalar sae", code: encodeX86EVEXVectorFloatCompare(2, 3, true, 3, 5, 1, 11, 47), op: "VCMPSS.SAE", args: []string{"$47", "X11", "X1", "K3", "K5"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VectorFloatCompareInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + scalarBroadcast := encodeX86EVEXVectorFloatCompare(2, 0, true, 0, 0, 0, 0, 0) + scalarBroadcast[len(scalarBroadcast)-2] = 0 + invalid := [][]byte{ + encodeX86VEXVectorFloatCompare(2, true, 0, 0, 0, 0), + encodeX86EVEXVectorFloatCompare(0, 3, false, 0, 0, 0, 0, 0), + scalarBroadcast, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86VectorFloatCompareInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } +} diff --git a/amd64_raw_vector_literal_test.go b/amd64_raw_vector_literal_test.go new file mode 100644 index 000000000..62098b995 --- /dev/null +++ b/amd64_raw_vector_literal_test.go @@ -0,0 +1,346 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawPackedMoveLiteral(evex bool, width int) []byte { + var code []byte + if evex { + length := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) + code = []byte{0x62, 0xf1, 0x7c, length<<5 | 0x08, 0x10, 0x05} + } else { + p1 := byte(0x7e) + if width == 16 { + p1 = 0x7a + } + code = []byte{0xc5, p1, 0x6f, 0x05} + } + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*7+3)) + } + return code +} + +func TestDecodeX86RawPackedMoveRIPDataCompleteWidths(t *testing.T) { + for _, test := range []struct { + evex bool + width int + }{ + {false, 16}, {false, 32}, + {true, 16}, {true, 32}, {true, 64}, + } { + name := fmt.Sprintf("evex%t/width%d", test.evex, test.width) + t.Run(name, func(t *testing.T) { + code := x86RawPackedMoveLiteral(test.evex, test.width) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || !decoded[0].x86RIPLiteral || + len(decoded[0].Args) < 2 || decoded[0].Args[0].Kind != OpSym || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want source-local vector constant", code, decoded) + } + }) + } +} + +func TestTranslateX86RawPackedMoveRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + for index, test := range []struct { + evex bool + width int + }{ + {false, 16}, {false, 32}, + {true, 16}, {true, 32}, {true, 64}, + } { + name := fmt.Sprintf("vectorLiteral%d", index) + code := x86RawPackedMoveLiteral(test.evex, test.width) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(sigs) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(sigs)) + } + for index, datum := range normalized.Data { + width := []int{16, 32, 16, 32, 64}[index] + code := x86RawPackedMoveLiteral(index >= 2, width) + want := code[len(code)-width:] + if datum.Width != int64(width) || !bytes.Equal(datum.Payload, want) { + t.Fatalf("constant %d = %x width %d, want %x", index, datum.Payload, datum.Width, want) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "vector-literal.ll", "vector-literal.o", ir) + }) + } +} + +func TestDecodeX86RawPackedMoveRIPDataRejectsUnsafeSources(t *testing.T) { + // The same opcode byte is used by scalar VMOVSS/SD; this packed-move + // decoder must leave those encodings to the scalar grammar. + for _, code := range [][]byte{ + {0xc5, 0xfa, 0x10, 0x05, 1, 0, 0, 0}, + {0xc5, 0xfb, 0x10, 0x05, 1, 0, 0, 0}, + } { + if _, _, _, matched, _ := decodeX86RawPackedMoveRIPData(code, 0, 64); matched { + t.Fatalf("packed move claimed scalar encoding %x", code) + } + } + for _, test := range []struct { + name string + edit func([]byte) []byte + }{ + {name: "instruction_overlap", edit: func(code []byte) []byte { + copy(code[6:10], []byte{0xfa, 0xff, 0xff, 0xff}) + return code + }}, + {name: "missing_pool", edit: func(code []byte) []byte { + code[6] = 0x7f + return code + }}, + {name: "truncated_pool", edit: func(code []byte) []byte { + code[6] = 2 + return code + }}, + {name: "segment_override", edit: func(code []byte) []byte { + return append([]byte{0x64}, code...) + }}, + {name: "address_override", edit: func(code []byte) []byte { + return append([]byte{0x67}, code...) + }}, + } { + t.Run(test.name, func(t *testing.T) { + code := test.edit(x86RawPackedMoveLiteral(true, 64)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, test.name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP vector source: %x", code) + } + }) + } +} + +func TestTranslateX86RawPackedMoveRIPDataRejectsAddressObservedText(t *testing.T) { + code := x86RawPackedMoveLiteral(true, 64) + var source strings.Builder + source.WriteString("TEXT address(SB),$0-0\n\tLEAQ raw+1(SB), AX\n\tRET\n") + source.WriteString("TEXT raw(SB),$0-0\n\tMOVQ AX, AX\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + _, err = Translate(file, Options{ + Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{ + "address": {Name: "address", Ret: Void}, + "raw": {Name: "raw", Ret: Void}, + }, + }) + if err == nil || !strings.Contains(err.Error(), "address-observed RIP-relative raw literal") { + t.Fatalf("address-observed raw TEXT was silently folded: %v", err) + } +} + +func x86RawEVEXPackedIntegerMoveLiteral(pp byte, wide bool, width, mask int, zeroing bool) []byte { + p1 := byte(0x7c) | pp + if wide { + p1 |= 0x80 + } + p2 := byte(map[int]int{16: 0, 32: 1, 64: 2}[width]<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + code := []byte{0x62, 0xf1, p1, p2, 0x6f, 0x05, 1, 0, 0, 0, 0xc3} + for index := 0; index < width; index++ { + code = append(code, byte(index*5+1)) + } + return code +} + +func TestDecodeX86RawEVEXPackedIntegerMoveRIPDataCompleteFamily(t *testing.T) { + for _, form := range []struct { + pp byte + wide bool + op Op + }{ + {1, false, "VMOVDQA32"}, {1, true, "VMOVDQA64"}, + {2, false, "VMOVDQU32"}, {2, true, "VMOVDQU64"}, + {3, false, "VMOVDQU8"}, {3, true, "VMOVDQU16"}, + } { + for _, width := range []int{16, 32, 64} { + for _, mask := range []int{0, 3, 7} { + zeroing := mask == 7 + name := fmt.Sprintf("%s/%d/mask%d", form.op, width, mask) + t.Run(name, func(t *testing.T) { + code := x86RawEVEXPackedIntegerMoveLiteral(form.pp, form.wide, width, mask, zeroing) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + op := form.op + if zeroing { + op += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != op || + len(decoded[0].Args) < 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data load", code, decoded, op) + } + }) + } + } + } +} + +func TestTranslateX86RawEVEXPackedIntegerMoveRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + forms := []struct { + pp byte + wide bool + }{ + {1, false}, {1, true}, {2, false}, + {2, true}, {3, false}, {3, true}, + } + for index, form := range forms { + name := fmt.Sprintf("evexIntegerLiteral%d", index) + code := x86RawEVEXPackedIntegerMoveLiteral(form.pp, form.wide, 64, 7, true) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "evex-integer-literal.ll", "evex-integer-literal.o", ir) + }) + } +} + +func TestDecodeX86RawEVEXPackedIntegerMoveRIPDataRejectsUnsafeSource(t *testing.T) { + for _, displacement := range []byte{0x7f, 2} { + code := x86RawEVEXPackedIntegerMoveLiteral(2, false, 64, 0, false) + code[6] = displacement + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "unsafe EVEX integer source", map[string]bool{}); err == nil { + t.Fatalf("accepted out-of-group EVEX integer source %x", code) + } + } +} + +func x86RawVBROADCASTI128Literal() []byte { + code := []byte{0xc4, 0xe2, 0x7d, 0x5a, 0x05, 1, 0, 0, 0, 0xc3} + for index := 0; index < 16; index++ { + code = append(code, byte(index*11+2)) + } + return code +} + +func TestDecodeX86RawVBROADCASTI128RIPData(t *testing.T) { + code := x86RawVBROADCASTI128Literal() + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VBROADCASTI128 literal", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VBROADCASTI128" || + len(decoded[0].Args) != 2 || decoded[0].Args[0].Kind != OpSym || + !decoded[0].x86RIPLiteral || decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want VBROADCASTI128 source-local data", code, decoded) + } +} + +func TestTranslateX86RawVBROADCASTI128RIPDataObjects(t *testing.T) { + code := x86RawVBROADCASTI128Literal() + var source strings.Builder + source.WriteString("TEXT blockLiteral(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != 1 || !bytes.Equal(normalized.Data[0].Payload, code[len(code)-16:]) { + t.Fatalf("VBROADCASTI128 constant = %#v, want the exact source bytes", normalized.Data) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"blockLiteral": {Name: "blockLiteral", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "block-literal.ll", "block-literal.o", ir) + }) + } +} + +func TestDecodeX86RawVBROADCASTI128RIPDataRejectsUnsafeSource(t *testing.T) { + for _, displacement := range []byte{3, 0x7f} { + code := x86RawVBROADCASTI128Literal() + code[5] = displacement + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "unsafe block broadcast", map[string]bool{}); err == nil { + t.Fatalf("accepted out-of-group VBROADCASTI128 source %x", code) + } + } +} diff --git a/amd64_raw_vector_zero.go b/amd64_raw_vector_zero.go new file mode 100644 index 000000000..25cc82dcb --- /dev/null +++ b/amd64_raw_vector_zero.go @@ -0,0 +1,48 @@ +package plan9asm + +import "fmt" + +// The pinned x/arch decoder calls both VEX.L variants VZEROUPPER and accepts +// reserved pp/vvvv values. Decode this complete zero-operand family before +// generic fallback, binding the actual bytes rather than its printed opcode. +func decodedX86VectorZeroInstruction(code []byte, mode int) (Instr, int, bool, error) { + i, forbidden := 0, false + for i < len(code) { + prefix := code[i] + switch { + case prefix == 0x26 || prefix == 0x2e || prefix == 0x36 || prefix == 0x3e || prefix == 0x64 || prefix == 0x65 || prefix == 0x67: + case prefix == 0x66 || prefix == 0xf0 || prefix == 0xf2 || prefix == 0xf3 || mode == 64 && prefix >= 0x40 && prefix <= 0x4f: + forbidden = true + default: + goto prefixEnd + } + i++ + } +prefixEnd: + if i >= len(code) || code[i] != 0xc4 && code[i] != 0xc5 { + return Instr{}, 0, false, nil + } + payload, opcode, length := i+1, i+2, i+3 + if code[i] == 0xc4 { + payload, opcode, length = i+2, i+3, i+4 + } + if opcode >= len(code) { + return Instr{}, 0, false, nil + } + if mode == 32 && code[i+1]&0xc0 != 0xc0 { + // These bytes are legacy LES/LDS, not a mode32 VEX prefix. + return Instr{}, 0, false, nil + } + if code[opcode] != 0x77 || code[i] == 0xc4 && code[i+1]&31 != 1 { + return Instr{}, 0, false, nil + } + if mode != 32 && mode != 64 || forbidden || length > 15 || code[payload]&3 != 0 || code[payload]&0x78 != 0x78 { + return Instr{}, 0, true, fmt.Errorf("reserved raw VEX vector-zero encoding %x in mode%d", code[:length], mode) + } + for op, spec := range x86VectorZeroSpecs { + if spec.vexLength == code[payload]>>2&1 { + return Instr{Op: op, Raw: string(op)}, length, true, nil + } + } + return Instr{}, 0, true, fmt.Errorf("unregistered raw VEX vector-zero length") +} diff --git a/amd64_raw_vex_binary_float_test.go b/amd64_raw_vex_binary_float_test.go new file mode 100644 index 000000000..e92f67b16 --- /dev/null +++ b/amd64_raw_vex_binary_float_test.go @@ -0,0 +1,368 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86VEXBinaryFloatTestOpcodes = map[byte]string{ + 0x58: "VADD", + 0x59: "VMUL", + 0x5c: "VSUB", + 0x5d: "VMIN", + 0x5e: "VDIV", + 0x5f: "VMAX", +} + +func encodeX86VEXBinaryFloat(opcode, pp byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func encodeX86EVEXBinaryFloat(opcode, pp, vectorBits byte, broadcastOrRounding, zeroing bool, mask, destination, source1, source2 int) []byte { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source2>>4)&1)<<6 | (1-(source2>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte((^source1&15)<<3 | 1<<2 | int(pp)) + if pp == 1 || pp == 3 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^source1>>4&1)<<3 | mask) + if broadcastOrRounding { + p2 |= 0x10 + } + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestTranslateRawVEXBinaryFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawBinary(SB), $0-0 + LONG $0x0e5cf2c5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawBinary": {Name: "rawBinary", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.experimental.constrained.fsub.f32(") || !strings.Contains(ir, "@__plan9asm_add_sub_special_f32(") { + t.Fatalf("raw VSUBSS lowering omitted scalar subtraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-binary-float.ll", "amd64-raw-vex-binary-float.o", ir) +} + +func TestTranslateRawScalarSqrtKelindarRegression(t *testing.T) { + const source = `TEXT rawScalarSqrt(SB), $0-0 + LONG $0xdc51dac5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: "amd64", + TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"rawScalarSqrt": {Name: "rawScalarSqrt", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "llvm.sqrt.f32") { + t.Fatalf("raw VSQRTSS omitted square root:\n%s", ir) + } +} + +func TestDecodedX86RawScalarSqrtCompleteGoRows(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + {"VEX single register", []byte{0xc5, 0xea, 0x51, 0xd9}, "VSQRTSS X1, X2, X3"}, + {"VEX double register", []byte{0xc5, 0xeb, 0x51, 0xd9}, "VSQRTSD X1, X2, X3"}, + {"VEX single memory", []byte{0xc5, 0xea, 0x51, 0x18}, "VSQRTSS 0(AX), X2, X3"}, + {"VEX double memory", []byte{0xc5, 0xeb, 0x51, 0x58, 0x10}, "VSQRTSD 16(AX), X2, X3"}, + {"EVEX single mask zero", []byte{0x62, 0xa1, 0x56, 0x81, 0x51, 0xf4}, "VSQRTSS.Z X20, X21, K1, X22"}, + {"EVEX double mask zero", []byte{0x62, 0xa1, 0xd7, 0x81, 0x51, 0xf4}, "VSQRTSD.Z X20, X21, K1, X22"}, + {"EVEX single rounding", []byte{0x62, 0xf1, 0x6e, 0x18, 0x51, 0xd9}, "VSQRTSS.RN_SAE X1, X2, X3"}, + {"EVEX double rounding", []byte{0x62, 0xf1, 0xef, 0x38, 0x51, 0xd9}, "VSQRTSD.RD_SAE X1, X2, X3"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, recognized, err := decodedX86VEXBinaryFloatInstruction(test.code, 64) + if err != nil || !recognized || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode %x: raw=%q length=%d recognized=%v err=%v; want %q", test.code, instruction.Raw, length, recognized, err, test.want) + } + }) + } +} + +func TestTranslateRawScalarSqrtLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + encodings := [][]byte{ + {0xc5, 0xea, 0x51, 0xd9}, + {0xc5, 0xeb, 0x51, 0x58, 0x10}, + {0x62, 0xf1, 0x6e, 0x18, 0x51, 0xd9}, + {0x62, 0xf1, 0xef, 0x38, 0x51, 0xd9}, + } + if target.goarch == "amd64" { + encodings = append(encodings, + []byte{0x62, 0xa1, 0x56, 0x81, 0x51, 0xf4}, + []byte{0x62, 0xa1, 0xd7, 0x81, 0x51, 0xf4}, + ) + } + var source strings.Builder + source.WriteString("TEXT rawScalarSqrtFamily(SB), $0-0\n") + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawScalarSqrtFamily": {Name: "rawScalarSqrtFamily", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-scalar-sqrt.ll", "raw-scalar-sqrt.o", ir) + }) + } +} + +func TestTranslateRawEVEXBinaryFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawBinaryEVEX(SB), $0-0 + LONG $0x48347162 + WORD $0x0e5c + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawBinaryEVEX": {Name: "rawBinaryEVEX", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.experimental.constrained.fsub.v16f32(") || !strings.Contains(ir, "@__plan9asm_add_sub_special_f32(") { + t.Fatalf("raw EVEX VSUBPS lowering omitted packed subtraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evex-binary-float.ll", "amd64-raw-evex-binary-float.o", ir) +} + +func TestDecodedX86VEXBinaryFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + suffixes := map[byte]string{0: "PS", 1: "PD", 2: "SS", 3: "SD"} + for opcode, stem := range x86VEXBinaryFloatTestOpcodes { + for pp, suffix := range suffixes { + scalar := pp >= 2 + for _, vex3 := range []bool{false, true} { + widths := []bool{false, true} + if scalar { + widths = []bool{false} + } + for _, width256 := range widths { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXBinaryFloat(opcode, pp, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + } + if count != 368640 { + t.Fatalf("covered %d VEX binary floating register encodings, want 368640", count) + } +} + +func TestDecodedX86EVEXBinaryFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + suffixes := map[byte]string{0: "PS", 1: "PD", 2: "SS", 3: "SD"} + for opcode, stem := range x86VEXBinaryFloatTestOpcodes { + for pp, suffix := range suffixes { + vectorLengths := []byte{0, 1, 2} + for _, vectorBits := range vectorLengths { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + if pp >= 2 { + prefix = "X" + } + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + code := encodeX86EVEXBinaryFloat(opcode, pp, vectorBits, false, false, 1, destination, source1, source2) + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + if count != 2359296 { + t.Fatalf("covered %d EVEX binary floating register encodings, want 2359296", count) + } +} + +func TestDecodedX86EVEXBinaryFloatModifiersAndMemory(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "packed zeroing", code: encodeX86EVEXBinaryFloat(0x58, 0, 1, false, true, 3, 29, 28, 21), op: "VADDPS.Z", args: []string{"Y21", "Y28", "K3", "Y29"}}, + {name: "packed broadcast zeroing", code: []byte{0x62, 0xf1, 0xed, 0xd9, 0x58, 0x08}, op: "VADDPD.BCST.Z", args: []string{"0(AX)", "Z2", "K1", "Z1"}}, + {name: "packed round nearest", code: []byte{0x62, 0xf1, 0xed, 0x19, 0x58, 0xcb}, op: "VADDPD.RN_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round down", code: []byte{0x62, 0xf1, 0xed, 0x39, 0x58, 0xcb}, op: "VADDPD.RD_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round up", code: []byte{0x62, 0xf1, 0xed, 0x59, 0x58, 0xcb}, op: "VADDPD.RU_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round zero zeroing", code: []byte{0x62, 0xf1, 0xed, 0xf9, 0x58, 0xcb}, op: "VADDPD.RZ_SAE.Z", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed sae", code: []byte{0x62, 0xf1, 0xed, 0x59, 0x5f, 0xcb}, op: "VMAXPD.SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "scalar round up", code: encodeX86EVEXBinaryFloat(0x5c, 2, 2, true, false, 2, 5, 6, 7), op: "VSUBSS.RU_SAE", args: []string{"X7", "X6", "K2", "X5"}}, + {name: "scalar sae zeroing", code: encodeX86EVEXBinaryFloat(0x5d, 3, 0, true, true, 4, 8, 9, 10), op: "VMINSD.SAE.Z", args: []string{"X10", "X9", "K4", "X8"}}, + {name: "compressed memory", code: []byte{0x62, 0x71, 0x34, 0x48, 0x5c, 0x4e, 0x02}, op: "VSUBPS", args: []string{"128(SI)", "Z9", "Z9"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + scalarBroadcast := encodeX86EVEXBinaryFloat(0x58, 2, 0, true, false, 0, 0, 0, 0) + scalarBroadcast[len(scalarBroadcast)-1] = 0 + wrongW := encodeX86EVEXBinaryFloat(0x58, 0, 0, false, false, 0, 0, 0, 0) + wrongW[2] ^= 0x80 + invalid := [][]byte{ + encodeX86EVEXBinaryFloat(0x58, 0, 0, false, true, 0, 0, 0, 0), + encodeX86EVEXBinaryFloat(0x58, 0, 3, false, false, 0, 0, 0, 0), + scalarBroadcast, + wrongW, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid EVEX encoding %x returned ok=%v err=%v", code, ok, err) + } + } + + mode32 := encodeX86EVEXBinaryFloat(0x58, 0, 0, false, false, 1, 0, 0, 0) + if _, _, ok, err := decodedX86VEXBinaryFloatInstruction(mode32, 32); !ok || err == nil { + t.Fatalf("masked EVEX form in 32-bit mode returned ok=%v err=%v", ok, err) + } +} + +func TestDecodedX86VEXBinaryFloatMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x01, 0xe5, 0x58, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXBinaryFloatInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VADDPD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXBinaryFloat(0x58, 0, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXBinaryFloat(0x5a, 0, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXBinaryFloatInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + scalarY := encodeX86VEXBinaryFloat(0x58, 2, true, true, false, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86VEXBinaryFloatInstruction(scalarY, 64); !matched || decodeErr == nil { + t.Fatalf("scalar VEX.256 encoding %x returned ok=%v err=%v", scalarY, matched, decodeErr) + } + extended := encodeX86VEXBinaryFloat(0x58, 0, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXBinaryFloatInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_fma3_test.go b/amd64_raw_vex_fma3_test.go new file mode 100644 index 000000000..c7d99768e --- /dev/null +++ b/amd64_raw_vex_fma3_test.go @@ -0,0 +1,258 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXFMA3(opcode byte, width64, width256 bool, destination, source1, source2 int) []byte { + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 2) + vex1 := byte(((^source1)&15)<<3 | 1) + if width64 { + vex1 |= 0x80 + } + if width256 { + vex1 |= 0x04 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func encodeX86EVEXFMA3(opcode byte, width64 bool, vectorBits byte, broadcastOrRounding, zeroing bool, mask, destination, source1, source2 int) []byte { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source2>>4)&1)<<6 | (1-(source2>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 2) + p1 := byte((^source1&15)<<3 | 1<<2 | 1) + if width64 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^source1>>4&1)<<3 | mask) + if broadcastOrRounding { + p2 |= 0x10 + } + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + return []byte{0x62, p0, p1, p2, opcode, modRM} +} + +func TestTranslateRawVEXFMA3WeaviateRegression(t *testing.T) { + const source = `TEXT rawFMA(SB), $0-0 + LONG $0x985de2c4; BYTE $0x1e + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawFMA": {Name: "rawFMA", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.fma.v8f32") { + t.Fatalf("raw VFMADD132PS lowering omitted vector FMA:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-fma3.ll", "amd64-raw-vex-fma3.o", ir) +} + +func TestTranslateRawEVEXFMA3WeaviateRegression(t *testing.T) { + const source = `TEXT rawFMAEVEX(SB), $0-0 + LONG $0x4835d262; WORD $0xc9b8 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawFMAEVEX": {Name: "rawFMAEVEX", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.fma.v16f32") { + t.Fatalf("raw EVEX VFMADD231PS lowering omitted vector FMA:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evex-fma3.ll", "amd64-raw-evex-fma3.o", ir) +} + +func TestDecodedX86VEXFMA3CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, spec := range decodedX86VEXFMA3Ops { + for _, width64 := range []bool{false, true} { + widths := []bool{false, true} + if spec.scalar { + widths = []bool{false} + } + for _, width256 := range widths { + prefix := "X" + if width256 { + prefix = "Y" + } + var registers [16]Reg + for index := range registers { + registers[index] = Reg(fmt.Sprintf("%s%d", prefix, index)) + } + wantOp := spec.op(width64) + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < 16; source2++ { + code := encodeX86VEXFMA3(opcode, width64, width256, destination, source1, source2) + got, length, ok, err := decodedX86VEXFMA3Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != wantOp || len(got.Args) != 3 || + got.Args[0].Kind != OpReg || got.Args[0].Reg != registers[source2] || + got.Args[1].Kind != OpReg || got.Args[1].Reg != registers[source1] || + got.Args[2].Kind != OpReg || got.Args[2].Reg != registers[destination] { + t.Fatalf("decode %x = %+v, want %s %s,%s,%s", code, got, wantOp, registers[source2], registers[source1], registers[destination]) + } + count++ + } + } + } + } + } + } + if count != 393216 { + t.Fatalf("covered %d VEX FMA3 register encodings, want 393216", count) + } + t.Logf("exhaustively checked %d VEX FMA3 register encodings", count) +} + +func TestDecodedX86EVEXFMA3CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, spec := range decodedX86VEXFMA3Ops { + for _, width64 := range []bool{false, true} { + vectorLengths := []byte{0, 1, 2} + for _, vectorBits := range vectorLengths { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + if spec.scalar { + prefix = "X" + } + var registers [32]Reg + for index := range registers { + registers[index] = Reg(fmt.Sprintf("%s%d", prefix, index)) + } + wantOp := spec.op(width64) + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + code := encodeX86EVEXFMA3(opcode, width64, vectorBits, false, false, 1, destination, source1, source2) + got, length, ok, err := decodedX86VEXFMA3Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != wantOp || len(got.Args) != 4 || + got.Args[0].Kind != OpReg || got.Args[0].Reg != registers[source2] || + got.Args[1].Kind != OpReg || got.Args[1].Reg != registers[source1] || + got.Args[2].Kind != OpReg || got.Args[2].Reg != "K1" || + got.Args[3].Kind != OpReg || got.Args[3].Reg != registers[destination] { + t.Fatalf("decode %x = %+v, want %s %s,%s,K1,%s", code, got, wantOp, registers[source2], registers[source1], registers[destination]) + } + count++ + } + } + } + } + } + } + if count != 5898240 { + t.Fatalf("covered %d EVEX FMA3 register encodings, want 5898240", count) + } + t.Logf("exhaustively checked %d EVEX FMA3 register encodings", count) +} + +func TestDecodedX86EVEXFMA3ModifiersAndMemory(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + args []string + }{ + {name: "packed zeroing", code: encodeX86EVEXFMA3(0xb8, false, 1, false, true, 3, 29, 28, 21), op: "VFMADD231PS.Z", args: []string{"Y21", "Y28", "K3", "Y29"}}, + {name: "packed broadcast zeroing", code: []byte{0x62, 0xf2, 0xed, 0xd9, 0x98, 0x08}, op: "VFMADD132PD.BCST.Z", args: []string{"0(AX)", "Z2", "K1", "Z1"}}, + {name: "packed round nearest", code: encodeX86EVEXFMA3(0xa8, true, 0, true, false, 1, 1, 2, 3), op: "VFMADD213PD.RN_SAE", args: []string{"Z3", "Z2", "K1", "Z1"}}, + {name: "packed round down", code: encodeX86EVEXFMA3(0xba, false, 1, true, false, 2, 4, 5, 6), op: "VFMSUB231PS.RD_SAE", args: []string{"Z6", "Z5", "K2", "Z4"}}, + {name: "packed round up", code: encodeX86EVEXFMA3(0x9c, true, 2, true, false, 3, 7, 8, 9), op: "VFNMADD132PD.RU_SAE", args: []string{"Z9", "Z8", "K3", "Z7"}}, + {name: "packed round zero zeroing", code: encodeX86EVEXFMA3(0xae, false, 3, true, true, 4, 10, 11, 12), op: "VFNMSUB213PS.RZ_SAE.Z", args: []string{"Z12", "Z11", "K4", "Z10"}}, + {name: "scalar round up", code: encodeX86EVEXFMA3(0xb9, false, 2, true, false, 5, 13, 14, 15), op: "VFMADD231SS.RU_SAE", args: []string{"X15", "X14", "K5", "X13"}}, + {name: "compressed memory", code: []byte{0x62, 0xd2, 0x35, 0x48, 0xb8, 0x49, 0x02}, op: "VFMADD231PS", args: []string{"128(R9)", "Z9", "Z1"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VEXFMA3Instruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != len(test.args) { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + for index := range test.args { + if got.Args[index].String() != test.args[index] { + t.Fatalf("decode %x = %+v, want %s %s", test.code, got, test.op, strings.Join(test.args, ", ")) + } + } + }) + } + + scalarBroadcast := encodeX86EVEXFMA3(0x99, false, 0, true, false, 0, 0, 0, 0) + scalarBroadcast[len(scalarBroadcast)-1] = 0 + wrongPP := encodeX86EVEXFMA3(0x98, false, 0, false, false, 0, 0, 0, 0) + wrongPP[2] &^= 1 + invalid := [][]byte{ + encodeX86EVEXFMA3(0x98, false, 0, false, true, 0, 0, 0, 0), + encodeX86EVEXFMA3(0x98, false, 3, false, false, 0, 0, 0, 0), + scalarBroadcast, + wrongPP, + } + for _, code := range invalid { + if _, _, ok, err := decodedX86VEXFMA3Instruction(code, 64); !ok || err == nil { + t.Fatalf("invalid EVEX encoding %x returned ok=%v err=%v", code, ok, err) + } + } + + mode32 := encodeX86EVEXFMA3(0x98, false, 0, false, false, 1, 0, 0, 0) + if _, _, ok, err := decodedX86VEXFMA3Instruction(mode32, 32); !ok || err == nil { + t.Fatalf("masked EVEX form in 32-bit mode returned ok=%v err=%v", ok, err) + } +} + +func TestDecodedX86VEXFMA3MemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x02, 0xe5, 0x98, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXFMA3Instruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VFMADD132PD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXFMA3(0x98, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 3, valid[2], valid[3], valid[4]}, + encodeX86VEXFMA3(0x95, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXFMA3Instruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := []byte{valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4]} + if _, _, matched, decodeErr := decodedX86VEXFMA3Instruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + scalarY := encodeX86VEXFMA3(0x99, false, true, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86VEXFMA3Instruction(scalarY, 64); !matched || decodeErr == nil { + t.Fatalf("scalar VEX.256 encoding %x returned ok=%v err=%v", scalarY, matched, decodeErr) + } + extended := encodeX86VEXFMA3(0x98, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXFMA3Instruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_horizontal_float_test.go b/amd64_raw_vex_horizontal_float_test.go new file mode 100644 index 000000000..a8f953f78 --- /dev/null +++ b/amd64_raw_vex_horizontal_float_test.go @@ -0,0 +1,158 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXHorizontalFloat(opcode, pp byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXHorizontalFloatWeaviateRegression(t *testing.T) { + const source = `TEXT rawHorizontal(SB), $0-0 + LONG $0xc07cffc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawHorizontal": {Name: "rawHorizontal", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "@llvm.experimental.constrained.fadd.v8f32") { + t.Fatalf("raw VHADDPS lowering omitted horizontal addition:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-horizontal-float.ll", "amd64-raw-vex-horizontal-float.o", ir) +} + +func TestDecodedX86VEXHorizontalFloatCompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, stem := range map[byte]string{0x7c: "VHADD", 0x7d: "VHSUB", 0xd0: "VADDSUB"} { + for pp, suffix := range map[byte]string{1: "PD", 3: "PS"} { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXHorizontalFloat(opcode, pp, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXHorizontalFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + } + if count != 122880 { + t.Fatalf("covered %d VEX horizontal and alternating floating register encodings, want 122880", count) + } +} + +func TestTranslateRawVEXAddSubComplexAVX2Regression(t *testing.T) { + const source = "TEXT rawAddSub(SB),$0-0\n\tLONG $0xc1d0f9c5\n\tRET\n" + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: map[string]FuncSig{ + "rawAddSub": {Name: "rawAddSub", Ret: Void}, + }}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "fsub double") || !strings.Contains(ir, "fadd double") { + t.Fatalf("VADDSUBPD omitted alternating arithmetic:\n%s", ir) + } + compileLLVMToObject(t, llc, triple, "raw-vaddsub.ll", "raw-vaddsub.o", ir) + }) + } +} + +func TestDecodedX86VEXHorizontalFloatMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x01, 0xe5, 0x7d, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXHorizontalFloatInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VHSUBPD" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + code[4] = 0xd0 + got, length, ok, err = decodedX86VEXHorizontalFloatInstruction(code, 64) + if err != nil || !ok || length != len(code) || got.Op != "VADDSUBPD" || + got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || + got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("VADDSUBPD memory source %x decoded as %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXHorizontalFloat(0x7c, 1, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXHorizontalFloat(0x7e, 1, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXHorizontalFloatInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := encodeX86VEXHorizontalFloat(0x7c, 0, true, false, false, 0, 0, 0) + if _, _, matched, decodeErr := decodedX86VEXHorizontalFloatInstruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + extended := encodeX86VEXHorizontalFloat(0x7c, 1, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXHorizontalFloatInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_integration_test.go b/amd64_raw_vex_integration_test.go new file mode 100644 index 000000000..bf5c3d4c1 --- /dev/null +++ b/amd64_raw_vex_integration_test.go @@ -0,0 +1,47 @@ +package plan9asm + +import "testing" + +func TestTranslateX86RawVEXVNNIAndVariableDwordPermuteAllTargets(t *testing.T) { + const source = `TEXT rawvexfamilies(SB),$0-0 + // {vex} VPDPBUSD X2, X1, X0. + BYTE $0xc4; BYTE $0xe2; BYTE $0x71; BYTE $0x50; BYTE $0xc2 + // {vex} VPERMD Y5, Y3, Y4. + BYTE $0xc4; BYTE $0xe2; BYTE $0x65; BYTE $0x36; BYTE $0xe5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name string + goarch string + triple string + }{ + {name: "darwin-amd64", goarch: "amd64", triple: "x86_64-apple-darwin"}, + {name: "linux-amd64", goarch: "amd64", triple: "x86_64-unknown-linux-gnu"}, + {name: "windows-amd64", goarch: "amd64", triple: "x86_64-pc-windows-msvc"}, + {name: "linux-386", goarch: "386", triple: "i386-unknown-linux-gnu"}, + {name: "windows-386", goarch: "386", triple: "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + requireX86GoAssemblerResult(t, target.goarch, source, true) + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "rawvexfamilies": {Name: "rawvexfamilies", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-vex-families-"+target.name+".ll", "raw-vex-families-"+target.name+".o", ll) + }) + } +} diff --git a/amd64_raw_vex_packed_float_logical_test.go b/amd64_raw_vex_packed_float_logical_test.go new file mode 100644 index 000000000..123cad817 --- /dev/null +++ b/amd64_raw_vex_packed_float_logical_test.go @@ -0,0 +1,154 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86VEXPackedFloatLogicalTestOpcodes = map[byte]string{ + 0x54: "VAND", + 0x55: "VANDN", + 0x56: "VOR", + 0x57: "VXOR", +} + +func encodeX86VEXPackedFloatLogical(opcode, pp byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXPackedFloatLogicalWeaviateRegression(t *testing.T) { + const source = `TEXT rawLogical(SB), $0-0 + WORD $0x8944; BYTE $0xc8 + WORD $0xe083; BYTE $0x03 + LONG $0xc057f8c5 + WORD $0xc931 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawLogical": {Name: "rawLogical", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "xor <4 x i32>") { + t.Fatalf("raw VXORPS lowering omitted vector xor:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-logical.ll", "amd64-raw-vex-logical.o", ir) +} + +func TestDecodedX86VEXPackedFloatLogicalCompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, stem := range x86VEXPackedFloatLogicalTestOpcodes { + for pp, suffix := range map[byte]string{0: "PS", 1: "PD"} { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedFloatLogical(opcode, pp, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXPackedFloatLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := Op(stem + suffix) + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + } + if count != 163840 { + t.Fatalf("covered %d VEX packed floating logical register encodings, want 163840", count) + } +} + +func TestDecodedX86VEXPackedFloatLogicalMemoryForms(t *testing.T) { + tests := []struct { + name string + code []byte + op Op + source MemRef + second Reg + dest Reg + }{ + {name: "base", code: []byte{0xc4, 0xe1, 0x78, 0x57, 0x00}, op: "VXORPS", source: MemRef{Base: AX}, second: "X0", dest: "X0"}, + {name: "extended SIB displacement and FS", code: []byte{0x64, 0xc4, 0x01, 0x65, 0x54, 0x64, 0x88, 0x20}, op: "VANDPD", source: MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32}, second: "Y3", dest: "Y12"}, + {name: "base displacement and GS", code: []byte{0x65, 0xc5, 0xec, 0x56, 0x4d, 0x7f}, op: "VORPS", source: MemRef{Segment: GS, Base: BP, Off: 0x7f}, second: "Y2", dest: "Y1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, length, ok, err := decodedX86VEXPackedFloatLogicalInstruction(test.code, 64) + if err != nil || !ok || length != len(test.code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", test.code, got, length, ok, err) + } + if got.Op != test.op || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != test.source || got.Args[1].Kind != OpReg || got.Args[1].Reg != test.second || got.Args[2].Kind != OpReg || got.Args[2].Reg != test.dest { + t.Fatalf("decode %x = %+v, want %s %+v, %s, %s", test.code, got, test.op, test.source, test.second, test.dest) + } + }) + } +} + +func TestDecodedX86VEXPackedFloatLogicalRejectsReservedForms(t *testing.T) { + valid := encodeX86VEXPackedFloatLogical(0x57, 0, true, false, false, 0, 0, 0) + for _, code := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXPackedFloatLogical(0x58, 0, true, false, false, 0, 0, 0), + } { + if got, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(code, 64); ok || err != nil { + t.Fatalf("reserved encoding %x decoded as %+v, ok=%v, err=%v", code, got, ok, err) + } + } + reservedPP := encodeX86VEXPackedFloatLogical(0x57, 2, true, false, false, 0, 0, 0) + if _, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(reservedPP, 64); !ok || err == nil { + t.Fatalf("reserved pp encoding %x returned ok=%v err=%v", reservedPP, ok, err) + } + ripRelative := []byte{0xc5, 0xec, 0x56, 0x0d, 0x78, 0x56, 0x34, 0x12} + if _, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(ripRelative, 64); !ok || err == nil { + t.Fatalf("source-layout-unsafe RIP-relative encoding %x returned ok=%v err=%v", ripRelative, ok, err) + } + extended := encodeX86VEXPackedFloatLogical(0x57, 0, true, false, false, 8, 8, 8) + if _, _, ok, err := decodedX86VEXPackedFloatLogicalInstruction(extended, 32); !ok || err == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", ok, err) + } +} diff --git a/amd64_raw_vex_packed_integer_logical_test.go b/amd64_raw_vex_packed_integer_logical_test.go new file mode 100644 index 000000000..576222fc5 --- /dev/null +++ b/amd64_raw_vex_packed_integer_logical_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +var x86VEXPackedIntegerLogicalTestOpcodes = map[byte]Op{ + 0xdb: "VPAND", + 0xdf: "VPANDN", + 0xeb: "VPOR", + 0xef: "VPXOR", +} + +func encodeX86VEXPackedIntegerLogical(opcode byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | 1) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXPackedIntegerLogicalWeaviateRegression(t *testing.T) { + const source = `TEXT rawLogical(SB), $0-0 + LONG $0xc0eff9c5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawLogical": {Name: "rawLogical", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "xor <16 x i8>") { + t.Fatalf("raw VPXOR lowering omitted vector xor:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-integer-logical.ll", "amd64-raw-vex-integer-logical.o", ir) +} + +func TestDecodedX86VEXPackedIntegerLogicalCompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, wantOp := range x86VEXPackedIntegerLogicalTestOpcodes { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedIntegerLogical(opcode, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXPackedIntegerLogicalInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 81920 { + t.Fatalf("covered %d VEX packed integer logical register encodings, want 81920", count) + } +} + +func TestDecodedX86VEXPackedIntegerLogicalMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x01, 0xe5, 0xdb, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXPackedIntegerLogicalInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VPAND" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXPackedIntegerLogical(0xef, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + {0xc4, valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXPackedIntegerLogical(0xee, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedIntegerLogicalInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := []byte{valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4]} + if _, _, matched, decodeErr := decodedX86VEXPackedIntegerLogicalInstruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + extended := encodeX86VEXPackedIntegerLogical(0xef, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXPackedIntegerLogicalInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_packed_integer_move_test.go b/amd64_raw_vex_packed_integer_move_test.go new file mode 100644 index 000000000..17ef6b597 --- /dev/null +++ b/amd64_raw_vex_packed_integer_move_test.go @@ -0,0 +1,62 @@ +package plan9asm + +import ( + "reflect" + "testing" +) + +func TestX86RawVEXPackedIntegerMoveCorpusEncoding(t *testing.T) { + const source = "TEXT corpusmove(SB),4,$0-0\n\tVMOVDQU -32(SI)(CX*4), Y14\n\tRET\n" + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for i, got := range decoded.Instrs { + if got.Op != want[i].Op || !reflect.DeepEqual(got.Args, want[i].Args) { + t.Fatalf("instruction %d=%+v, want %+v", i, got, want[i]) + } + } +} + +func TestX86RawVEXPackedIntegerMoveRejectsReservedForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {"missing ModRM", []byte{0xc5, 0xfe, 0x6f}, 64}, + {"missing SIB", []byte{0xc5, 0xfe, 0x6f, 0x04}, 64}, + {"missing displacement", []byte{0xc5, 0xfe, 0x6f, 0x44, 0x24}, 64}, + {"reserved vvvv", []byte{0xc5, 0xf6, 0x6f, 0xc0}, 64}, + {"extended register in 386", []byte{0xc5, 0x7e, 0x6f, 0xc0}, 32}, + {"address override", []byte{0x67, 0xc5, 0xfe, 0x6f, 0xc0}, 64}, + } { + t.Run(test.name, func(t *testing.T) { + if _, _, matched, err := decodedX86PackedMoveInstruction(test.code, test.mode); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", test.code, matched, err) + } + }) + } +} + +func TestX86RawVEXPackedIntegerMoveIgnoresWBit(t *testing.T) { + for _, code := range [][]byte{ + {0xc4, 0xe1, 0x7e, 0x6f, 0xc0}, + {0xc4, 0xe1, 0xfe, 0x6f, 0xc0}, + } { + got, size, matched, err := decodedX86PackedMoveInstruction(code, 64) + if err != nil || !matched || size != len(code) || got.Op != "VMOVDQU" || + !reflect.DeepEqual(got.Args, []Operand{{Kind: OpReg, Reg: "Y0"}, {Kind: OpReg, Reg: "Y0"}}) { + t.Fatalf("decode %x=%+v size=%d matched=%v err=%v", code, got, size, matched, err) + } + } +} diff --git a/amd64_raw_vex_packed_madd_literal_test.go b/amd64_raw_vex_packed_madd_literal_test.go new file mode 100644 index 000000000..84df43a1a --- /dev/null +++ b/amd64_raw_vex_packed_madd_literal_test.go @@ -0,0 +1,137 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVEXPackedMADDConstant(opcode byte, opcodeMap int, vex3, width256, w bool) []byte { + code := encodeX86VEXPackedMADD(opcode, opcodeMap, vex3, width256, w, 0, 0, 0) + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + width := 16 + if width256 { + width = 32 + } + for i := 0; i < width; i++ { + code = append(code, byte(i*29+13)) + } + return code +} + +func TestDecodeX86RawVEXPackedMADDLiteralCompleteFamily(t *testing.T) { + for _, form := range []struct { + opcode byte + mapID int + op Op + vex2 bool + }{ + {0xf5, 1, "VPMADDWD", true}, + {0x04, 2, "VPMADDUBSW", false}, + } { + for _, vex3 := range []bool{false, true} { + if !vex3 && !form.vex2 { + continue + } + for _, width256 := range []bool{false, true} { + for _, w := range []bool{false, true} { + if !vex3 && w { + continue + } + name := fmt.Sprintf("%s/vex3=%t/y=%t/w=%t", form.op, vex3, width256, w) + t.Run(name, func(t *testing.T) { + code := x86RawVEXPackedMADDConstant(form.opcode, form.mapID, vex3, width256, w) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != form.op || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, form.op) + } + }) + } + } + } + } +} + +func TestDecodeX86RawVEXPackedMADDLiteralRejectsUnsafeSources(t *testing.T) { + valid := x86RawVEXPackedMADDConstant(0xf5, 1, false, false, false) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(code []byte) []byte { return code[:len(code)-1] }, + "overlap": func(code []byte) []byte { + code[4] = 0 + return code + }, + "outside": func(code []byte) []byte { + code[4] = 127 + return code + }, + "segment": func(code []byte) []byte { return append([]byte{0x64}, code...) }, + "address": func(code []byte) []byte { return append([]byte{0x67}, code...) }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVEXPackedMADDLiteralObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, form := range []struct { + opcode byte + mapID int + }{ + {0xf5, 1}, {0x04, 2}, + } { + code := x86RawVEXPackedMADDConstant(form.opcode, form.mapID, true, true, false) + name := fmt.Sprintf("packedMADDConstant%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + pools = append(pools, code[len(code)-32:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "packed-madd-literal.ll", "packed-madd-literal.o", ir) + }) + } +} diff --git a/amd64_raw_vex_packed_madd_test.go b/amd64_raw_vex_packed_madd_test.go new file mode 100644 index 000000000..052c64c0e --- /dev/null +++ b/amd64_raw_vex_packed_madd_test.go @@ -0,0 +1,130 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedMADD(opcode byte, opcodeMap int, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | 1) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | opcodeMap) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestTranslateRawVEXPackedMADDWeaviateRegression(t *testing.T) { + const source = `TEXT rawMADD(SB), $0-0 + LONG $0xd4f5edc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawMADD": {Name: "rawMADD", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "mul <16 x i32>") { + t.Fatalf("raw VPMADDWD lowering omitted packed multiplication:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-packed-madd.ll", "amd64-raw-vex-packed-madd.o", ir) +} + +func TestDecodedX86VEXPackedMADDCompleteRegisterFamily(t *testing.T) { + tests := []struct { + opcode byte + opcodeMap int + op Op + vex2 bool + }{ + {opcode: 0xf5, opcodeMap: 1, op: "VPMADDWD", vex2: true}, + {opcode: 0x04, opcodeMap: 2, op: "VPMADDUBSW"}, + } + count := 0 + for _, test := range tests { + for _, vex3 := range []bool{false, true} { + if !vex3 && !test.vex2 { + continue + } + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedMADD(test.opcode, test.opcodeMap, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86VEXPackedMADDInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != test.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, test.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 36864 { + t.Fatalf("covered %d VEX packed multiply-add register encodings, want 36864", count) + } +} + +func TestDecodedX86VEXPackedMADDMemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x02, 0xe5, 0x04, 0x64, 0x88, 0x20} + got, length, ok, err := decodedX86VEXPackedMADDInstruction(code, 64) + wantSource := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VPMADDUBSW" || len(got.Args) != 3 || got.Args[0].Kind != OpMem || got.Args[0].Mem != wantSource || got.Args[1].String() != "Y3" || got.Args[2].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXPackedMADD(0xf5, 1, true, false, false, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc5}, + encodeX86VEXPackedMADD(0xf4, 1, true, false, false, 0, 0, 0), + encodeX86VEXPackedMADD(0x04, 1, true, false, false, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedMADDInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + invalidPP := []byte{valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4]} + if _, _, matched, decodeErr := decodedX86VEXPackedMADDInstruction(invalidPP, 64); !matched || decodeErr == nil { + t.Fatalf("invalid pp encoding %x returned ok=%v err=%v", invalidPP, matched, decodeErr) + } + extended := encodeX86VEXPackedMADD(0xf5, 1, true, false, false, 8, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXPackedMADDInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vex_packed_shuffle_test.go b/amd64_raw_vex_packed_shuffle_test.go new file mode 100644 index 000000000..eaef494b4 --- /dev/null +++ b/amd64_raw_vex_packed_shuffle_test.go @@ -0,0 +1,159 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedShuffle(pp byte, vex3, width256, widthIgnored bool, destination, source int, immediate byte) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | 15<<3 | int(l)<<2 | int(pp)) + return []byte{0xc5, vex1, 0x70, modRM, immediate} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source/8)<<5 | 1) + vex1 := byte(15<<3 | int(l)<<2 | int(pp)) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, 0x70, modRM, immediate} +} + +func TestTranslateRawVEXPackedShuffleWeaviateRegression(t *testing.T) { + const source = `TEXT rawShuffle(SB), $0-0 + LONG $0xc870f9c5 + BYTE $0x1b + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawShuffle": {Name: "rawShuffle", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <16 x i8>") || + !strings.Contains(ir, "<16 x i32> ") { + t.Fatalf("raw VPSHUFD lowering omitted packed shuffle:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-packed-shuffle.ll", "amd64-raw-vex-packed-shuffle.o", ir) +} + +func TestDecodedX86PackedShuffleCompleteVEXRegisterFamily(t *testing.T) { + tests := []struct { + pp byte + op Op + }{ + {pp: 1, op: "VPSHUFD"}, + {pp: 2, op: "VPSHUFHW"}, + {pp: 3, op: "VPSHUFLW"}, + } + count := 0 + for _, test := range tests { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + sourceLimit := 16 + if !vex3 { + sourceLimit = 8 + } + for destination := 0; destination < 16; destination++ { + for source := 0; source < sourceLimit; source++ { + immediate := byte(destination<<4 | source) + code := encodeX86VEXPackedShuffle(test.pp, vex3, width256, widthIgnored, destination, source, immediate) + got, length, ok, err := decodedX86PackedShuffleInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("$%d", immediate), fmt.Sprintf("%s%d", prefix, source), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != test.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, test.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + if count != 3840 { + t.Fatalf("covered %d VEX packed shuffle register encodings, want 3840", count) + } +} + +func TestDecodedX86PackedShuffleCompleteGo127Family(t *testing.T) { + // Go 1.27 assigns these three instructions the shared _yvpshufd table. + // The cases cover VEX/EVEX X/Y/Z, WIG, high registers, memory, masks, + // zeroing, the VPSHUFD-only broadcast, and compressed disp8. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex d reported weaviate", code: []byte{0xc5, 0xf9, 0x70, 0xc8, 0x1b}, want: "VPSHUFD $27, X0, X1"}, + {name: "vex hw y WIG memory", code: []byte{0x64, 0xc4, 0x01, 0xfe, 0x70, 0x64, 0x88, 0x20, 0x5a}, want: "VPSHUFHW $90, 32(R8)(R9*4)(FS), Y12"}, + {name: "evex d x high masked", code: []byte{0x62, 0x71, 0x7d, 0x0c, 0x70, 0xca, 0x7e}, want: "VPSHUFD $126, X2, K4, X9"}, + {name: "evex hw x high masked WIG", code: []byte{0x62, 0x41, 0xfe, 0x0a, 0x70, 0xfb, 0x0d}, want: "VPSHUFHW $13, X11, K2, X31"}, + {name: "evex lw y high masked", code: []byte{0x62, 0xe1, 0x7f, 0x2a, 0x70, 0xdf, 0x00}, want: "VPSHUFLW $0, Y7, K2, Y19"}, + {name: "evex d z broadcast zeroing", code: []byte{0x62, 0xf1, 0x7d, 0xdb, 0x70, 0x50, 0x7f, 0x09}, want: "VPSHUFD.BCST.Z $9, 508(AX), K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedShuffleInstruction(test.code, 64) + if !ok { + t.Fatal("packed shuffle encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodedX86PackedShuffleRejectsInvalidForms(t *testing.T) { + valid := encodeX86VEXPackedShuffle(1, true, false, false, 0, 0, 1) + invalidVVVV := append([]byte(nil), valid...) + invalidVVVV[2] &^= 0x08 + for name, code := range map[string][]byte{ + "truncated": {0xc5}, + "wrong opcode": {0xc5, 0xf9, 0x71, 0xc0, 1}, + "wrong map": {0xc4, 0xe2, 0x79, 0x70, 0xc0, 1}, + "unrelated pp": {0xc5, 0xf8, 0x70, 0xc0, 1}, + "EVEX D with W1": {0x62, 0xf1, 0xfd, 0x08, 0x70, 0xc0, 1}, + } { + if instruction, _, matched, decodeErr := decodedX86PackedShuffleInstruction(code, 64); matched || decodeErr != nil { + t.Fatalf("%s encoding %x decoded as %+v, ok=%v, err=%v", name, code, instruction, matched, decodeErr) + } + } + if _, _, matched, decodeErr := decodedX86PackedShuffleInstruction(invalidVVVV, 64); !matched || decodeErr == nil { + t.Fatalf("non-reserved VEX.vvvv encoding %x returned ok=%v err=%v", invalidVVVV, matched, decodeErr) + } + extended := encodeX86VEXPackedShuffle(1, true, false, false, 8, 8, 1) + if _, _, matched, decodeErr := decodedX86PackedShuffleInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } + invalidBroadcast := []byte{0x62, 0xf1, 0x7e, 0x5b, 0x70, 0x50, 0x7f, 1} + if _, _, matched, decodeErr := decodedX86PackedShuffleInstruction(invalidBroadcast, 64); !matched || decodeErr == nil { + t.Fatalf("VPSHUFHW broadcast returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vextract128_test.go b/amd64_raw_vextract128_test.go new file mode 100644 index 000000000..3cde8176d --- /dev/null +++ b/amd64_raw_vextract128_test.go @@ -0,0 +1,248 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXTRACT128(opcode, immediate byte, source, destination int) []byte { + vex0 := byte((1-source/8)<<7 | 1<<6 | (1-destination/8)<<5 | 3) + modRM := byte(0xc0 | (source&7)<<3 | destination&7) + return []byte{0xc4, vex0, 0x7d, opcode, modRM, immediate} +} + +func encodeX86EVEXTRACT(opcode, immediate byte, width64 bool, vectorBits byte, mask int, zeroing bool, source, destination int) []byte { + p0 := byte((1-source/8&1)<<7 | (1-destination/16)<<6 | (1-destination/8&1)<<5 | (1-source/16)<<4 | 3) + p1 := byte(0x7d) + if width64 { + p1 |= 0x80 + } + p2 := byte(vectorBits<<5 | 1<<3 | byte(mask)) + if zeroing { + p2 |= 0x80 + } + modRM := byte(0xc0 | (source&7)<<3 | destination&7) + return []byte{0x62, p0, p1, p2, opcode, modRM, immediate} +} + +func TestTranslateRawVEXTRACT128WeaviateRegression(t *testing.T) { + const source = `TEXT rawExtract(SB), $0-0 + LONG $0x197de3c4; WORD $0x01c1 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawExtract": {Name: "rawExtract", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <32 x i8>") { + t.Fatalf("raw VEXTRACTF128 lowering omitted vector extract:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vextract128.ll", "amd64-raw-vextract128.o", ir) +} + +func TestDecodedX86VEXTRACT128CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, wantOp := range map[byte]Op{0x19: "VEXTRACTF128", 0x39: "VEXTRACTI128"} { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + for immediate := 0; immediate < 256; immediate++ { + code := encodeX86VEXTRACT128(opcode, byte(immediate), source, destination) + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + if got.Op != wantOp || len(got.Args) != 3 || got.Args[0].Imm != int64(immediate) || got.Args[1].String() != fmt.Sprintf("Y%d", source) || got.Args[2].String() != fmt.Sprintf("X%d", destination) { + t.Fatalf("decode %x = %+v, want %s $%d, Y%d, X%d", code, got, wantOp, immediate, source, destination) + } + count++ + } + } + } + } + if count != 131072 { + t.Fatalf("covered %d VEXTRACT128 register encodings, want 131072", count) + } +} + +func TestDecodedX86VEXTRACT128MemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x03, 0x7d, 0x39, 0x64, 0x88, 0x20, 0xff} + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + wantDestination := MemRef{Segment: FS, Base: "R8", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VEXTRACTI128" || len(got.Args) != 3 || got.Args[0].Imm != 0xff || got.Args[1].String() != "Y12" || got.Args[2].Kind != OpMem || got.Args[2].Mem != wantDestination { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VEXTRACT128(0x19, 1, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + encodeX86VEXTRACT128(0x18, 1, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + for _, invalid := range [][]byte{ + {valid[0], valid[1], valid[2] &^ 1, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] &^ 4, valid[3], valid[4], valid[5]}, + {valid[0], valid[1], valid[2] | 0x80, valid[3], valid[4], valid[5]}, + } { + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(invalid, 64); !matched || decodeErr == nil { + t.Fatalf("reserved encoding %x returned ok=%v err=%v", invalid, matched, decodeErr) + } + } + extended := encodeX86VEXTRACT128(0x19, 1, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("32-bit extended-register form returned ok=%v err=%v", matched, decodeErr) + } +} + +func TestTranslateRawEVEXTRACTWeaviateRegression(t *testing.T) { + const source = `TEXT rawEVEXTRACT(SB), $0-0 + LONG $0x48fdf362 + WORD $0xc13b + BYTE $0x01 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawEVEXTRACT": {Name: "rawEVEXTRACT", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "shufflevector <64 x i8>") { + t.Fatalf("raw VEXTRACTI64X4 lowering omitted 256-bit block extract:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-evextract.ll", "amd64-raw-evextract.o", ir) +} + +func TestDecodedX86VEXTRACTCompleteEVEXRegisterFamily(t *testing.T) { + tests := []struct { + opcode byte + width64 bool + op Op + outputBytes int + vectorBits []byte + }{ + {opcode: 0x19, op: "VEXTRACTF32X4", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x19, width64: true, op: "VEXTRACTF64X2", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x1b, op: "VEXTRACTF32X8", outputBytes: 32, vectorBits: []byte{2}}, + {opcode: 0x1b, width64: true, op: "VEXTRACTF64X4", outputBytes: 32, vectorBits: []byte{2}}, + {opcode: 0x39, op: "VEXTRACTI32X4", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x39, width64: true, op: "VEXTRACTI64X2", outputBytes: 16, vectorBits: []byte{1, 2}}, + {opcode: 0x3b, op: "VEXTRACTI32X8", outputBytes: 32, vectorBits: []byte{2}}, + {opcode: 0x3b, width64: true, op: "VEXTRACTI64X4", outputBytes: 32, vectorBits: []byte{2}}, + } + count := 0 + for _, test := range tests { + for _, vectorBits := range test.vectorBits { + sourcePrefix := [...]string{"", "Y", "Z"}[vectorBits] + destinationPrefix := "X" + if test.outputBytes == 32 { + destinationPrefix = "Y" + } + for _, masking := range []struct { + mask int + zeroing bool + suffix string + }{ + {}, + {mask: 3}, + {mask: 7, zeroing: true, suffix: ".Z"}, + } { + for source := 0; source < 32; source++ { + for destination := 0; destination < 32; destination++ { + immediate := byte(source<<3 | destination&7) + code := encodeX86EVEXTRACT(test.opcode, immediate, test.width64, vectorBits, masking.mask, masking.zeroing, source, destination) + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("$%d", immediate), fmt.Sprintf("%s%d", sourcePrefix, source)} + if masking.mask != 0 { + wantArgs = append(wantArgs, fmt.Sprintf("K%d", masking.mask)) + } + wantArgs = append(wantArgs, fmt.Sprintf("%s%d", destinationPrefix, destination)) + wantOp := test.op + Op(masking.suffix) + if got.Op != wantOp || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + } + if count != 36864 { + t.Fatalf("covered %d EVEX vector extract register encodings, want 36864", count) + } +} + +func TestDecodedX86VEXTRACTCompleteEVEXMemoryAndInvalidForms(t *testing.T) { + tests := []struct { + opcode byte + width64 bool + op Op + outputBytes int + vectorBits byte + }{ + {opcode: 0x19, op: "VEXTRACTF32X4", outputBytes: 16, vectorBits: 1}, + {opcode: 0x19, width64: true, op: "VEXTRACTF64X2", outputBytes: 16, vectorBits: 2}, + {opcode: 0x1b, op: "VEXTRACTF32X8", outputBytes: 32, vectorBits: 2}, + {opcode: 0x1b, width64: true, op: "VEXTRACTF64X4", outputBytes: 32, vectorBits: 2}, + {opcode: 0x39, op: "VEXTRACTI32X4", outputBytes: 16, vectorBits: 1}, + {opcode: 0x39, width64: true, op: "VEXTRACTI64X2", outputBytes: 16, vectorBits: 2}, + {opcode: 0x3b, op: "VEXTRACTI32X8", outputBytes: 32, vectorBits: 2}, + {opcode: 0x3b, width64: true, op: "VEXTRACTI64X4", outputBytes: 32, vectorBits: 2}, + } + for _, test := range tests { + code := encodeX86EVEXTRACT(test.opcode, 0xff, test.width64, test.vectorBits, 2, false, 21, 0) + code[5] = 0x68 + code = append(code[:6], append([]byte{0x7f}, code[6:]...)...) + got, length, ok, err := decodedX86VEXTRACT128Instruction(code, 64) + wantSourcePrefix := [...]string{"", "Y", "Z"}[test.vectorBits] + wantDestination := fmt.Sprintf("%d(AX)", test.outputBytes*127) + if err != nil || !ok || length != len(code) || got.Op != test.op || len(got.Args) != 4 || got.Args[0].Imm != 0xff || got.Args[1].String() != fmt.Sprintf("%s21", wantSourcePrefix) || got.Args[2].String() != "K2" || got.Args[3].String() != wantDestination { + t.Fatalf("decode %x = %+v, length=%d ok=%v err=%v; want %s $255, %s21, K2, %s", code, got, length, ok, err, test.op, wantSourcePrefix, wantDestination) + } + } + + valid := encodeX86EVEXTRACT(0x3b, 1, true, 2, 1, false, 0, 0) + for name, mutate := range map[string]func([]byte){ + "used vvvv": func(code []byte) { code[2] &^= 0x08 }, + "broadcast": func(code []byte) { code[3] |= 0x10 }, + "reserved length": func(code []byte) { code[3] |= 0x20 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(code, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, code, matched, decodeErr) + } + } + memoryZero := encodeX86EVEXTRACT(0x3b, 1, true, 2, 1, true, 0, 0) + memoryZero[5] = 0x00 + memoryZero = append(memoryZero[:6], append([]byte{0, 0, 0, 0}, memoryZero[6:]...)...) + if _, _, matched, decodeErr := decodedX86VEXTRACT128Instruction(memoryZero, 64); !matched || decodeErr == nil { + t.Fatalf("zeroing memory encoding returned ok=%v err=%v", matched, decodeErr) + } +} diff --git a/amd64_raw_vinserti128_test.go b/amd64_raw_vinserti128_test.go new file mode 100644 index 000000000..122fbff64 --- /dev/null +++ b/amd64_raw_vinserti128_test.go @@ -0,0 +1,226 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +func encodeX86VINSERT128(opcode, immediate byte, firstSource, secondSource, destination int) []byte { + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-firstSource/8)<<5 | 3) + vexBits := byte((^secondSource&15)<<3 | 1<<2 | 1) + modRM := byte(0xc0 | (destination&7)<<3 | firstSource&7) + return []byte{0xc4, vex0, vexBits, opcode, modRM, immediate} +} + +func encodeX86VEXPackedBroadcast(opcode byte, wide bool, source, destination int) []byte { + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source/8)<<5 | 2) + vexBits := byte(0x79) + if wide { + vexBits |= 1 << 2 + } + modRM := byte(0xc0 | (destination&7)<<3 | source&7) + return []byte{0xc4, vex0, vexBits, opcode, modRM} +} + +func TestDecodeX86RawDirectiveGroupReportedInfectiousVINSERTI128(t *testing.T) { + code := []byte{ + 0xc4, 0xe3, 0x4d, 0x38, 0xf6, 0x01, + 0xc4, 0xe3, 0x45, 0x38, 0xff, 0x01, + 0xc4, 0x62, 0x7d, 0x78, 0xc5, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "infectious VINSERTI128 sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := []string{ + "VINSERTI128 $1, X6, Y6, Y6", + "VINSERTI128 $1, X7, Y7, Y7", + "VPBROADCASTB X5, Y8", + } + if len(decoded) != len(want) { + t.Fatalf("decoded %d instructions as %#v, want %d", len(decoded), decoded, len(want)) + } + for index := range want { + if !strings.HasPrefix(decoded[index].Raw, want[index]+" ") { + t.Fatalf("instruction %d decoded as %q, want %q", index, decoded[index].Raw, want[index]) + } + } +} + +func TestDecodedX86RawVINSERTI128Go127OracleCases(t *testing.T) { + for _, test := range []struct { + code []byte + want string + }{ + {code: []byte{0xc4, 0xe3, 0x05, 0x38, 0x13, 0x07}, want: "VINSERTI128 $7, 0(BX), Y15, Y2"}, + {code: []byte{0xc4, 0xc3, 0x05, 0x38, 0x13, 0x07}, want: "VINSERTI128 $7, 0(R11), Y15, Y2"}, + {code: []byte{0xc4, 0xe3, 0x05, 0x38, 0xd2, 0x07}, want: "VINSERTI128 $7, X2, Y15, Y2"}, + {code: []byte{0xc4, 0x43, 0x05, 0x38, 0xdb, 0x07}, want: "VINSERTI128 $7, X11, Y15, Y11"}, + } { + instruction, length, ok, err := decodedX86VINSERT128Instruction(test.code, 64) + if !ok || err != nil || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as (%#v, %d, %v, %v), want %q", test.code, instruction, length, ok, err, test.want) + } + } +} + +func TestDecodedX86RawVINSERT128CompleteRegisterFamily(t *testing.T) { + count := 0 + for opcode, wantOp := range map[byte]Op{0x18: "VINSERTF128", 0x38: "VINSERTI128"} { + for firstSource := 0; firstSource < 16; firstSource++ { + for secondSource := 0; secondSource < 16; secondSource++ { + for destination := 0; destination < 16; destination++ { + immediate := byte(firstSource*16 + destination) + code := encodeX86VINSERT128(opcode, immediate, firstSource, secondSource, destination) + got, length, ok, err := decodedX86VINSERT128Instruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{ + fmt.Sprintf("$%d", immediate), + fmt.Sprintf("X%d", firstSource), + fmt.Sprintf("Y%d", secondSource), + fmt.Sprintf("Y%d", destination), + } + if got.Op != wantOp || len(got.Args) != len(wantArgs) { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + for index := range wantArgs { + if got.Args[index].String() != wantArgs[index] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + } + count++ + } + } + } + } + if count != 8192 { + t.Fatalf("covered %d VINSERT128 register encodings, want 8192", count) + } +} + +func TestDecodedX86RawVINSERT128MemoryAndInvalidForms(t *testing.T) { + code := []byte{0x64, 0xc4, 0x03, 0x15, 0x38, 0x64, 0x8b, 0x20, 0xff} + got, length, ok, err := decodedX86VINSERT128Instruction(code, 64) + wantMemory := MemRef{Segment: FS, Base: "R11", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != "VINSERTI128" || len(got.Args) != 4 || got.Args[0].Imm != 0xff || got.Args[1].Kind != OpMem || !reflect.DeepEqual(got.Args[1].Mem, wantMemory) || got.Args[2].String() != "Y13" || got.Args[3].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + + valid := encodeX86VINSERT128(0x38, 1, 0, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4], valid[5]}, + encodeX86VINSERT128(0x39, 1, 0, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VINSERT128Instruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + for name, mutate := range map[string]func([]byte){ + "wrong pp": func(code []byte) { code[2] &^= 1 }, + "VEX.W": func(code []byte) { code[2] |= 0x80 }, + "VEX.128": func(code []byte) { code[2] &^= 4 }, + "address override": func(code []byte) { copy(code[1:], code); code[0] = 0x67 }, + } { + code := append([]byte(nil), valid...) + if name == "address override" { + code = append(code, 0) + } + mutate(code) + if _, _, matched, decodeErr := decodedX86VINSERT128Instruction(code, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, code, matched, decodeErr) + } + } + truncatedImmediate := valid[:len(valid)-1] + if _, _, matched, decodeErr := decodedX86VINSERT128Instruction(truncatedImmediate, 64); !matched || decodeErr == nil { + t.Fatalf("truncated immediate returned ok=%v err=%v", matched, decodeErr) + } + if _, _, matched, decodeErr := decodedX86VINSERT128Instruction(valid, 32); !matched || decodeErr == nil { + t.Fatalf("386 VINSERT128 returned ok=%v err=%v", matched, decodeErr) + } +} + +func TestDecodedX86RawVEXPackedBroadcastCompleteRegisterFamily(t *testing.T) { + operations := map[byte]Op{ + 0x78: "VPBROADCASTB", + 0x79: "VPBROADCASTW", + 0x58: "VPBROADCASTD", + 0x59: "VPBROADCASTQ", + } + count := 0 + for opcode, wantOp := range operations { + for _, wide := range []bool{false, true} { + for source := 0; source < 16; source++ { + for destination := 0; destination < 16; destination++ { + code := encodeX86VEXPackedBroadcast(opcode, wide, source, destination) + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + prefix := "X" + if wide { + prefix = "Y" + } + wantArgs := []string{fmt.Sprintf("X%d", source), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != wantOp || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, wantOp, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 2048 { + t.Fatalf("covered %d VEX packed-broadcast register encodings, want 2048", count) + } +} + +func TestDecodedX86RawVEXPackedBroadcastMemoryAndInvalidForms(t *testing.T) { + for opcode, wantOp := range map[byte]Op{0x78: "VPBROADCASTB", 0x79: "VPBROADCASTW", 0x58: "VPBROADCASTD", 0x59: "VPBROADCASTQ"} { + code := []byte{0x65, 0xc4, 0x02, 0x7d, opcode, 0x64, 0x8b, 0x20} + got, length, ok, err := decodedX86VEXPackedBroadcastInstruction(code, 64) + wantMemory := MemRef{Segment: GS, Base: "R11", Index: "R9", Scale: 4, Off: 32} + if err != nil || !ok || length != len(code) || got.Op != wantOp || len(got.Args) != 2 || got.Args[0].Kind != OpMem || !reflect.DeepEqual(got.Args[0].Mem, wantMemory) || got.Args[1].String() != "Y12" { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + } + + valid := encodeX86VEXPackedBroadcast(0x78, true, 0, 0) + for _, invalid := range [][]byte{ + {0xc4}, + {valid[0], valid[1] ^ 1, valid[2], valid[3], valid[4]}, + encodeX86VEXPackedBroadcast(0x77, true, 0, 0), + } { + if instruction, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(invalid, 64); matched || decodeErr != nil { + t.Fatalf("invalid encoding %x decoded as %+v, ok=%v, err=%v", invalid, instruction, matched, decodeErr) + } + } + for name, mutate := range map[string]func([]byte){ + "wrong pp": func(code []byte) { code[2] &^= 1 }, + "VEX.W": func(code []byte) { code[2] |= 0x80 }, + "used vvvv": func(code []byte) { code[2] &^= 0x08 }, + } { + code := append([]byte(nil), valid...) + mutate(code) + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(code, 64); !matched || decodeErr == nil { + t.Fatalf("%s encoding %x returned ok=%v err=%v", name, code, matched, decodeErr) + } + } + ripRelative := []byte{0xc4, 0xe2, 0x7d, 0x78, 0x05, 0, 0, 0, 0} + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(ripRelative, 64); !matched || decodeErr == nil { + t.Fatalf("RIP-relative encoding returned ok=%v err=%v", matched, decodeErr) + } + extended := encodeX86VEXPackedBroadcast(0x78, true, 8, 8) + if _, _, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(extended, 32); !matched || decodeErr == nil { + t.Fatalf("386 extended-register form returned ok=%v err=%v", matched, decodeErr) + } + valid386 := encodeX86VEXPackedBroadcast(0x78, true, 7, 7) + got, length, matched, decodeErr := decodedX86VEXPackedBroadcastInstruction(valid386, 32) + if decodeErr != nil || !matched || length != len(valid386) || got.Raw != "VPBROADCASTB X7, Y7" { + t.Fatalf("386 base-register form decoded as %+v, length=%d ok=%v err=%v", got, length, matched, decodeErr) + } +} diff --git a/amd64_raw_vmov_half_memory.go b/amd64_raw_vmov_half_memory.go new file mode 100644 index 000000000..61bc8ca44 --- /dev/null +++ b/amd64_raw_vmov_half_memory.go @@ -0,0 +1,92 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// Go 1.27 uses one _yvmovhpd operand table for all four VEX/EVEX half-vector +// memory moves. Register-only 0F 12/16 forms are the separate VMOVHLPS and +// VMOVLHPS family and are decoded before this function. +func decodedX86RawVectorHalfMemoryInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, ok := decodeX86RawVectorEncoding(code) + if !ok || p.mapNumber != 1 || (p.pp != 0 && p.pp != 1) { + return Instr{}, 0, false, nil + } + if p.opcode != 0x12 && p.opcode != 0x13 && p.opcode != 0x16 && p.opcode != 0x17 { + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("vector half memory move: %s", message) + } + if mode != 32 && mode != 64 { + return fail("unsupported x86 mode") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if p.vectorLength != 0 || p.mask != 0 || p.zero || p.broadcast { + return fail("invalid vector length, mask, zeroing, or broadcast") + } + if p.evex && (!p.fixed || p.w != (p.pp == 1)) { + return fail("invalid EVEX fixed or W bit") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + if code[p.modRM]>>6 == 3 { + if p.pp == 0 { + return Instr{}, 0, false, nil + } + return fail("Go's VMOVHPD/VMOVLPD forms require memory") + } + + var memory Operand + var consumed int + var err error + if p.evex { + memory, consumed, err = decodedX86EVEXRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X", 8) + } else { + memory, consumed, err = decodedX86VEXVectorRMOperand(code[p.modRM:], mode, p.b, p.x, p.segment, "X") + } + if err != nil { + return Instr{}, 0, true, err + } + if memory.Kind != OpMem { + return fail("expected ModRM memory operand") + } + + op := Op("VMOVLPS") + if p.pp == 1 { + op = "VMOVLPD" + } + if p.opcode == 0x16 || p.opcode == 0x17 { + if p.pp == 0 { + op = "VMOVHPS" + } else { + op = "VMOVHPD" + } + } + store := p.opcode == 0x13 || p.opcode == 0x17 + vector := int(code[p.modRM]>>3&7) + p.r*8 + if mode == 32 && (vector >= 8 || p.upper >= 8) { + return fail("extended vector register in 32-bit mode") + } + register := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", vector))} + args := []Operand{memory, {Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))}, register} + if store { + if p.upper != 0 { + return fail("store reserves the VEX/EVEX vvvv field") + } + args = []Operand{register, memory} + } + printed := make([]string, len(args)) + for i, arg := range args { + printed[i] = arg.String() + } + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s", op, strings.Join(printed, ", ")), + }, p.modRM + consumed, true, nil +} diff --git a/amd64_raw_vmov_half_memory_test.go b/amd64_raw_vmov_half_memory_test.go new file mode 100644 index 000000000..b030f5a1c --- /dev/null +++ b/amd64_raw_vmov_half_memory_test.go @@ -0,0 +1,238 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawVectorHalfMemory(op Op, encoding string, vector, upper int, store, wig bool) []byte { + high := strings.Contains(string(op), "H") + pd := strings.HasSuffix(string(op), "PD") + opcode := byte(0x12) + if high { + opcode = 0x16 + } + if store { + opcode++ + upper = 0 + } + pp := byte(0) + if pd { + pp = 1 + } + modRM := byte((vector&7)<<3 | 2) + rExt := byte(1 - ((vector >> 3) & 1)) + switch encoding { + case "VEX2": + p1 := rExt<<7 | byte(^upper&15)<<3 | pp + return []byte{0xc5, p1, opcode, modRM} + case "VEX3": + p0 := rExt<<7 | 1 // SIB index and base are R13 and R12. + p1 := byte(^upper&15)<<3 | pp + if wig { + p1 |= 0x80 + } + return []byte{0xc4, p0, p1, opcode, (modRM & 0x38) | 0x44, 0xac, 2} + case "EVEX": + r2Ext := byte(1 - ((vector >> 4) & 1)) + p0 := rExt<<7 | r2Ext<<4 | 1 + p1 := byte(^upper&15)<<3 | 4 | pp + if pd { + p1 |= 0x80 + } + p2 := byte((1 - ((upper >> 4) & 1)) << 3) + return []byte{0x62, p0, p1, p2, opcode, (modRM & 0x38) | 0x44, 0xac, 2} + default: + panic("unknown half-vector encoding") + } +} + +func TestTranslateRawVMOVHPDPathtracerRegression(t *testing.T) { + const source = ` +TEXT rawVMOVHPD(SB),$0-0 + MOVQ $0, DX + LONG $0x0217f9c5 // VMOVHPD X0, (DX) + RET +` + requireX86GoAssemblerResult(t, "amd64", source, true) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: "x86_64-unknown-linux-gnu", + Goarch: "amd64", + Sigs: map[string]FuncSig{ + "rawVMOVHPD": {Name: "rawVMOVHPD", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ll, "extractelement <2 x i64>") { + t.Fatalf("raw VMOVHPD did not store the high lane:\n%s", ll) + } +} + +func TestDecodedX86RawVectorHalfMemoryCompleteGo127Family(t *testing.T) { + for _, op := range []Op{"VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD"} { + for _, store := range []bool{false, true} { + for _, encoding := range []string{"VEX2", "VEX3", "EVEX"} { + vector, upper := 3, 4 + if encoding == "VEX3" { + vector, upper = 11, 12 + } + if encoding == "EVEX" { + vector, upper = 20, 21 + } + code := encodeX86RawVectorHalfMemory(op, encoding, vector, upper, store, encoding == "VEX3") + got, length, recognized, err := decodedX86RawVectorHalfMemoryInstruction(code, 64) + if err != nil || !recognized || length != len(code) || got.Op != op { + t.Fatalf("decode %s %s store=%v bytes=%x: got=%+v length=%d recognized=%v err=%v", op, encoding, store, code, got, length, recognized, err) + } + memoryIndex := 0 + registerIndex := 2 + if store { + memoryIndex, registerIndex = 1, 0 + } + if got.Args[memoryIndex].Kind != OpMem || got.Args[registerIndex].Reg != Reg(fmt.Sprintf("X%d", vector)) { + t.Fatalf("decoded wrong memory/register operands for %x: %+v", code, got.Args) + } + memory := got.Args[memoryIndex].Mem + if encoding == "VEX2" && (memory.Base != DX || memory.Off != 0) { + t.Fatalf("decoded wrong VEX2 memory operand for %x: %+v", code, memory) + } + if encoding == "VEX3" && (memory.Base != "R12" || memory.Index != "R13" || memory.Scale != 4 || memory.Off != 2) { + t.Fatalf("decoded wrong VEX3 memory operand for %x: %+v", code, memory) + } + if encoding == "EVEX" && (memory.Base != "R12" || memory.Index != "R13" || memory.Scale != 4 || memory.Off != 16) { + t.Fatalf("decoded wrong EVEX memory operand for %x: %+v", code, memory) + } + if !store && got.Args[1].Reg != Reg(fmt.Sprintf("X%d", upper)) { + t.Fatalf("decoded wrong upper-lane register for %x: %+v", code, got.Args) + } + } + } + } +} + +func TestDecodedX86RawVectorHalfMemoryRejectsInvalidForms(t *testing.T) { + valid := encodeX86RawVectorHalfMemory("VMOVHPD", "EVEX", 20, 21, false, false) + for _, test := range []struct { + name string + code []byte + }{ + {"missing ModRM", valid[:5]}, + {"wrong fixed bit", append([]byte(nil), valid...)}, + {"wrong W bit", append([]byte(nil), valid...)}, + {"vector length", append([]byte(nil), valid...)}, + {"mask", append([]byte(nil), valid...)}, + {"zeroing", append([]byte(nil), valid...)}, + {"broadcast", append([]byte(nil), valid...)}, + {"register operand", append([]byte(nil), valid...)}, + {"RIP relative", append([]byte(nil), valid...)}, + } { + switch test.name { + case "wrong fixed bit": + test.code[2] &^= 4 + case "wrong W bit": + test.code[2] &^= 0x80 + case "vector length": + test.code[3] |= 0x20 + case "mask": + test.code[3] |= 1 + case "zeroing": + test.code[3] |= 0x80 + case "broadcast": + test.code[3] |= 0x10 + case "register operand": + test.code[5] |= 0xc0 + case "RIP relative": + test.code[5] = 0x05 + } + t.Run(test.name, func(t *testing.T) { + if _, _, recognized, err := decodedX86RawVectorHalfMemoryInstruction(test.code, 64); !recognized || err == nil { + t.Fatalf("accepted invalid half-vector encoding %x: recognized=%v err=%v", test.code, recognized, err) + } + }) + } + store := encodeX86RawVectorHalfMemory("VMOVHPD", "VEX2", 3, 0, true, false) + store[1] &^= 0x08 // A non-reserved VEX.vvvv field is not a Go store form. + if _, _, recognized, err := decodedX86RawVectorHalfMemoryInstruction(store, 64); !recognized || err == nil { + t.Fatalf("accepted store with non-reserved vvvv: recognized=%v err=%v", recognized, err) + } + extended := encodeX86RawVectorHalfMemory("VMOVHPD", "VEX3", 11, 12, false, false) + if _, _, recognized, err := decodedX86RawVectorHalfMemoryInstruction(extended, 32); !recognized || err == nil { + t.Fatalf("accepted 386 extended register form: recognized=%v err=%v", recognized, err) + } +} + +func TestTranslateRawX86VectorHalfMemoryLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawHalfMemory(SB),$0-0\n") + if target.goarch == "386" { + source.WriteString("\tMOVL $0, DX\n") + } else { + source.WriteString("\tMOVQ $0, DX\n") + } + for _, op := range []Op{"VMOVLPS", "VMOVHPS", "VMOVLPD", "VMOVHPD"} { + for _, store := range []bool{false, true} { + encodings := []string{"VEX2"} + if target.goarch == "amd64" { + encodings = append(encodings, "VEX3", "EVEX") + } + for _, encoding := range encodings { + vector, upper := 3, 4 + if encoding == "VEX3" { + vector, upper = 11, 12 + } + if encoding == "EVEX" { + vector, upper = 20, 21 + } + for _, value := range encodeX86RawVectorHalfMemory(op, encoding, vector, upper, store, false) { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ll, err := Translate(file, Options{ + TargetTriple: target.triple, + Goarch: target.goarch, + Sigs: map[string]FuncSig{ + "rawHalfMemory": {Name: "rawHalfMemory", Ret: Void}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"extractelement <2 x i64>", "insertelement <2 x i64>"} { + if !strings.Contains(ll, want) { + t.Fatalf("half-vector lowering omitted %q", want) + } + } + compileLLVMToObject(t, llc, target.triple, "raw-half-memory.ll", "raw-half-memory.o", ll) + }) + } +} diff --git a/amd64_raw_vmovq_test.go b/amd64_raw_vmovq_test.go new file mode 100644 index 000000000..9b8fb18ea --- /dev/null +++ b/amd64_raw_vmovq_test.go @@ -0,0 +1,191 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestTranslateRawVEXVMOVDWeaviateRegression(t *testing.T) { + const source = `TEXT rawVMOVD(SB), $0-0 + LONG $0x7e79c1c4 + BYTE $0xc7 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawVMOVD": {Name: "rawVMOVD", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "extractelement <4 x i32>") { + t.Fatalf("raw VMOVD lowering omitted scalar extraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-vex-vmovd.ll", "amd64-raw-vex-vmovd.o", ir) +} + +func TestDecodedX86VMOVQInstructionCompleteVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + want string + }{ + {name: "x_to_gp", code: []byte{0xc4, 0xe1, 0xf9, 0x7e, 0xc8}, want: "VMOVQ X1, AX"}, + {name: "x_to_x", code: []byte{0xc4, 0xe1, 0x79, 0xd6, 0xc8}, want: "VMOVQ X1, X0"}, + {name: "gp_to_x", code: []byte{0xc4, 0xe1, 0xf9, 0x6e, 0xc8}, want: "VMOVQ AX, X1"}, + {name: "x_to_x_f3", code: []byte{0xc5, 0xfa, 0x7e, 0xc8}, want: "VMOVQ X0, X1"}, + {name: "memory_to_extended_x", code: []byte{0xc5, 0x7a, 0x7e, 0x5e, 0x20}, want: "VMOVQ 32(SI), X11"}, + {name: "extended_x_to_sib_memory", code: []byte{0xc4, 0x21, 0xf9, 0x7e, 0x94, 0x8c, 0x78, 0x56, 0x34, 0x12}, want: "VMOVQ X10, 305419896(SP)(R9*4)"}, + {name: "reported_weaviate_vmovd", code: []byte{0xc4, 0xc1, 0x79, 0x7e, 0xc7}, want: "VMOVD X0, R15"}, + {name: "vmovd_extended_gp_to_x", code: []byte{0xc4, 0xc1, 0x79, 0x6e, 0xcf}, want: "VMOVD R15, X1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VMOVQInstruction(test.code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func encodeX86VEXVMOVDQ(opcode, pp byte, vex3, width64 bool, vector, rm int) []byte { + modRM := byte(0xc0 | (vector&7)<<3 | rm&7) + if !vex3 { + vex1 := byte((1-vector/8)<<7 | 15<<3 | int(pp)) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-vector/8)<<7 | 1<<6 | (1-rm/8)<<5 | 1) + vex1 := byte(15<<3 | int(pp)) + if width64 { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestDecodedX86VMOVDQCompleteVEXRegisterFamily(t *testing.T) { + tests := []struct { + opcode byte + pp byte + width64 bool + op Op + reverse bool + vectorRM bool + vex2 bool + }{ + {opcode: 0x7e, pp: 1, op: "VMOVD", vex2: true}, + {opcode: 0x6e, pp: 1, op: "VMOVD", reverse: true, vex2: true}, + {opcode: 0x7e, pp: 1, width64: true, op: "VMOVQ"}, + {opcode: 0xd6, pp: 1, op: "VMOVQ", vectorRM: true, vex2: true}, + {opcode: 0x6e, pp: 1, width64: true, op: "VMOVQ", reverse: true}, + {opcode: 0x7e, pp: 2, op: "VMOVQ", reverse: true, vectorRM: true, vex2: true}, + } + count := 0 + for _, test := range tests { + for _, vex3 := range []bool{false, true} { + if !vex3 && !test.vex2 { + continue + } + rmLimit := 16 + if !vex3 { + rmLimit = 8 + } + for vector := 0; vector < 16; vector++ { + for rm := 0; rm < rmLimit; rm++ { + code := encodeX86VEXVMOVDQ(test.opcode, test.pp, vex3, test.width64, vector, rm) + got, length, ok, err := decodedX86VMOVQInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + rmName := fmt.Sprintf("R%d", rm) + if rm < 8 { + rmName = [...]string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"}[rm] + } + if test.vectorRM { + rmName = fmt.Sprintf("X%d", rm) + } + wantArgs := []string{fmt.Sprintf("X%d", vector), rmName} + if test.reverse { + wantArgs[0], wantArgs[1] = wantArgs[1], wantArgs[0] + } + if got.Op != test.op || len(got.Args) != 2 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, test.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + if count != 2048 { + t.Fatalf("covered %d VEX VMOVD/VMOVQ register encodings, want 2048", count) + } +} + +func TestDecodedX86VMOVDQCompleteGo127EVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + want string + }{ + {name: "d high x to gp", code: []byte{0x62, 0x61, 0x7d, 0x08, 0x7e, 0xf8}, want: "VMOVD X31, AX"}, + {name: "d gp to high x", code: []byte{0x62, 0x61, 0x7d, 0x08, 0x6e, 0xf8}, want: "VMOVD AX, X31"}, + {name: "q high x to gp", code: []byte{0x62, 0x61, 0xfd, 0x08, 0x7e, 0xf8}, want: "VMOVQ X31, AX"}, + {name: "q x to high x", code: []byte{0x62, 0x91, 0xfd, 0x08, 0xd6, 0xc7}, want: "VMOVQ X0, X31"}, + {name: "q gp to high x", code: []byte{0x62, 0x61, 0xfd, 0x08, 0x6e, 0xf8}, want: "VMOVQ AX, X31"}, + {name: "q high x to x alternate", code: []byte{0x62, 0xf1, 0xfe, 0x08, 0x7e, 0xc7}, want: "VMOVQ X7, X0"}, + {name: "d high x to compressed memory", code: []byte{0x62, 0x61, 0x7d, 0x08, 0x7e, 0x78, 0x7f}, want: "VMOVD X31, 508(AX)"}, + {name: "q compressed memory to high x", code: []byte{0x62, 0x61, 0xfd, 0x08, 0x6e, 0x78, 0x7f}, want: "VMOVQ 1016(AX), X31"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VMOVQInstruction(test.code, 64) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as %+v length=%d ok=%v, want %q", test.code, instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodedX86VMOVQInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc5, 0xfa, 0x7e}, mode: 64}, + {name: "vex_l_one", code: []byte{0xc5, 0xfe, 0x7e, 0xc8}, mode: 64}, + {name: "used_vvvv", code: []byte{0xc5, 0xf2, 0x7e, 0xc8}, mode: 64}, + {name: "w_one_in_386", code: []byte{0xc4, 0xe1, 0xf9, 0x6e, 0xc8}, mode: 32}, + {name: "extended_register_in_386", code: []byte{0xc5, 0x7a, 0x7e, 0xc0}, mode: 32}, + {name: "rip_relative", code: []byte{0xc5, 0xfa, 0x7e, 0x05, 0, 0, 0, 0}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VMOVQInstruction(test.code, test.mode) + if !ok { + t.Fatal("VMOVQ-family encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} diff --git a/amd64_raw_vmovw_test.go b/amd64_raw_vmovw_test.go new file mode 100644 index 000000000..f1ee90f31 --- /dev/null +++ b/amd64_raw_vmovw_test.go @@ -0,0 +1,392 @@ +package plan9asm + +import ( + "fmt" + "runtime" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Intel AVX512-FP16 section 5.61: EVEX.128.66.MAP5.WIG 6E/7E. +// The ModRM.reg operand is always X; ModRM.r/m is GP or m16, never X. +func rawVMOVW(load, wide bool, vector, general int, memory bool) []byte { + p0 := byte(0xf5) + p0 &^= byte((vector>>3&1)<<7 | (vector>>4&1)<<4 | (general>>3&1)<<5) + p1 := byte(0x7d) + if wide { + p1 |= 0x80 + } + opcode := byte(0x7e) + if load { + opcode = 0x6e + } + modRM := byte(0xc0 | (vector&7)<<3 | general&7) + if memory { + modRM = 0x40 | byte(vector&7)<<3 | byte(general&7) + } + code := []byte{0x62, p0, p1, 0x08, opcode, modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestRawVMOVWCompleteFormats(t *testing.T) { + for _, mode := range []int{32, 64} { + vectors, generals := 8, 8 + if mode == 64 { + vectors, generals = 32, 16 + } + for _, load := range []bool{false, true} { + for _, wide := range []bool{false, true} { + for vector := 0; vector < vectors; vector++ { + for general := 0; general < generals; general++ { + code := rawVMOVW(load, wide, vector, general, false) + got, n, ok, err := decodedX86VMOVQInstruction(code, mode) + if err != nil || !ok || n != len(code) || got.Op != "VMOVW" || !got.x86Encoded { + t.Fatalf("mode %d %x: %+v, n=%d, matched=%v, err=%v", mode, code, got, n, ok, err) + } + xIndex := 0 + if load { + xIndex = 1 + } + wantGeneral, _ := decodedX86GeneralRegister(general) + if got.Args[xIndex].String() != fmt.Sprintf("X%d", vector) || got.Args[1-xIndex].Kind != OpReg || + got.Args[1-xIndex].Reg != wantGeneral { + t.Fatalf("%x: wrong operands %+v", code, got.Args) + } + } + } + code := rawVMOVW(load, wide, 1, 0, true) + got, _, ok, err := decodedX86VMOVQInstruction(code, mode) + if err != nil || !ok { + t.Fatalf("%x: matched=%v, err=%v", code, ok, err) + } + memoryIndex := 1 + if load { + memoryIndex = 0 + } + if got.Args[memoryIndex].String() != "2(AX)" { + t.Fatalf("%x: expected two-byte displacement scaling, got %+v", code, got.Args) + } + } + } + } +} + +func TestRawVMOVWInvalidFormats(t *testing.T) { + for _, change := range []struct { + index int + bits byte + }{ + {2, 4}, {2, 8}, {3, 8}, {3, 1}, {3, 0x80}, {3, 0x10}, {3, 0x20}, {3, 0x40}, {1, 0x40}, + } { + code := rawVMOVW(true, false, 0, 1, false) + code[change.index] ^= change.bits + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, "invalid VMOVW", map[string]bool{}); err == nil { + t.Fatalf("accepted reserved VMOVW %x", code) + } + } + for _, code := range [][]byte{rawVMOVW(true, false, 8, 0, false), rawVMOVW(false, true, 0, 8, false)} { + if _, err := decodeX86RawDirectiveGroup(code, 32, 0, "386 high VMOVW", map[string]bool{}); err == nil { + t.Fatalf("accepted 386 extended register %x", code) + } + } +} + +func TestRawVMOVWLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + type form struct { + name string + code []byte + load, memory bool + } + var forms []form + var complete []byte + for _, load := range []bool{false, true} { + for _, wide := range []bool{false, true} { + for _, memory := range []bool{false, true} { + code := rawVMOVW(load, wide, 1, 0, memory) + forms = append(forms, form{fmt.Sprintf("rawWordForm%d", len(forms)), code, load, memory}) + complete = append(complete, code...) + } + } + } + var source strings.Builder + // Retain the original eight-form, sequential function. Independent + // one-form functions prevent an unrelated transfer from satisfying a + // width, native-memory effect or GP-zero-extension assertion. + source.WriteString("TEXT rawWordMove(SB),4,$0-0\n") + source.WriteString(x86RawDisplacementDirectives(complete)) + source.WriteString("RET\n") + sigs := map[string]FuncSig{"rawWordMove": {Name: "rawWordMove", Ret: Void}} + for _, f := range forms { + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-0\n", f.name) + source.WriteString(x86RawDisplacementDirectives(f.code)) + source.WriteString("RET\n") + sigs[f.name] = FuncSig{Name: f.name, Ret: Void} + } + for _, arch := range []string{"amd64", "386"} { + // BYTE acceptance alone is not an encoding oracle: require the exact + // contiguous stock-Go object text, including the original final RET. + x86HandoffOriginalBytes(t, arch, append(append([]byte(nil), complete...), 0xc3)) + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-apple-darwin"}, {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + goAssemblerOracleOutput(t, target.arch, source.String(), "", true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + ctx := llvm.NewContext() + defer ctx.Dispose() + module, err := parseIRModuleInContext(ctx, ir) + if !module.IsNil() { + defer module.Dispose() + } + if err != nil { + t.Fatal(err) + } + if err := rawVMOVWFunctionEffects(module, "rawWordMove", 2, 2, 2); err != nil { + t.Fatal(err) + } + mode := 64 + if target.arch == "386" { + mode = 32 + } + for _, f := range forms { + t.Run(f.name, func(t *testing.T) { + ins, size, known, err := decodedX86VMOVQInstruction(f.code, mode) + if err != nil || !known || size != len(f.code) || ins.Op != "VMOVW" || !ins.x86Encoded { + t.Fatalf("raw-only decoder receipt changed: %+v size=%d known=%t err=%v", ins, size, known, err) + } + properties, known, err := x86GPSIMDTransferProperties(ins, target.arch) + if err != nil || !known || properties.bits != 16 || !properties.spec.rawOnly || properties.mmx || + properties.destinationX != f.load || properties.sourceX == f.load || (properties.memory >= 0) != f.memory || + properties.memoryWrite != (f.memory && !f.load) { + t.Fatalf("16-bit raw-only transfer role changed: %+v known=%t err=%v", properties, known, err) + } + loads, stores, extensions := 0, 0, 0 + if f.memory { + if f.load { + loads = 1 + } else { + stores = 1 + } + } else if !f.load { + extensions = 1 + } + if err := rawVMOVWFunctionEffects(module, f.name, loads, stores, extensions); err != nil { + t.Fatal(err) + } + }) + } + rawVMOVWStoreCheckerMutations(t, ctx, module) + compileLLVMToObject(t, llc, target.triple, "raw-vmovw.ll", "raw-vmovw.o", ir) + }) + } +} + +func rawVMOVWFunctionEffects(module llvm.Module, name string, wantLoads, wantStores, wantExtensions int) error { + function := module.NamedFunction(name) + if function.IsNil() { + return fmt.Errorf("missing raw VMOVW function %s", name) + } + feature := false + for _, attribute := range function.GetFunctionAttributes() { + if attribute.IsString() && attribute.GetStringKind() == "target-features" { + for _, item := range strings.Split(attribute.GetStringValue(), ",") { + feature = feature || item == "+avx512fp16" + } + } + } + if !feature { + return fmt.Errorf("%s lost its private raw VMOVW ISA attribute", name) + } + loads, stores, extensions := 0, 0, 0 + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for ins := block.FirstInstruction(); !ins.IsNil(); ins = llvm.NextInstruction(ins) { + if !ins.IsAZExtInst().IsNil() && ins.Operand(0).Type().TypeKind() == llvm.IntegerTypeKind && + ins.Operand(0).Type().IntTypeWidth() == 16 && ins.Type().IntTypeWidth() == 64 { + extensions++ + } + var typ llvm.Type + var pointer llvm.Value + load := !ins.IsALoadInst().IsNil() + if load { + typ, pointer = ins.Type(), ins.Operand(0) + } else if !ins.IsAStoreInst().IsNil() { + typ, pointer = ins.Operand(0).Type(), ins.Operand(1) + } else { + continue + } + if x86GPSIMDStoreTargetsBackingAlloca(pointer) { + continue // Virtual register banks are not original native memory. + } + if typ.TypeKind() != llvm.IntegerTypeKind || typ.IntTypeWidth() != 16 || !ins.IsVolatile() || + ins.Alignment() != 1 || pointer.IsAIntToPtrInst().IsNil() { + return fmt.Errorf("%s lost exact native m16 type/effect/address: %s", name, ins.String()) + } + if load { + loads++ + } else { + stores++ + } + } + } + if loads != wantLoads || stores != wantStores || extensions != wantExtensions { + return fmt.Errorf("%s m16 loads/stores/GP zero-extensions=%d/%d/%d, want=%d/%d/%d", + name, loads, stores, extensions, wantLoads, wantStores, wantExtensions) + } + return nil +} + +func rawVMOVWStoreCheckerMutations(t *testing.T, ctx llvm.Context, module llvm.Module) { + t.Helper() + function := module.NamedFunction("rawWordForm1") // X1 -> m16, W=0. + var store llvm.Value + for block := function.FirstBasicBlock(); !block.IsNil(); block = llvm.NextBasicBlock(block) { + for ins := block.FirstInstruction(); !ins.IsNil(); ins = llvm.NextInstruction(ins) { + if !ins.IsAStoreInst().IsNil() && !x86GPSIMDStoreTargetsBackingAlloca(ins.Operand(1)) { + if !store.IsNil() { + t.Fatal("mutation fixture has multiple native stores") + } + store = ins + } + } + } + if store.IsNil() { + t.Fatal("mutation fixture has no native store") + } + original := function.String() + for _, mutation := range []struct{ name, replacement string }{ + {"volatility", strings.Replace(store.String(), "store volatile", "store", 1)}, + {"width", "store volatile i32 1" + store.String()[strings.IndexByte(store.String(), ','):]}, + {"alignment", strings.Replace(store.String(), "align 1", "align 2", 1)}, + {"missing", "; removed original native m16 store"}, + } { + t.Run("checker-"+mutation.name, func(t *testing.T) { + if strings.Count(original, store.String()) != 1 || strings.Count(module.String(), original) != 1 { + t.Fatal("mutation does not bind the exact one-form function/store") + } + changed := strings.Replace(module.String(), original, strings.Replace(original, store.String(), mutation.replacement, 1), 1) + candidate, err := parseIRModuleInContext(ctx, changed) + if !candidate.IsNil() { + defer candidate.Dispose() + } + if err != nil { + t.Fatal("mutation must remain valid LLVM IR", err) + } + if err := llvm.VerifyModule(candidate, llvm.ReturnStatusAction); err != nil { + t.Fatal("mutation must remain a valid LLVM module", err) + } + if err := rawVMOVWFunctionEffects(candidate, "rawWordForm1", 0, 1, 0); err == nil { + t.Fatal("native m16 mutation escaped the per-function checker") + } + }) + } +} + +func TestRawVMOVWPortableRuntime(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + source.WriteString("TEXT wordMove(SB),4,$0-24\nMOVQ input+0(FP), AX\nMOVQ out+8(FP), SI\nMOVQ vector+16(FP), DI\nVMOVDQU64 (DI), Z0\nMOVQ $-1, CX\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7d, 0x08, 0x6e, 0x00}, // m16 -> X0; no wider memory access. + {0x62, 0xf5, 0xfd, 0x08, 0x7e, 0xc1}, // X0 -> CX; WIG, zero-extend. + } { + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + source.WriteString("MOVQ CX, (SI)\nVMOVDQU64 Z0, 8(SI)\nMOVQ $-1, CX\n") + for _, code := range [][]byte{ + {0x62, 0xf5, 0x7d, 0x08, 0x6e, 0xc1}, // CX -> X0, truncate and clear all upper lanes. + {0x62, 0xf5, 0xfd, 0x08, 0x7e, 0x00}, // X0 -> m16; exactly two bytes. + } { + for _, b := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", b) + } + } + source.WriteString("VMOVDQU64 Z0, 72(SI)\nRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: testTargetTriple(runtime.GOOS, runtime.GOARCH), + Sigs: map[string]FuncSig{"wordMove": { + Name: "wordMove", Args: []LLVMType{Ptr, Ptr, Ptr}, Ret: Void, Attrs: "nounwind", + Frame: FrameLayout{Params: []FrameSlot{ + {Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}, + {Offset: 16, Type: Ptr, Index: 2, Field: -1}, + }}, + }}, + }) + if err != nil { + t.Fatal(err) + } + mainC := "#include \n#include \n#include \n#include \n" + x86TestGuardPagesC + ` +extern void wordMove(void *, void *, const void *); +int main(void) { + if (setup_guard()) return 1; + uint64_t output[17], vector[8]; + memset(vector, 0xff, sizeof(vector)); + for (int offset = 2; offset <= 3; offset++) { + unsigned char *pointer = guard + page - offset; + uint16_t input = 0x8123; + memcpy(pointer, &input, 2); + wordMove(pointer, output, vector); + if (output[0] != input || output[1] != input || output[9] != 0xffff) return 2; + for (int lane = 2; lane < 9; lane++) if (output[lane] != 0) return 3; + for (int lane = 10; lane < 17; lane++) if (output[lane] != 0) return 4; + memcpy(&input, pointer, 2); + if (input != 0xffff) return 5; + } + return free_guard(); +} +` + compileAndRunRuntimeTest(t, llc, clang, "vmovw_portable", ir, mainC) +} + +func TestRawVMOVWRIPLiteralAndNamedRejection(t *testing.T) { + code := []byte{0x62, 0xf5, 0x7d, 8, 0x6e, 5, 1, 0, 0, 0, 0xc3, 0x23, 0x81} + got, err := decodeX86RawDirectiveGroup(code, 64, 0, "word RIP literal", map[string]bool{}) + if err != nil || len(got) != 2 || !got[0].x86RIPLiteral || got[0].Args[0].Kind != OpImm || got[0].Args[0].Imm != 0x8123 { + t.Fatalf("literal: %+v, err=%v", got, err) + } + if _, err := decodeX86RawDirectiveGroup(code[:len(code)-1], 64, 0, "truncated word pool", map[string]bool{}); err == nil { + t.Fatal("accepted truncated word pool") + } + const source = "TEXT namedWordMove(SB),4,$0-0\nVMOVW X0, AX\nRET\n" + for _, arch := range []string{"amd64", "386"} { + goAssemblerOracleOutput(t, arch, source, "", false) + t.Run(arch+"/named-three-APIs", func(t *testing.T) { + x86RequireZeroMaskAPIOutcome(t, arch, strings.Replace(source, "namedWordMove", "f", 1), false) + }) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + if _, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", + Sigs: map[string]FuncSig{"namedWordMove": {Name: "namedWordMove", Ret: Void}}, + }); err == nil { + t.Fatal("accepted VMOVW absent from Go's named assembler table") + } +} diff --git a/amd64_raw_vnni_test.go b/amd64_raw_vnni_test.go new file mode 100644 index 000000000..29b03b3db --- /dev/null +++ b/amd64_raw_vnni_test.go @@ -0,0 +1,125 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VNNIInstructionCompleteVEXFamily(t *testing.T) { + for _, test := range []struct { + opcode byte + op Op + }{ + {0x50, "VPDPBUSD"}, + {0x51, "VPDPBUSDS"}, + {0x52, "VPDPWSSD"}, + {0x53, "VPDPWSSDS"}, + } { + t.Run(string(test.op), func(t *testing.T) { + code := []byte{0xc4, 0xe2, 0x71, test.opcode, 0xc2} + instruction, length, ok, err := decodedX86VNNIInstruction(code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(code) { + t.Fatalf("length = %d, want %d", length, len(code)) + } + want := string(test.op) + " X2, X1, X0" + if instruction.Op != test.op || instruction.Raw != want { + t.Fatalf("instruction = %#v, want %q", instruction, want) + } + }) + } + + for _, test := range []struct { + name string + code []byte + mode int + want string + }{ + { + name: "y_extended_registers", + code: []byte{0xc4, 0x42, 0x15, 0x50, 0xdc}, + mode: 64, + want: "VPDPBUSD Y12, Y13, Y11", + }, + { + name: "x_memory_sib_disp8", + code: []byte{0xc4, 0x62, 0x11, 0x50, 0x5c, 0x88, 0x20}, + mode: 64, + want: "VPDPBUSD 32(AX)(CX*4), X13, X11", + }, + { + name: "y_memory_negative_disp8", + code: []byte{0xc4, 0x62, 0x65, 0x51, 0x64, 0x35, 0xef}, + mode: 64, + want: "VPDPBUSDS -17(BP)(SI*1), Y3, Y12", + }, + { + name: "x_memory_extended_address", + code: []byte{0xc4, 0x82, 0x39, 0x52, 0x74, 0x7e, 0x07}, + mode: 64, + want: "VPDPWSSD 7(R14)(R15*2), X8, X6", + }, + { + name: "y_386_memory", + code: []byte{0xc4, 0xe2, 0x65, 0x53, 0x64, 0x35, 0xef}, + mode: 32, + want: "VPDPWSSDS -17(BP)(SI*1), Y3, Y4", + }, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VNNIInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode = (%#v, %d, %v), want %q", instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodedX86VNNIInstructionRejectsInvalidVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe2, 0x71, 0x50}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc4, 0xe2, 0x70, 0x50, 0xc2}, mode: 64}, + {name: "vex_w_one", code: []byte{0xc4, 0xe2, 0xf1, 0x50, 0xc2}, mode: 64}, + {name: "address_override", code: []byte{0x67, 0xc4, 0xe2, 0x71, 0x50, 0xc2}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x42, 0x15, 0x50, 0xdc}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe2, 0x71, 0x50, 0x05, 0, 0, 0, 0}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VNNIInstruction(test.code, test.mode) + if !ok { + t.Fatal("VNNI encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVNNIReportedSequence(t *testing.T) { + code := []byte{ + 0xc4, 0x42, 0x15, 0x50, 0xdc, + 0xc4, 0x62, 0x7d, 0x50, 0xc7, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "go-pherence VEX VNNI sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 { + t.Fatalf("decoded %d instructions, want 2: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPDPBUSD" { + t.Fatalf("decoded op = %s, want VPDPBUSD", instruction.Op) + } + } +} diff --git a/amd64_raw_vpadd_test.go b/amd64_raw_vpadd_test.go new file mode 100644 index 000000000..75be16de6 --- /dev/null +++ b/amd64_raw_vpadd_test.go @@ -0,0 +1,233 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86VEXPackedIntegerArithmetic(opcode byte, vex3, width256, widthIgnored bool, destination, source1, source2 int) []byte { + l := byte(0) + if width256 { + l = 1 + } + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + if !vex3 { + vex1 := byte((1-destination/8)<<7 | ((^source1)&15)<<3 | int(l)<<2 | 1) + return []byte{0xc5, vex1, opcode, modRM} + } + vex0 := byte((1-destination/8)<<7 | 1<<6 | (1-source2/8)<<5 | 1) + vex1 := byte(((^source1)&15)<<3 | int(l)<<2 | 1) + if widthIgnored { + vex1 |= 0x80 + } + return []byte{0xc4, vex0, vex1, opcode, modRM} +} + +func TestDecodedX86PackedIntegerAddCompleteVEXRegisterFamily(t *testing.T) { + count := 0 + for opcode, properties := range decodedX86PackedIntegerArithmeticOps { + if !strings.HasPrefix(string(properties.op), "VPADD") { + continue + } + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedIntegerArithmetic(opcode, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != properties.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, properties.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 163840 { + t.Fatalf("covered %d VEX packed integer add register encodings, want 163840", count) + } +} + +func TestDecodedX86PackedIntegerSubtractCompleteVEXRegisterFamily(t *testing.T) { + subtractOps := map[byte]struct { + op Op + laneBytes int + }{ + 0xf8: {op: "VPSUBB", laneBytes: 1}, + 0xf9: {op: "VPSUBW", laneBytes: 2}, + 0xfa: {op: "VPSUBD", laneBytes: 4}, + 0xfb: {op: "VPSUBQ", laneBytes: 8}, + 0xe8: {op: "VPSUBSB", laneBytes: 1}, + 0xe9: {op: "VPSUBSW", laneBytes: 2}, + 0xd8: {op: "VPSUBUSB", laneBytes: 1}, + 0xd9: {op: "VPSUBUSW", laneBytes: 2}, + } + count := 0 + for opcode, properties := range subtractOps { + for _, vex3 := range []bool{false, true} { + for _, width256 := range []bool{false, true} { + prefix := "X" + if width256 { + prefix = "Y" + } + for _, widthIgnored := range []bool{false, true} { + if !vex3 && widthIgnored { + continue + } + source2Limit := 16 + if !vex3 { + source2Limit = 8 + } + for destination := 0; destination < 16; destination++ { + for source1 := 0; source1 < 16; source1++ { + for source2 := 0; source2 < source2Limit; source2++ { + code := encodeX86VEXPackedIntegerArithmetic(opcode, vex3, width256, widthIgnored, destination, source1, source2) + got, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != properties.op || len(got.Args) != 3 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, properties.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + } + } + if count != 163840 { + t.Fatalf("covered %d VEX packed integer subtract register encodings, want 163840", count) + } +} + +func TestDecodedX86PackedIntegerArithmeticCompleteEVEXRegisterFamily(t *testing.T) { + count := 0 + for opcode, properties := range decodedX86PackedIntegerArithmeticOps { + for vectorBits := byte(0); vectorBits < 3; vectorBits++ { + prefix := [...]string{"X", "Y", "Z"}[vectorBits] + for destination := 0; destination < 32; destination++ { + for source1 := 0; source1 < 32; source1++ { + for source2 := 0; source2 < 32; source2++ { + p0 := byte((1-(destination>>3)&1)<<7 | (1-(source2>>4)&1)<<6 | (1-(source2>>3)&1)<<5 | (1-(destination>>4)&1)<<4 | 1) + p1 := byte((^source1&15)<<3 | 1<<2 | 1) + if properties.laneBytes == 8 { + p1 |= 0x80 + } + p2 := byte(int(vectorBits)<<5 | (^source1>>4&1)<<3 | 1) + modRM := byte(0xc0 | (destination&7)<<3 | source2&7) + code := []byte{0x62, p0, p1, p2, opcode, modRM} + got, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x returned %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantArgs := []string{fmt.Sprintf("%s%d", prefix, source2), fmt.Sprintf("%s%d", prefix, source1), "K1", fmt.Sprintf("%s%d", prefix, destination)} + if got.Op != properties.op || len(got.Args) != 4 || got.Args[0].String() != wantArgs[0] || got.Args[1].String() != wantArgs[1] || got.Args[2].String() != wantArgs[2] || got.Args[3].String() != wantArgs[3] { + t.Fatalf("decode %x = %+v, want %s %s", code, got, properties.op, strings.Join(wantArgs, ", ")) + } + count++ + } + } + } + } + } + if count != 1572864 { + t.Fatalf("covered %d EVEX packed integer arithmetic encodings, want 1572864", count) + } +} + +func TestTranslateRawVPSUBDWeaviateRegression(t *testing.T) { + const source = `TEXT rawPackedSubtract(SB), $0-0 + LONG $0xd5faedc5 + RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: "x86_64-unknown-linux-gnu", Sigs: map[string]FuncSig{"rawPackedSubtract": {Name: "rawPackedSubtract", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "sub <8 x i32>") { + t.Fatalf("raw VPSUBD lowering omitted vector subtraction:\n%s", ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + compileLLVMToObject(t, llc, "x86_64-unknown-linux-gnu", "amd64-raw-packed-subtract.ll", "amd64-raw-packed-subtract.o", ir) +} + +func TestDecodedX86PackedIntegerArithmeticCompleteGo127Family(t *testing.T) { + // Go 1.27 uses _yvandnpd for all sixteen VPADD/VPSUB variants. These LLVM 22 + // encodings cover every opcode plus VEX/EVEX X/Y/Z, high registers, + // memory, masking, zeroing, D/Q broadcast, and compressed disp8. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex byte x high", code: []byte{0xc5, 0x79, 0xfc, 0xc1}, want: "VPADDB X1, X0, X8"}, + {name: "vex qword y high memory", code: []byte{0xc4, 0x41, 0x6d, 0xd4, 0x4c, 0x24, 0x20}, want: "VPADDQ 32(R12), Y2, Y9"}, + {name: "byte x high masked", code: []byte{0x62, 0xa1, 0x55, 0x01, 0xfc, 0xf4}, want: "VPADDB X20, X21, K1, X22"}, + {name: "word y memory zeroing", code: []byte{0x62, 0xd1, 0x75, 0xaa, 0xfd, 0x50, 0x03}, want: "VPADDW.Z 96(R8), Y1, K2, Y2"}, + {name: "dword reported minio", code: []byte{0x62, 0x51, 0x45, 0x48, 0xfe, 0xc4}, want: "VPADDD Z12, Z7, Z8"}, + {name: "qword z broadcast zeroing", code: []byte{0x62, 0xf1, 0xf5, 0xdb, 0xd4, 0x50, 0x7f}, want: "VPADDQ.BCST.Z 1016(AX), Z1, K3, Z2"}, + {name: "signed byte z high", code: []byte{0x62, 0x31, 0x7d, 0x40, 0xec, 0xc9}, want: "VPADDSB Z17, Z16, Z9"}, + {name: "signed word x high memory masked", code: []byte{0x62, 0x41, 0x3d, 0x04, 0xed, 0x4c, 0x24, 0x02}, want: "VPADDSW 32(R12), X24, K4, X25"}, + {name: "unsigned byte y high masked", code: []byte{0x62, 0xa1, 0x55, 0x21, 0xdc, 0xf4}, want: "VPADDUSB Y20, Y21, K1, Y22"}, + {name: "unsigned word z memory zeroing", code: []byte{0x62, 0xf1, 0x75, 0xcb, 0xdd, 0x50, 0x7f}, want: "VPADDUSW.Z 8128(AX), Z1, K3, Z2"}, + {name: "subtract dword weaviate", code: []byte{0xc5, 0xed, 0xfa, 0xd5}, want: "VPSUBD Y5, Y2, Y2"}, + {name: "subtract qword broadcast zeroing", code: []byte{0x62, 0xf1, 0xf5, 0xdb, 0xfb, 0x50, 0x7f}, want: "VPSUBQ.BCST.Z 1016(AX), Z1, K3, Z2"}, + {name: "subtract unsigned word z memory zeroing", code: []byte{0x62, 0xf1, 0x75, 0xcb, 0xd9, 0x50, 0x7f}, want: "VPSUBUSW.Z 8128(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedIntegerArithmeticInstruction(test.code, 64) + if !ok { + t.Fatal("VPADD encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPADDD(t *testing.T) { + code := []byte{0x62, 0x51, 0x45, 0x48, 0xfe, 0xc4} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPADDD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPADDD" || !strings.HasPrefix(decoded[0].Raw, "VPADDD Z12, Z7, Z8 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpermd_test.go b/amd64_raw_vpermd_test.go new file mode 100644 index 000000000..924a867e6 --- /dev/null +++ b/amd64_raw_vpermd_test.go @@ -0,0 +1,313 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func encodeX86RawEVEXVariableDwordPermute(opcode, vectorBits, mask int, zeroing, broadcast, memory bool) []byte { + p2 := byte(vectorBits<<5 | 0x08 | mask) + if zeroing { + p2 |= 0x80 + } + if broadcast { + p2 |= 0x10 + } + modRM := byte(0xd3) // data register 3, destination register 2. + if memory { + modRM = 0x50 // disp8(AX) data, destination register 2. + } + code := []byte{0x62, 0xf2, 0x75, p2, byte(opcode), modRM} + if memory { + code = append(code, 1) + } + return code +} + +func TestDecodedX86VariableDwordPermuteInstructionCompleteVEXFamily(t *testing.T) { + for _, test := range []struct { + opcode byte + op Op + }{ + {0x36, "VPERMD"}, + {0x16, "VPERMPS"}, + } { + t.Run(string(test.op), func(t *testing.T) { + code := []byte{0xc4, 0xe2, 0x25, test.opcode, 0xd6} + instruction, length, ok, err := decodedX86VariableDwordPermuteInstruction(code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(code) { + t.Fatalf("length = %d, want %d", length, len(code)) + } + want := string(test.op) + " Y6, Y11, Y2" + if instruction.Op != test.op || instruction.Raw != want { + t.Fatalf("instruction = %#v, want %q", instruction, want) + } + }) + } + + for _, test := range []struct { + name string + code []byte + mode int + want string + }{ + { + name: "extended_registers", + code: []byte{0xc4, 0x42, 0x05, 0x36, 0xdb}, + mode: 64, + want: "VPERMD Y11, Y15, Y11", + }, + { + name: "memory_sib_disp8", + code: []byte{0xc4, 0x62, 0x15, 0x36, 0x5c, 0x88, 0x20}, + mode: 64, + want: "VPERMD 32(AX)(CX*4), Y13, Y11", + }, + { + name: "memory_negative_disp8", + code: []byte{0xc4, 0x62, 0x65, 0x16, 0x64, 0x35, 0xef}, + mode: 64, + want: "VPERMPS -17(BP)(SI*1), Y3, Y12", + }, + { + name: "386_registers", + code: []byte{0xc4, 0xe2, 0x65, 0x36, 0xe5}, + mode: 32, + want: "VPERMD Y5, Y3, Y4", + }, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VariableDwordPermuteInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode = (%#v, %d, %v), want %q", instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodeX86RawEVEXVPERMDSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPERMD Z2, Z0, Z3. + code := []byte{0x62, 0xf2, 0x7d, 0x48, 0x36, 0xda, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "cumMinInt32 AVX512 VPERMD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPERMD" || + decoded[0].Args[0].Reg != "Z2" || decoded[0].Args[1].Reg != "Z0" || + decoded[0].Args[2].Reg != "Z3" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXVariableDwordPermuteCompleteGoForms(t *testing.T) { + for _, family := range []struct { + opcode int + op Op + }{ + {0x36, "VPERMD"}, + {0x16, "VPERMPS"}, + } { + for vectorIndex, vector := range []string{"Y", "Z"} { + vectorBits := vectorIndex + 1 + for _, variant := range []struct { + name string + mask int + zeroing bool + broadcast bool + memory bool + }{ + {name: "register"}, + {name: "masked register", mask: 3}, + {name: "memory", memory: true}, + {name: "zeroing memory", mask: 7, zeroing: true, memory: true}, + {name: "broadcast memory", broadcast: true, memory: true}, + {name: "broadcast zeroing memory", mask: 1, zeroing: true, broadcast: true, memory: true}, + } { + name := fmt.Sprintf("%s/%s/%s", family.op, vector, variant.name) + t.Run(name, func(t *testing.T) { + code := encodeX86RawEVEXVariableDwordPermute(family.opcode, vectorBits, variant.mask, variant.zeroing, variant.broadcast, variant.memory) + got, length, ok, err := decodedX86EVEXVariableDwordPermuteInstruction(code, 64) + if err != nil || !ok || length != len(code) { + t.Fatalf("decode %x = %+v, length=%d, ok=%v, err=%v", code, got, length, ok, err) + } + wantOp := family.op + if variant.broadcast { + wantOp += ".BCST" + } + if variant.zeroing { + wantOp += ".Z" + } + if got.Op != wantOp || got.Args[1].String() != vector+"1" || + got.Args[len(got.Args)-1].String() != vector+"2" || + !variant.memory && got.Args[0].String() != vector+"3" || + len(got.Args) != 3+map[bool]int{true: 1}[variant.mask != 0] { + t.Fatalf("decode %x = %+v, want %s", code, got, wantOp) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXVariableDwordPermuteRIPLiteral(t *testing.T) { + for _, family := range []struct { + opcode int + op Op + }{ + {0x36, "VPERMD"}, + {0x16, "VPERMPS"}, + } { + for vectorBits := 1; vectorBits <= 2; vectorBits++ { + for _, broadcast := range []bool{false, true} { + code := encodeX86RawEVEXVariableDwordPermute(family.opcode, vectorBits, 1, false, broadcast, false) + code[5] = 0x15 // RIP-relative first source. + code = append(code, 1, 0, 0, 0, 0xc3) + literalWidth := 16 << vectorBits + if broadcast { + literalWidth = 4 + } + for index := 0; index < literalWidth; index++ { + code = append(code, byte(index+1)) + } + name := fmt.Sprintf("%s/width%d/broadcast%t", family.op, vectorBits, broadcast) + t.Run(name, func(t *testing.T) { + got, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || !got[0].x86RIPLiteral || + len(got[0].x86RIPLiteralData) != literalWidth { + t.Fatalf("decode %x = %+v", code, got) + } + }) + } + } + } +} + +func TestDecodeX86RawEVEXVariableDwordPermuteRejectsInvalidForms(t *testing.T) { + base := encodeX86RawEVEXVariableDwordPermute(0x36, 2, 0, false, false, false) + for _, mutate := range []func([]byte){ + func(code []byte) { code[2] |= 0x80 }, // Go's EVEX.W is zero + func(code []byte) { code[3] &^= 0x60 }, // no X-width EVEX form + func(code []byte) { code[3] |= 0x80 }, // zeroing without mask + func(code []byte) { code[3] |= 0x10 }, // broadcast requires memory + } { + code := append([]byte(nil), base...) + mutate(code) + if _, _, ok, err := decodedX86EVEXVariableDwordPermuteInstruction(code, 64); !ok || err == nil { + t.Fatalf("invalid encoding %x returned ok=%v err=%v", code, ok, err) + } + } + if _, _, ok, err := decodedX86EVEXVariableDwordPermuteInstruction( + encodeX86RawEVEXVariableDwordPermute(0x16, 2, 1, false, false, false), 32, + ); !ok || err == nil { + t.Fatalf("accepted 386 masked form: ok=%v err=%v", ok, err) + } +} + +func TestTranslateX86RawEVEXVariableDwordPermuteLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + goarch string + triple string + }{ + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + t.Run(target.triple, func(t *testing.T) { + var source strings.Builder + source.WriteString("TEXT rawVariableDwordPermute(SB),$0-0\n") + for _, opcode := range []int{0x36, 0x16} { + for _, vectorBits := range []int{1, 2} { + mask := 1 + if target.goarch == "386" { + mask = 0 + } + for _, memory := range []bool{false, true} { + code := encodeX86RawEVEXVariableDwordPermute(opcode, vectorBits, mask, false, false, memory) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + } + } + source.WriteString("\tRET\n") + requireX86GoAssemblerResult(t, target.goarch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"rawVariableDwordPermute": {Name: "rawVariableDwordPermute", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-vpermd.ll", "raw-vpermd.o", ir) + }) + } +} + +func TestDecodedX86VariableDwordPermuteInstructionRejectsInvalidVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe2, 0x25, 0x36}, mode: 64}, + {name: "vex_l_zero", code: []byte{0xc4, 0xe2, 0x21, 0x36, 0xd6}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc4, 0xe2, 0x24, 0x36, 0xd6}, mode: 64}, + {name: "vex_w_one", code: []byte{0xc4, 0xe2, 0xa5, 0x36, 0xd6}, mode: 64}, + {name: "address_override", code: []byte{0x67, 0xc4, 0xe2, 0x25, 0x36, 0xd6}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x42, 0x05, 0x36, 0xdb}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe2, 0x25, 0x36, 0x05, 0, 0, 0, 0}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VariableDwordPermuteInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPERMD/VPERMPS encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPERMDReportedSequence(t *testing.T) { + code := []byte{ + 0xc4, 0xe2, 0x25, 0x36, 0xd6, + 0xc4, 0xe2, 0x7d, 0x36, 0xd6, + 0xc4, 0xe2, 0x35, 0x36, 0xd6, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "go-pherence VEX VPERMD sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 { + t.Fatalf("decoded %d instructions, want 3: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPERMD" { + t.Fatalf("decoded op = %s, want VPERMD", instruction.Op) + } + } +} diff --git a/amd64_raw_vpermi2_test.go b/amd64_raw_vpermi2_test.go new file mode 100644 index 000000000..53087be30 --- /dev/null +++ b/amd64_raw_vpermi2_test.go @@ -0,0 +1,51 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86VPERMI2CompleteGo127Family(t *testing.T) { + // Go 1.27 shares _yvblendmpd across VPERMI2B/W/D/Q/PS/PD. These + // LLVM-verified bytes cover all six opcodes, X/Y/Z widths, high registers, + // memory, masking, zeroing, broadcast, and compressed disp8 scaling. + tests := []struct { + name string + code []byte + want string + }{ + {name: "b x high masked", code: []byte{0x62, 0xa2, 0x55, 0x01, 0x75, 0xf4}, want: "VPERMI2B X20, X21, K1, X22"}, + {name: "w y memory zeroing", code: []byte{0x62, 0xd2, 0xf5, 0xaa, 0x75, 0x50, 0x03}, want: "VPERMI2W.Z 96(R8), Y1, K2, Y2"}, + {name: "d z high", code: []byte{0x62, 0x32, 0x7d, 0x40, 0x76, 0xc9}, want: "VPERMI2D Z17, Z16, Z9"}, + {name: "d z broadcast zeroing", code: []byte{0x62, 0xf2, 0x75, 0xdb, 0x76, 0x50, 0x7f}, want: "VPERMI2D.BCST.Z 508(AX), Z1, K3, Z2"}, + {name: "q reported minio", code: []byte{0x62, 0x22, 0xb5, 0x48, 0x76, 0xf2}, want: "VPERMI2Q Z18, Z9, Z30"}, + {name: "q y high memory masked", code: []byte{0x62, 0x42, 0xbd, 0x24, 0x76, 0x4c, 0x24, 0x02}, want: "VPERMI2Q 64(R12), Y24, K4, Y25"}, + {name: "ps x high masked", code: []byte{0x62, 0xa2, 0x55, 0x01, 0x77, 0xf4}, want: "VPERMI2PS X20, X21, K1, X22"}, + {name: "pd z broadcast zeroing", code: []byte{0x62, 0xf2, 0xf5, 0xdb, 0x77, 0x50, 0x7f}, want: "VPERMI2PD.BCST.Z 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86IndexedPermuteInstruction(test.code, 64) + if !ok { + t.Fatal("VPERMI2 encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPERMI2Q(t *testing.T) { + code := []byte{0x62, 0x22, 0xb5, 0x48, 0x76, 0xf2} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPERMI2Q", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPERMI2Q" || !strings.HasPrefix(decoded[0].Raw, "VPERMI2Q Z18, Z9, Z30 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpermq_test.go b/amd64_raw_vpermq_test.go new file mode 100644 index 000000000..61ca796fc --- /dev/null +++ b/amd64_raw_vpermq_test.go @@ -0,0 +1,79 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VPERMQInstructionCompleteVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + want string + }{ + {name: "register", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0xc9, 0x39}, mode: 64, want: "VPERMQ $57, Y1, Y1"}, + {name: "extended_registers", code: []byte{0xc4, 0x43, 0xfd, 0x00, 0xca, 0xff}, mode: 64, want: "VPERMQ $255, Y10, Y9"}, + {name: "memory_disp8", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0x4e, 0x20, 0x01}, mode: 64, want: "VPERMQ $1, 32(SI), Y1"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPERMQInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func TestDecodedX86VPERMQInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe3, 0xfd, 0x00}, mode: 64}, + {name: "truncated_immediate", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0xc9}, mode: 64}, + {name: "vex_l_zero", code: []byte{0xc4, 0xe3, 0xf9, 0x00, 0xc9, 0x39}, mode: 64}, + {name: "used_vvvv", code: []byte{0xc4, 0xe3, 0xed, 0x00, 0xc9, 0x39}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x43, 0xfd, 0x00, 0xca, 0xff}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe3, 0xfd, 0x00, 0x0d, 0, 0, 0, 0, 0x39}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VPERMQInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPERMQ encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPERMQSequence(t *testing.T) { + code := []byte{ + 0xc4, 0xe3, 0xfd, 0x00, 0xc9, 0x39, + 0xc4, 0xe3, 0xfd, 0x00, 0xd2, 0x4e, + 0xc4, 0xe3, 0xfd, 0x00, 0xdb, 0x93, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VPERMQ raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 3 { + t.Fatalf("decoded %d instructions, want 3: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPERMQ" { + t.Fatalf("decoded op = %s, want VPERMQ", instruction.Op) + } + } +} diff --git a/amd64_raw_vpinsr_evex.go b/amd64_raw_vpinsr_evex.go new file mode 100644 index 000000000..ee6d7a69e --- /dev/null +++ b/amd64_raw_vpinsr_evex.go @@ -0,0 +1,62 @@ +package plan9asm + +import "fmt" + +// Go 1.27's _yvpinsrb table has one EVEX.128 row for each scalar width. +// VEX forms use decodedX86VPINSRInstruction; the EVEX row extends X +// base/destination registers to X31 and scales disp8 by the scalar width. +func decodedX86EVEXVPINSRInstruction(code []byte, mode int) (Instr, int, bool, error) { + p, matched := decodeX86RawVectorEncoding(code) + if !matched || !p.evex || p.pp != 1 { + return Instr{}, 0, false, nil + } + op := Op("") + laneBytes := 0 + switch { + case p.mapNumber == 3 && p.opcode == 0x20 && !p.w: + op, laneBytes = "VPINSRB", 1 + case p.mapNumber == 1 && p.opcode == 0xc4 && !p.w: + op, laneBytes = "VPINSRW", 2 + case p.mapNumber == 3 && p.opcode == 0x22 && !p.w: + op, laneBytes = "VPINSRD", 4 + case p.mapNumber == 3 && p.opcode == 0x22 && p.w: + op, laneBytes = "VPINSRQ", 8 + default: + return Instr{}, 0, false, nil + } + fail := func(message string) (Instr, int, bool, error) { + return Instr{}, 0, true, fmt.Errorf("raw EVEX %s: %s", op, message) + } + if mode != 64 { + return fail("Go assembler exposes this EVEX form only on amd64") + } + if p.addressOverride { + return fail("address-size override is not source-layout safe") + } + if !p.fixed || p.vectorLength != 0 || p.mask != 0 || p.zero || p.broadcast { + return fail("invalid EVEX fixed, length, mask, zeroing or broadcast bits") + } + if len(code) <= p.modRM { + return fail("missing ModRM byte") + } + modRM := code[p.modRM] + source, consumed, err := decodedX86RMSource( + code[p.modRM:], mode, p.b, p.x, p.segment, laneBytes, + ) + if err != nil { + return Instr{}, 0, true, err + } + immediateIndex := p.modRM + consumed + if len(code) <= immediateIndex { + return fail("missing immediate byte") + } + immediate := Operand{Kind: OpImm, Imm: int64(code[immediateIndex])} + base := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", p.upper))} + destination := Operand{Kind: OpReg, Reg: Reg(fmt.Sprintf("X%d", int(modRM>>3&7)+p.r*8))} + args := []Operand{immediate, source, base, destination} + return Instr{ + Op: op, + Args: args, + Raw: fmt.Sprintf("%s %s, %s, %s, %s", op, immediate.String(), source.String(), base.String(), destination.String()), + }, immediateIndex + 1, true, nil +} diff --git a/amd64_raw_vpinsr_evex_test.go b/amd64_raw_vpinsr_evex_test.go new file mode 100644 index 000000000..dda252a6c --- /dev/null +++ b/amd64_raw_vpinsr_evex_test.go @@ -0,0 +1,122 @@ +package plan9asm + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +var x86RawEVEXVPINSRForms = []struct { + op Op + mapNumber int + opcode byte + width64 bool + laneBytes int +}{ + {"VPINSRB", 3, 0x20, false, 1}, + {"VPINSRW", 1, 0xc4, false, 2}, + {"VPINSRD", 3, 0x22, false, 4}, + {"VPINSRQ", 3, 0x22, true, 8}, +} + +func x86RawEVEXVPINSRGoForms() string { + var source strings.Builder + source.WriteString("TEXT rawVPINSR(SB),4,$0-0\n") + for _, form := range x86RawEVEXVPINSRForms { + fmt.Fprintf(&source, "\t%s $5, AX, X20, X21\n", form.op) + fmt.Fprintf(&source, "\t%s $255, %d(BX), X20, X21\n", form.op, form.laneBytes) + } + source.WriteString("\tRET\n") + return source.String() +} + +func TestDecodeX86RawEVEXVPINSRSeBiShogunRegression(t *testing.T) { + // Go disassembles these bytes as VPINSRB $5, 1(SI)(R9*1), X6, X16. + code := []byte{0x62, 0xa3, 0x4d, 0x08, 0x20, 0x44, 0x0e, 0x01, 0x05, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "bitshuffleU8 AVX512 VPINSRB", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 2 || decoded[0].Op != "VPINSRB" || + decoded[0].Args[0].Imm != 5 || decoded[0].Args[2].Reg != "X6" || + decoded[0].Args[3].Reg != "X16" { + t.Fatalf("decoded %x as %#v", code, decoded) + } +} + +func TestDecodeX86RawEVEXVPINSRCompleteGoForms(t *testing.T) { + if len(x86RawEVEXVPINSRForms) != 4 { + t.Fatalf("EVEX VPINSR grammar has %d entries, want four", len(x86RawEVEXVPINSRForms)) + } + source := x86RawEVEXVPINSRGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + decoded, err := decodeX86RawDirectives(rawX86Function(code), "amd64") + if err != nil { + t.Fatal(err) + } + named, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + want := named.Funcs[0].Instrs[1:] + if len(decoded.Instrs) != len(want) { + t.Fatalf("decoded %d instructions, want %d", len(decoded.Instrs), len(want)) + } + for index, got := range decoded.Instrs { + if got.Op != want[index].Op || !reflect.DeepEqual(got.Args, want[index].Args) { + t.Fatalf("instruction %d=%+v, want %+v", index, got, want[index]) + } + } +} + +func TestDecodeX86RawEVEXVPINSRRejectsInvalidForms(t *testing.T) { + base := []byte{0x62, 0xa3, 0x4d, 0x08, 0x20, 0x44, 0x0e, 0x01, 0x05} + for _, variant := range []struct { + name string + mutate func([]byte) []byte + }{ + {"mask", func(code []byte) []byte { code[3] |= 1; return code }}, + {"zeroing", func(code []byte) []byte { code[3] |= 0x80; return code }}, + {"broadcast", func(code []byte) []byte { code[3] |= 0x10; return code }}, + {"vector length", func(code []byte) []byte { code[3] |= 0x20; return code }}, + {"address override", func(code []byte) []byte { return append([]byte{0x67}, code...) }}, + {"truncated immediate", func(code []byte) []byte { return code[:len(code)-1] }}, + } { + t.Run(variant.name, func(t *testing.T) { + code := variant.mutate(append([]byte(nil), base...)) + if _, _, matched, err := decodedX86EVEXVPINSRInstruction(code, 64); !matched || err == nil { + t.Fatalf("invalid %x: matched=%v err=%v", code, matched, err) + } + }) + } + if _, _, matched, err := decodedX86EVEXVPINSRInstruction(base, 32); !matched || err == nil { + t.Fatalf("accepted 386 EVEX VPINSR: matched=%v err=%v", matched, err) + } +} + +func TestTranslateX86RawEVEXVPINSRLLVM22Objects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + source := x86RawEVEXVPINSRGoForms() + code := assembleX87ControlBytes(t, "amd64", source) + file := x86RawUnprovedReturnFormProbeFile(t, "amd64", "rawVPINSR", code) + ir, err := Translate(file, Options{ + Goarch: "amd64", TargetTriple: triple, + Sigs: map[string]FuncSig{"rawVPINSR": {Name: "rawVPINSR", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "raw-vpinsr.ll", "raw-vpinsr.o", ir) + }) + } +} diff --git a/amd64_raw_vpinsr_test.go b/amd64_raw_vpinsr_test.go new file mode 100644 index 000000000..56b13e983 --- /dev/null +++ b/amd64_raw_vpinsr_test.go @@ -0,0 +1,74 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VPINSRInstructionCompleteVEXFamily(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + want string + }{ + {name: "byte_register", code: []byte{0xc4, 0xe3, 0x79, 0x20, 0xc1, 0x02}, mode: 64, want: "VPINSRB $2, CX, X0, X0"}, + {name: "word_two_byte_vex", code: []byte{0xc5, 0xf9, 0xc4, 0xc1, 0x02}, mode: 64, want: "VPINSRW $2, CX, X0, X0"}, + {name: "dword_register", code: []byte{0xc4, 0xe3, 0x79, 0x22, 0xc1, 0x02}, mode: 64, want: "VPINSRD $2, CX, X0, X0"}, + {name: "qword_register", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0xc1, 0x02}, mode: 64, want: "VPINSRQ $2, CX, X0, X0"}, + {name: "qword_memory_disp8", code: []byte{0xc4, 0x63, 0xa1, 0x22, 0x5e, 0x28, 0x01}, mode: 64, want: "VPINSRQ $1, 40(SI), X11, X11"}, + {name: "extended_registers", code: []byte{0xc4, 0x43, 0x31, 0x22, 0xca, 0x01}, mode: 64, want: "VPINSRD $1, R10, X9, X9"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPINSRInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func TestDecodedX86VPINSRInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc4, 0xe3, 0xf9, 0x22}, mode: 64}, + {name: "truncated_immediate", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0xc1}, mode: 64}, + {name: "vex_l_one", code: []byte{0xc4, 0xe3, 0xfd, 0x22, 0xc1, 0x02}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc4, 0xe3, 0xf8, 0x22, 0xc1, 0x02}, mode: 64}, + {name: "qword_in_386", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0xc1, 0x02}, mode: 32}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x43, 0x31, 0x22, 0xca, 0x01}, mode: 32}, + {name: "rip_relative", code: []byte{0xc4, 0xe3, 0xf9, 0x22, 0x05, 0, 0, 0, 0, 0x02}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VPINSRInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPINSR encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPINSRQSequence(t *testing.T) { + code := []byte{0xc4, 0x63, 0xa1, 0x22, 0x5e, 0x28, 0x01} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VPINSRQ raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPINSRQ" { + t.Fatalf("decoded = %#v, want one VPINSRQ", decoded) + } +} diff --git a/amd64_raw_vpror_test.go b/amd64_raw_vpror_test.go new file mode 100644 index 000000000..ec8393ff6 --- /dev/null +++ b/amd64_raw_vpror_test.go @@ -0,0 +1,48 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86PackedImmediateRotateCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvprold table covers the four EVEX immediate rotate + // instructions across X/Y/Z, memory, masks, zeroing, broadcast, high + // registers, and compressed disp8. LLVM 22 verified these bytes. + tests := []struct { + name string + code []byte + want string + }{ + {name: "left dword x high masked", code: []byte{0x62, 0xb1, 0x4d, 0x01, 0x72, 0xcc, 0x06}, want: "VPROLD $6, X20, K1, X22"}, + {name: "left qword y memory zeroing", code: []byte{0x62, 0xd1, 0xed, 0xaa, 0x72, 0x48, 0x03, 0x07}, want: "VPROLQ.Z $7, 96(R8), K2, Y2"}, + {name: "right dword reported minio", code: []byte{0x62, 0xf1, 0x2d, 0x48, 0x72, 0xc4, 0x06}, want: "VPRORD $6, Z4, Z10"}, + {name: "right dword z broadcast high masked", code: []byte{0x62, 0xd1, 0x35, 0x5c, 0x72, 0x44, 0x24, 0x7f, 0x44}, want: "VPRORD.BCST $68, 508(R12), K4, Z9"}, + {name: "right qword z broadcast zeroing", code: []byte{0x62, 0xf1, 0xed, 0xdb, 0x72, 0x40, 0x7f, 0x09}, want: "VPRORQ.BCST.Z $9, 1016(AX), K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedImmediateRotateInstruction(test.code, 64) + if !ok { + t.Fatal("packed immediate rotate encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPRORD(t *testing.T) { + code := []byte{0x62, 0xf1, 0x2d, 0x48, 0x72, 0xc4, 0x06} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPRORD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPRORD" || !strings.HasPrefix(decoded[0].Raw, "VPRORD $6, Z4, Z10 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpshift_test.go b/amd64_raw_vpshift_test.go new file mode 100644 index 000000000..1e5101199 --- /dev/null +++ b/amd64_raw_vpshift_test.go @@ -0,0 +1,145 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestDecodedX86PackedLogicalShiftCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvpslld table covers six logical shifts with both immediate + // and XMM/m128 uniform-count encodings. These LLVM 22 bytes also cover + // VEX/EVEX, every width, masks, zeroing, broadcast, high registers, and + // compressed disp8. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex immediate right dword", code: []byte{0xc5, 0xb5, 0x72, 0xd2, 0x07}, want: "VPSRLD $7, Y2, Y9"}, + {name: "vex count left qword memory", code: []byte{0xc4, 0x41, 0x6d, 0xf3, 0x4c, 0x24, 0x20}, want: "VPSLLQ 32(R12), Y2, Y9"}, + {name: "immediate left word", code: []byte{0x62, 0xb1, 0x4d, 0x01, 0x71, 0xf4, 0x06}, want: "VPSLLW $6, X20, K1, X22"}, + {name: "immediate left dword", code: []byte{0x62, 0xd1, 0x6d, 0xaa, 0x72, 0x70, 0x03, 0x07}, want: "VPSLLD.Z $7, 96(R8), K2, Y2"}, + {name: "immediate left qword", code: []byte{0x62, 0xb1, 0xb5, 0x48, 0x73, 0xf1, 0x44}, want: "VPSLLQ $68, Z17, Z9"}, + {name: "immediate right word", code: []byte{0x62, 0xd1, 0x35, 0x24, 0x71, 0x54, 0x24, 0x02, 0x0a}, want: "VPSRLW $10, 64(R12), K4, Y25"}, + {name: "immediate right dword reported minio", code: []byte{0x62, 0x91, 0x05, 0x48, 0x72, 0xd6, 0x0a}, want: "VPSRLD $10, Z30, Z15"}, + {name: "immediate right qword broadcast", code: []byte{0x62, 0xf1, 0xed, 0xdb, 0x73, 0x50, 0x7f, 0x09}, want: "VPSRLQ.BCST.Z $9, 1016(AX), K3, Z2"}, + {name: "count left word", code: []byte{0x62, 0xa1, 0x55, 0x01, 0xf1, 0xf4}, want: "VPSLLW X20, X21, K1, X22"}, + {name: "count left dword memory", code: []byte{0x62, 0xd1, 0x75, 0xaa, 0xf2, 0x50, 0x03}, want: "VPSLLD.Z 48(R8), Y1, K2, Y2"}, + {name: "count left qword", code: []byte{0x62, 0x31, 0xfd, 0x40, 0xf3, 0xc9}, want: "VPSLLQ X17, Z16, Z9"}, + {name: "count right word memory", code: []byte{0x62, 0x41, 0x3d, 0x04, 0xd1, 0x4c, 0x24, 0x02}, want: "VPSRLW 32(R12), X24, K4, X25"}, + {name: "count right dword", code: []byte{0x62, 0xa1, 0x55, 0x21, 0xd2, 0xf4}, want: "VPSRLD X20, Y21, K1, Y22"}, + {name: "count right qword memory", code: []byte{0x62, 0xf1, 0xf5, 0xcb, 0xd3, 0x50, 0x7f}, want: "VPSRLQ.Z 2032(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86PackedUniformShiftInstruction(test.code, 64) + if !ok { + t.Fatal("packed logical shift encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodedX86PackedArithmeticShiftStreamVByteRegression(t *testing.T) { + code := []byte{0xc5, 0xfd, 0x72, 0xe0, 0x1f} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "streamvbyte VPSRAD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPSRAD" || !strings.HasPrefix(decoded[0].Raw, "VPSRAD $31, Y0, Y0 ") { + t.Fatalf("decoded streamvbyte VPSRAD as %#v", decoded) + } +} + +func TestDecodedX86PackedArithmeticShiftCompleteGo127Family(t *testing.T) { + for _, test := range []struct { + name string + code []byte + want string + }{ + {"VEX immediate word", []byte{0xc5, 0xe9, 0x71, 0xe1, 0x03}, "VPSRAW $3, X1, X2"}, + {"VEX immediate dword", []byte{0xc5, 0xed, 0x72, 0xe1, 0x07}, "VPSRAD $7, Y1, Y2"}, + {"EVEX immediate qword", []byte{0x62, 0xf1, 0xed, 0x48, 0x72, 0xe1, 0x07}, "VPSRAQ $7, Z1, Z2"}, + {"VEX packed count word", []byte{0xc5, 0xed, 0xe1, 0xd9}, "VPSRAW X1, Y2, Y3"}, + {"EVEX packed count dword", []byte{0x62, 0xf1, 0x6d, 0x48, 0xe2, 0xd9}, "VPSRAD X1, Z2, Z3"}, + {"EVEX packed count qword", []byte{0x62, 0xf1, 0xed, 0x48, 0xe2, 0xd9}, "VPSRAQ X1, Z2, Z3"}, + {"EVEX dword broadcast zeroing", []byte{0x62, 0xf1, 0x6d, 0xbb, 0x72, 0x20, 0x07}, "VPSRAD.BCST.Z $7, 0(AX), K3, Y2"}, + {"EVEX qword broadcast high register", []byte{0x62, 0xf1, 0xed, 0xd3, 0x72, 0x60, 0x01, 0x07}, "VPSRAQ.BCST.Z $7, 8(AX), K3, Z18"}, + {"EVEX qword high register packed count", []byte{0x62, 0xa1, 0xd5, 0xc1, 0xe2, 0xf4}, "VPSRAQ.Z X20, Z21, K1, Z22"}, + {"EVEX word memory zeroing", []byte{0x62, 0xd1, 0x35, 0xaa, 0x71, 0x60, 0x01, 0x03}, "VPSRAW.Z $3, 32(R8), K2, Y9"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, recognized, err := decodedX86PackedUniformShiftInstruction(test.code, 64) + if err != nil || !recognized || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode %x: raw=%q length=%d recognized=%v err=%v; want %q", test.code, instruction.Raw, length, recognized, err, test.want) + } + }) + } +} + +func TestTranslateRawPackedArithmeticShiftLLVM22Targets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + name, goarch, triple string + }{ + {"darwin-amd64", "amd64", "x86_64-apple-darwin"}, + {"linux-amd64", "amd64", "x86_64-unknown-linux-gnu"}, + {"windows-amd64", "amd64", "x86_64-pc-windows-msvc"}, + {"linux-386", "386", "i386-unknown-linux-gnu"}, + {"windows-386", "386", "i686-pc-windows-msvc"}, + } { + t.Run(target.name, func(t *testing.T) { + encodings := [][]byte{ + {0xc5, 0xe9, 0x71, 0xe1, 0x03}, + {0xc5, 0xed, 0x72, 0xe1, 0x07}, + {0x62, 0xf1, 0xed, 0x48, 0x72, 0xe1, 0x07}, + {0xc5, 0xed, 0xe1, 0xd9}, + {0x62, 0xf1, 0x6d, 0x48, 0xe2, 0xd9}, + {0x62, 0xf1, 0xed, 0x48, 0xe2, 0xd9}, + } + encodings = append(encodings, []byte{0x62, 0xf1, 0x6d, 0xbb, 0x72, 0x20, 0x07}) + var source strings.Builder + source.WriteString("TEXT packedArithmeticShiftRaw(SB), $0-0\n") + for _, code := range encodings { + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $0x%02x\n", value) + } + } + source.WriteString("\tRET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.goarch, + TargetTriple: target.triple, + Sigs: map[string]FuncSig{"packedArithmeticShiftRaw": {Name: "packedArithmeticShiftRaw", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "raw-arithmetic-shift.ll", "raw-arithmetic-shift.o", ir) + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPSRLD(t *testing.T) { + code := []byte{0x62, 0x91, 0x05, 0x48, 0x72, 0xd6, 0x0a} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPSRLD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPSRLD" || !strings.HasPrefix(decoded[0].Raw, "VPSRLD $10, Z30, Z15 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpshufb_literal_test.go b/amd64_raw_vpshufb_literal_test.go new file mode 100644 index 000000000..ab6713b2c --- /dev/null +++ b/amd64_raw_vpshufb_literal_test.go @@ -0,0 +1,151 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func x86RawVPSHUFBLiteral(evex bool, width, mask int, zeroing bool) []byte { + code := encodeRawScalarMove(evex, 1, 0, 0, 0, 0, mask, zeroing) + code[1] = code[1]&0xf0 | 2 + if evex { + code[3] |= byte(map[int]int{16: 0, 32: 1, 64: 2}[width]) << 5 + } else if width == 32 { + code[2] |= 0x04 + } + code[len(code)-1] = 0x05 + code = append(code, 1, 0, 0, 0, 0xc3) + for index := 0; index < width; index++ { + code = append(code, byte(index*23+7)) + } + return code +} + +func TestDecodeX86RawVPSHUFBRIPDataCompleteFamily(t *testing.T) { + for _, evex := range []bool{false, true} { + widths := []int{16, 32} + if evex { + widths = append(widths, 64) + } + for _, width := range widths { + masks := []int{0} + if evex { + masks = append(masks, 7) + } + for _, mask := range masks { + name := fmt.Sprintf("evex%t/%d/mask%d", evex, width, mask) + t.Run(name, func(t *testing.T) { + code := x86RawVPSHUFBLiteral(evex, width, mask, mask != 0) + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}) + if err != nil { + t.Fatal(err) + } + want := Op("VPSHUFB") + if mask != 0 { + want += ".Z" + } + if len(decoded) != 2 || decoded[0].Op != want || + decoded[0].Args[0].Kind != OpSym || !decoded[0].x86RIPLiteral || + decoded[1].Op != OpRET { + t.Fatalf("decoded %x as %#v, want %s source-local data", code, decoded, want) + } + }) + } + } + } +} + +func TestDecodeX86RawVPSHUFBRIPDataRejectsUnsafeSources(t *testing.T) { + valid := x86RawVPSHUFBLiteral(true, 64, 7, true) + for name, mutate := range map[string]func([]byte) []byte{ + "truncated source": func(code []byte) []byte { return code[:len(code)-1] }, + "overlapping instruction": func(code []byte) []byte { + code[6] = 0 + return code + }, + "outside directive group": func(code []byte) []byte { + code[6] = 127 + return code + }, + "segment override": func(code []byte) []byte { + return append([]byte{0x65}, code...) + }, + "address override": func(code []byte) []byte { + return append([]byte{0x67}, code...) + }, + } { + t.Run(name, func(t *testing.T) { + code := mutate(append([]byte(nil), valid...)) + if _, err := decodeX86RawDirectiveGroup(code, 64, 0, name, map[string]bool{}); err == nil { + t.Fatalf("accepted unsafe RIP source %x", code) + } + }) + } +} + +func TestTranslateX86RawVPSHUFBRIPDataObjects(t *testing.T) { + var source strings.Builder + sigs := make(map[string]FuncSig) + var pools [][]byte + for index, form := range []struct { + evex bool + width int + mask int + }{ + {false, 16, 0}, {false, 32, 0}, + {true, 16, 0}, {true, 32, 7}, {true, 64, 7}, + } { + code := x86RawVPSHUFBLiteral(form.evex, form.width, form.mask, form.mask != 0) + name := fmt.Sprintf("vpshufbLiteral%d", index) + fmt.Fprintf(&source, "TEXT %s(SB),$0-0\n", name) + for _, value := range code { + fmt.Fprintf(&source, "\tBYTE $%#02x\n", value) + } + sigs[name] = FuncSig{Name: name, Ret: Void} + pools = append(pools, code[len(code)-form.width:]) + } + requireX86GoAssemblerResult(t, "amd64", source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + normalized, err := normalizeX86RawFile(file, "amd64") + if err != nil { + t.Fatal(err) + } + if len(normalized.Data) != len(pools) { + t.Fatalf("materialized %d constants, want %d", len(normalized.Data), len(pools)) + } + for index, datum := range normalized.Data { + if !bytes.Equal(datum.Payload, pools[index]) { + t.Fatalf("constant %d = %x, want %x", index, datum.Payload, pools[index]) + } + } + again, err := normalizeX86RawFile(normalized, "amd64") + if err != nil { + t.Fatal(err) + } + if len(again.Data) != len(pools) { + t.Fatalf("second normalization materialized %d constants, want %d", len(again.Data), len(pools)) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, triple := range []string{ + "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", + } { + t.Run(triple, func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, "<16 x i32> ") { + t.Fatal("missing lane-local and high-bit-zero byte shuffle") + } + compileLLVMToObject(t, llc, triple, "vpshufb-literal.ll", "vpshufb-literal.o", ir) + }) + } +} diff --git a/amd64_raw_vpshufb_test.go b/amd64_raw_vpshufb_test.go new file mode 100644 index 000000000..fb8282dda --- /dev/null +++ b/amd64_raw_vpshufb_test.go @@ -0,0 +1,50 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86VPSHUFBCompleteGo127Family(t *testing.T) { + // Go 1.27's _yvandnpd table gives VPSHUFB VEX X/Y forms and EVEX + // X/Y/Z forms, including high registers, memory, K masking, zeroing, + // and EVEX compressed disp8 scaling. LLVM 22 verified these bytes. + tests := []struct { + name string + code []byte + want string + }{ + {name: "vex x high destination", code: []byte{0xc4, 0x62, 0x79, 0x00, 0xc1}, want: "VPSHUFB X1, X0, X8"}, + {name: "vex y high memory", code: []byte{0xc4, 0x42, 0x6d, 0x00, 0x4c, 0x24, 0x20}, want: "VPSHUFB 32(R12), Y2, Y9"}, + {name: "evex x high masked", code: []byte{0x62, 0xa2, 0x55, 0x01, 0x00, 0xf4}, want: "VPSHUFB X20, X21, K1, X22"}, + {name: "evex y memory zeroing", code: []byte{0x62, 0xd2, 0x75, 0xaa, 0x00, 0x50, 0x03}, want: "VPSHUFB.Z 96(R8), Y1, K2, Y2"}, + {name: "evex z high", code: []byte{0x62, 0x32, 0x7d, 0x40, 0x00, 0xc9}, want: "VPSHUFB Z17, Z16, Z9"}, + {name: "evex z compressed memory zeroing", code: []byte{0x62, 0xf2, 0x75, 0xcb, 0x00, 0x50, 0x7f}, want: "VPSHUFB.Z 8128(AX), Z1, K3, Z2"}, + {name: "reported minio", code: []byte{0x62, 0xc2, 0x7d, 0x40, 0x00, 0xc3}, want: "VPSHUFB Z11, Z16, Z16"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPSHUFBInstruction(test.code, 64) + if !ok { + t.Fatal("VPSHUFB encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPSHUFB(t *testing.T) { + code := []byte{0x62, 0xc2, 0x7d, 0x40, 0x00, 0xc3} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPSHUFB", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPSHUFB" || !strings.HasPrefix(decoded[0].Raw, "VPSHUFB Z11, Z16, Z16 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpternlog_test.go b/amd64_raw_vpternlog_test.go new file mode 100644 index 000000000..99149643e --- /dev/null +++ b/amd64_raw_vpternlog_test.go @@ -0,0 +1,87 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestX86RawImmediateThreeVectorSourceLocalRIPData(t *testing.T) { + for _, family := range []struct { + opcode byte + name string + }{{0x25, "VPTERNLOG"}, {0x03, "VALIGN"}} { + for _, qword := range []bool{false, true} { + for vectorLength := 0; vectorLength < 3; vectorLength++ { + for _, broadcast := range []bool{false, true} { + p1 := byte(0x7d) + laneWidth := 4 + suffix := "D" + if qword { + p1 |= 0x80 + laneWidth = 8 + suffix = "Q" + } + p2 := byte(0x08 | vectorLength<<5) + width := 16 << vectorLength + if broadcast { + p2 |= 0x10 + width = laneWidth + } + code := []byte{0x62, 0xf3, p1, p2, family.opcode, 0x05, 1, 0, 0, 0, 0xca, 0xc3} + for value := 0; value < width; value++ { + code = append(code, byte(value)) + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "immediate three-vector RIP", map[string]bool{}) + if err != nil { + t.Fatalf("%s%s width %d broadcast=%t: %v", family.name, suffix, width, broadcast, err) + } + if len(decoded) != 2 || len(decoded[0].x86RIPLiteralData) != width { + t.Fatalf("%s%s width %d broadcast=%t: %#v", family.name, suffix, width, broadcast, decoded) + } + } + } + } + } +} + +func TestDecodedX86TernaryLogicCompleteGo127Family(t *testing.T) { + // Go 1.27 shares _yvalignd between VPTERNLOGD/Q. These LLVM 22 bytes + // cover X/Y/Z, high registers, memory, masks, zeroing, broadcast, and + // compressed disp8 for both lane widths. + tests := []struct { + name string + code []byte + want string + }{ + {name: "dword x high masked", code: []byte{0x62, 0xa3, 0x55, 0x01, 0x25, 0xf4, 0xca}, want: "VPTERNLOGD $202, X20, X21, K1, X22"}, + {name: "qword y memory zeroing", code: []byte{0x62, 0xd3, 0xf5, 0xaa, 0x25, 0x50, 0x03, 0x96}, want: "VPTERNLOGQ.Z $150, 96(R8), Y1, K2, Y2"}, + {name: "dword reported minio", code: []byte{0x62, 0x73, 0x55, 0x48, 0x25, 0xce, 0xca}, want: "VPTERNLOGD $202, Z6, Z5, Z9"}, + {name: "dword z broadcast high masked", code: []byte{0x62, 0x53, 0x7d, 0x54, 0x25, 0x4c, 0x24, 0x7f, 0x44}, want: "VPTERNLOGD.BCST $68, 508(R12), Z16, K4, Z9"}, + {name: "qword z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x25, 0x50, 0x7f, 0x09}, want: "VPTERNLOGQ.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86TernaryLogicInstruction(test.code, 64) + if !ok { + t.Fatal("ternary-logic encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVPTERNLOGD(t *testing.T) { + code := []byte{0x62, 0x73, 0x55, 0x48, 0x25, 0xce, 0xca} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VPTERNLOGD", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VPTERNLOGD" || !strings.HasPrefix(decoded[0].Raw, "VPTERNLOGD $202, Z6, Z5, Z9 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vpunpck_test.go b/amd64_raw_vpunpck_test.go new file mode 100644 index 000000000..166dffb4d --- /dev/null +++ b/amd64_raw_vpunpck_test.go @@ -0,0 +1,103 @@ +package plan9asm + +import "testing" + +func TestDecodedX86VPUNPCKInstructionCompleteVEXFamily(t *testing.T) { + for _, test := range []struct { + opcode byte + op Op + }{ + {0x60, "VPUNPCKLBW"}, + {0x61, "VPUNPCKLWD"}, + {0x62, "VPUNPCKLDQ"}, + {0x6c, "VPUNPCKLQDQ"}, + {0x68, "VPUNPCKHBW"}, + {0x69, "VPUNPCKHWD"}, + {0x6a, "VPUNPCKHDQ"}, + {0x6d, "VPUNPCKHQDQ"}, + } { + t.Run(string(test.op), func(t *testing.T) { + code := []byte{0xc5, 0xe9, test.opcode, 0xcb} + instruction, length, ok, err := decodedX86VPUNPCKInstruction(code, 64) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(code) { + t.Fatalf("length = %d, want %d", length, len(code)) + } + if instruction.Op != test.op || instruction.Raw != string(test.op)+" X3, X2, X1" { + t.Fatalf("instruction = %#v", instruction) + } + }) + } + + for _, test := range []struct { + name string + code []byte + want string + }{ + {name: "y_memory", code: []byte{0xc5, 0xed, 0x6d, 0x4e, 0x20}, want: "VPUNPCKHQDQ 32(SI), Y2, Y1"}, + {name: "extended_registers", code: []byte{0xc4, 0x41, 0x41, 0x6d, 0xf7}, want: "VPUNPCKHQDQ X15, X7, X14"}, + } { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VPUNPCKInstruction(test.code, 64) + if err != nil { + t.Fatal(err) + } + if !ok || length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decode = (%#v, %d, %v), want %q", instruction, length, ok, test.want) + } + }) + } +} + +func TestDecodedX86VPUNPCKInstructionRejectsInvalidVEXForms(t *testing.T) { + for _, test := range []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc5, 0xe9, 0x6d}, mode: 64}, + {name: "vex_pp_zero", code: []byte{0xc5, 0xe8, 0x6d, 0xcb}, mode: 64}, + {name: "vex_w_one", code: []byte{0xc4, 0xe1, 0xe9, 0x6d, 0xcb}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc4, 0x41, 0x41, 0x6d, 0xf7}, mode: 32}, + {name: "rip_relative", code: []byte{0xc5, 0xe9, 0x6d, 0x0d, 0, 0, 0, 0}, mode: 64}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VPUNPCKInstruction(test.code, test.mode) + if !ok { + t.Fatal("VPUNPCK encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVPUNPCKSequence(t *testing.T) { + code := []byte{ + 0xc4, 0xc1, 0x41, 0x6d, 0xf7, + 0xc5, 0x41, 0x6c, 0xff, + 0xc4, 0xc1, 0x11, 0x6d, 0xff, + 0xc5, 0x61, 0x6c, 0xfb, + 0xc4, 0xc1, 0x69, 0x6d, 0xd7, + 0xc4, 0x41, 0x09, 0x6c, 0xfe, + 0xc4, 0xc1, 0x61, 0x6d, 0xdf, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VPUNPCK raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 7 { + t.Fatalf("decoded %d instructions, want 7: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VPUNPCKHQDQ" && instruction.Op != "VPUNPCKLQDQ" { + t.Fatalf("decoded unexpected op %s", instruction.Op) + } + } +} diff --git a/amd64_raw_vshuff_blocks_test.go b/amd64_raw_vshuff_blocks_test.go new file mode 100644 index 000000000..9a508d672 --- /dev/null +++ b/amd64_raw_vshuff_blocks_test.go @@ -0,0 +1,51 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86Shuffle128BitBlocksCompleteGo127Family(t *testing.T) { + // Go 1.27 shares _yvshuff32x4 across these four instructions. Its + // EVEX-only rows cover Y/Z register or memory sources, high registers, + // K masking, zeroing, scalar broadcast, and compressed disp8 scaling. + tests := []struct { + name string + code []byte + want string + }{ + {name: "f32 y high masked", code: []byte{0x62, 0xa3, 0x55, 0x21, 0x23, 0xf4, 0x06}, want: "VSHUFF32X4 $6, Y20, Y21, K1, Y22"}, + {name: "f32 z memory zeroing", code: []byte{0x62, 0xd3, 0x75, 0xca, 0x23, 0x50, 0x03, 0x07}, want: "VSHUFF32X4.Z $7, 192(R8), Z1, K2, Z2"}, + {name: "f64 reported minio", code: []byte{0x62, 0x23, 0x8d, 0x40, 0x23, 0xc0, 0xee}, want: "VSHUFF64X2 $238, Z16, Z30, Z24"}, + {name: "f64 z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x23, 0x50, 0x7f, 0x09}, want: "VSHUFF64X2.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + {name: "i32 y high memory masked", code: []byte{0x62, 0x43, 0x3d, 0x24, 0x43, 0x4c, 0x24, 0x02, 0x0a}, want: "VSHUFI32X4 $10, 64(R12), Y24, K4, Y25"}, + {name: "i32 z high", code: []byte{0x62, 0x33, 0x7d, 0x40, 0x43, 0xc9, 0x44}, want: "VSHUFI32X4 $68, Z17, Z16, Z9"}, + {name: "i64 y high masked", code: []byte{0x62, 0xa3, 0xd5, 0x21, 0x43, 0xf4, 0x06}, want: "VSHUFI64X2 $6, Y20, Y21, K1, Y22"}, + {name: "i64 z broadcast zeroing", code: []byte{0x62, 0xf3, 0xf5, 0xdb, 0x43, 0x50, 0x7f, 0x09}, want: "VSHUFI64X2.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86Shuffle128BitBlocksInstruction(test.code, 64) + if !ok { + t.Fatal("shuffle-block encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) || instruction.Raw != test.want { + t.Fatalf("decoded %x as length=%d raw=%q, want length=%d raw=%q", test.code, length, instruction.Raw, len(test.code), test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioVSHUFF64X2(t *testing.T) { + code := []byte{0x62, 0x23, 0x8d, 0x40, 0x23, 0xc0, 0xee} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio VSHUFF64X2", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VSHUFF64X2" || !strings.HasPrefix(decoded[0].Raw, "VSHUFF64X2 $238, Z16, Z30, Z24 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} diff --git a/amd64_raw_vshufps_test.go b/amd64_raw_vshufps_test.go new file mode 100644 index 000000000..70960c673 --- /dev/null +++ b/amd64_raw_vshufps_test.go @@ -0,0 +1,135 @@ +package plan9asm + +import ( + "strings" + "testing" +) + +func TestDecodedX86VSHUFPSInstructionCompleteVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + want string + }{ + {name: "x_register", code: []byte{0xc5, 0x78, 0xc6, 0xc1, 0x44}, mode: 64, want: "VSHUFPS $68, X1, X0, X8"}, + {name: "y_register", code: []byte{0xc5, 0x6c, 0xc6, 0xcb, 0x44}, mode: 64, want: "VSHUFPS $68, Y3, Y2, Y9"}, + {name: "extended_registers", code: []byte{0xc4, 0x41, 0x3c, 0xc6, 0xd1, 0xdd}, mode: 64, want: "VSHUFPS $221, Y9, Y8, Y10"}, + {name: "memory_disp8", code: []byte{0xc5, 0xec, 0xc6, 0x56, 0x20, 0x01}, mode: 64, want: "VSHUFPS $1, 32(SI), Y2, Y2"}, + {name: "pd extended registers", code: []byte{0xc4, 0x41, 0x3d, 0xc6, 0xd1, 0xdd}, mode: 64, want: "VSHUFPD $221, Y9, Y8, Y10"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VSHUFPSInstruction(test.code, test.mode) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("encoding was not recognized") + } + if length != len(test.code) { + t.Fatalf("length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("instruction = %q, want %q", instruction.Raw, test.want) + } + }) + } +} + +func TestDecodedX86PackedFloatShuffleCompleteEVEXFamily(t *testing.T) { + // Go 1.27 assigns VSHUFPS and VSHUFPD the shared + // _yvgf2p8affineinvqb table. Its EVEX rows cover X/Y/Z register or + // memory sources, high registers, K masking, zeroing, and scalar + // broadcast. The byte sequences below are verified by LLVM 22. + tests := []struct { + name string + code []byte + want string + }{ + {name: "ps x high masked", code: []byte{0x62, 0xa1, 0x54, 0x01, 0xc6, 0xf4, 0x06}, want: "VSHUFPS $6, X20, X21, K1, X22"}, + {name: "ps y memory zeroing", code: []byte{0x62, 0xd1, 0x74, 0xaa, 0xc6, 0x50, 0x03, 0x07}, want: "VSHUFPS.Z $7, 96(R8), Y1, K2, Y2"}, + {name: "ps z high", code: []byte{0x62, 0x31, 0x7c, 0x40, 0xc6, 0xc9, 0x44}, want: "VSHUFPS $68, Z17, Z16, Z9"}, + {name: "ps z broadcast zeroing", code: []byte{0x62, 0xf1, 0x74, 0xdb, 0xc6, 0x50, 0x7f, 0x09}, want: "VSHUFPS.BCST.Z $9, 508(AX), Z1, K3, Z2"}, + {name: "pd x high masked", code: []byte{0x62, 0xa1, 0xd5, 0x01, 0xc6, 0xf4, 0x06}, want: "VSHUFPD $6, X20, X21, K1, X22"}, + {name: "pd y memory zeroing", code: []byte{0x62, 0xd1, 0xf5, 0xaa, 0xc6, 0x50, 0x03, 0x07}, want: "VSHUFPD.Z $7, 96(R8), Y1, K2, Y2"}, + {name: "pd z high", code: []byte{0x62, 0x31, 0xfd, 0x40, 0xc6, 0xc9, 0x44}, want: "VSHUFPD $68, Z17, Z16, Z9"}, + {name: "pd z broadcast zeroing", code: []byte{0x62, 0xf1, 0xf5, 0xdb, 0xc6, 0x50, 0x7f, 0x09}, want: "VSHUFPD.BCST.Z $9, 1016(AX), Z1, K3, Z2"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + instruction, length, ok, err := decodedX86VSHUFPSInstruction(test.code, 64) + if !ok { + t.Fatal("packed float shuffle encoding was not recognized") + } + if err != nil { + t.Fatal(err) + } + if length != len(test.code) { + t.Fatalf("decoded length = %d, want %d", length, len(test.code)) + } + if instruction.Raw != test.want { + t.Fatalf("decoded %x as %q, want %q", test.code, instruction.Raw, test.want) + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupReportedMinioEVEXVSHUFPS(t *testing.T) { + code := []byte{0x62, 0x31, 0x7c, 0x40, 0xc6, 0xc9, 0x44} + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "minio EVEX VSHUFPS", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 1 || decoded[0].Op != "VSHUFPS" || !strings.HasPrefix(decoded[0].Raw, "VSHUFPS $68, Z17, Z16, Z9 ") { + t.Fatalf("decoded reported minio bytes as %#v", decoded) + } +} + +func TestDecodedX86VSHUFPSInstructionRejectsInvalidVEXForms(t *testing.T) { + tests := []struct { + name string + code []byte + mode int + }{ + {name: "truncated_modrm", code: []byte{0xc5, 0x7c, 0xc6}, mode: 64}, + {name: "truncated_immediate", code: []byte{0xc5, 0x7c, 0xc6, 0xc1}, mode: 64}, + {name: "vex_pp_f3", code: []byte{0xc5, 0x7e, 0xc6, 0xc1, 0x44}, mode: 64}, + {name: "vex_w_one", code: []byte{0xc4, 0xe1, 0xbc, 0xc6, 0xc1, 0x44}, mode: 64}, + {name: "extended_register_in_386", code: []byte{0xc5, 0x78, 0xc6, 0xc1, 0x44}, mode: 32}, + {name: "rip_relative", code: []byte{0xc5, 0xfc, 0xc6, 0x05, 0, 0, 0, 0, 0x44}, mode: 64}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, ok, err := decodedX86VSHUFPSInstruction(test.code, test.mode) + if !ok { + t.Fatal("VSHUFPS encoding was not recognized") + } + if err == nil { + t.Fatal("invalid encoding was accepted") + } + }) + } +} + +func TestDecodeX86RawDirectiveGroupVSHUFPSSequence(t *testing.T) { + code := []byte{ + 0xc5, 0x7c, 0xc6, 0xc1, 0x44, + 0xc5, 0xfc, 0xc6, 0xc1, 0xee, + 0xc5, 0x6c, 0xc6, 0xcb, 0x44, + 0xc5, 0xec, 0xc6, 0xd3, 0xee, + 0xc4, 0xc1, 0x3c, 0xc6, 0xd9, 0xdd, + } + decoded, err := decodeX86RawDirectiveGroup(code, 64, 0, "VSHUFPS raw sequence", map[string]bool{}) + if err != nil { + t.Fatal(err) + } + if len(decoded) != 5 { + t.Fatalf("decoded %d instructions, want 5: %#v", len(decoded), decoded) + } + for _, instruction := range decoded { + if instruction.Op != "VSHUFPS" { + t.Fatalf("decoded op = %s, want VSHUFPS", instruction.Op) + } + } +} diff --git a/amd64_raw_waitpkg.go b/amd64_raw_waitpkg.go new file mode 100644 index 000000000..ea3d5d3f5 --- /dev/null +++ b/amd64_raw_waitpkg.go @@ -0,0 +1,81 @@ +package plan9asm + +import "fmt" + +// WAITPKG shares the 0F AE group with fences, CLWB and FSGSBASE. Mandatory +// prefix plus mod=3 and /6 select the family; REX.B selects the GP register. +// See Intel SDM volume 2B (UMONITOR, UMWAIT, TPAUSE), not x86asm's legacy +// MFENCE decoding of these bytes. +func decodedX86WaitPackageInstruction(code []byte, mode int) (Instr, int, bool, error) { + if mode != 32 && mode != 64 { + return Instr{}, 0, false, nil + } + i := 0 + var repeat, segment, rex byte + operandOverride, addressOverride, locked := false, false, false + for i < len(code) { + switch code[i] { + case 0xf2, 0xf3: + repeat, rex = code[i], 0 + case 0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65: + segment, rex = code[i], 0 + case 0x66: + operandOverride, rex = true, 0 + case 0x67: + addressOverride, rex = true, 0 + case 0xf0: + locked, rex = true, 0 + default: + if mode == 64 && code[i] >= 0x40 && code[i] <= 0x4f { + rex = code[i] + i++ + continue + } + goto opcode + } + i++ + } + +opcode: + prefix := repeat + if prefix == 0 && operandOverride { + prefix = 0x66 + } + if prefix == 0 || len(code) < i+2 || code[i] != 0x0f || code[i+1] != 0xae { + return Instr{}, 0, false, nil + } + if len(code) < i+3 { + return Instr{}, 0, true, fmt.Errorf("truncated WAITPKG ModRM") + } + modrm := code[i+2] + if modrm&0x38 != 0x30 || modrm>>6 != 3 { + // In particular, 66 0F AE /6 with a memory operand is CLWB. + return Instr{}, 0, false, nil + } + if locked || i+3 > 15 { + return Instr{}, 0, true, fmt.Errorf("invalid WAITPKG LOCK prefix or instruction length") + } + var op Op + for name, spec := range amd64ImplicitSystemSpecs { + if spec.rawPrefix == prefix { + op = name + break + } + } + register, ok := decodedX86GeneralRegister(int(modrm&7) + int(rex&1)*8) + if !ok || op == "" { + return Instr{}, 0, true, fmt.Errorf("invalid WAITPKG register or mandatory prefix") + } + ins := Instr{ + Op: op, Args: []Operand{{Kind: OpReg, Reg: register}}, + Raw: fmt.Sprintf("%s %s", op, register), x86Encoded: true, + } + if amd64ImplicitSystemSpecs[op].operand == amd64ImplicitSystemAddressGP { + ins.x86AddressBits = mode + if addressOverride { + ins.x86AddressBits /= 2 + } + ins.x86SegmentPrefix = segment + } + return ins, i + 3, true, nil +} diff --git a/amd64_raw_waitpkg_test.go b/amd64_raw_waitpkg_test.go new file mode 100644 index 000000000..76d69364d --- /dev/null +++ b/amd64_raw_waitpkg_test.go @@ -0,0 +1,178 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +// Intel SDM volume 2B: UMONITOR/UMWAIT/TPAUSE share 0F AE /6, mod=3. +// Go 1.27 uses ywrfsbase's single Yrl operand for all three spellings. +func TestX86RawWaitPackageFamily(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct{ arch, triple string }{ + {"amd64", "x86_64-unknown-linux-gnu"}, + {"amd64", "x86_64-apple-darwin"}, + {"amd64", "x86_64-pc-windows-msvc"}, + {"386", "i386-unknown-linux-gnu"}, + {"386", "i686-pc-windows-msvc"}, + } { + for _, form := range []struct { + op Op + prefix byte + }{{"UMONITOR", 0xf3}, {"UMWAIT", 0xf2}, {"TPAUSE", 0x66}} { + t.Run(target.triple+"/"+string(form.op), func(t *testing.T) { + code := []byte{form.prefix} + if target.arch == "amd64" { + // ixl-go's UMONITOR uses REX.W; W does not select its width. + code = append(code, 0x48) + } + code = append(code, 0x0f, 0xae, 0xf0) + var source strings.Builder + source.WriteString("TEXT waitpkg(SB),$0-0\n") + for _, value := range code { + fmt.Fprintf(&source, "BYTE $0x%02x\n", value) + } + source.WriteString("RET\n") + requireX86GoAssemblerResult(t, target.arch, source.String(), true) + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{ + Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"waitpkg": {Name: "waitpkg", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, strings.ToLower(string(form.op))) { + t.Fatalf("decoded %x as another instruction", code) + } + compileLLVMToObject(t, llc, target.triple, "waitpkg.ll", "waitpkg.o", ir) + }) + } + } +} + +func TestX86WaitPackageClearsOtherFlags(t *testing.T) { + for _, op := range []string{"UMWAIT", "TPAUSE"} { + file, err := Parse(ArchAMD64, "TEXT flags(SB),$0-0\n"+op+" BX\nRET\n") + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: "amd64", Sigs: map[string]FuncSig{ + "flags": {Name: "flags", Ret: Void}, + }}) + if err != nil { + t.Fatal(err) + } + _, after, ok := strings.Cut(ir, "asm sideeffect") + if !ok { + t.Fatal("no hardware wait operation") + } + for _, flag := range []string{"z", "slt", "pf", "of"} { + if !strings.Contains(after, "store i1 false, ptr %flags_"+flag) { + t.Errorf("%s did not clear %s after capturing carry", op, flag) + } + } + } +} + +func TestX86RawWaitPackageRegisterGrammar(t *testing.T) { + for _, mode := range []int{32, 64} { + registerCount := 8 + if mode == 64 { + registerCount = 16 + } + for _, form := range []struct { + op Op + prefix byte + }{{"UMONITOR", 0xf3}, {"UMWAIT", 0xf2}, {"TPAUSE", 0x66}} { + for register := 0; register < registerCount; register++ { + for _, ignoredREX := range []byte{0, 2, 4, 8, 14} { + code := []byte{form.prefix, 0x67} + if mode == 64 { + code = append(code, 0x40|ignoredREX|byte(register>>3)) + } + code = append(code, 0x0f, 0xae, 0xf0|byte(register&7)) + ins, length, ok, err := decodedX86WaitPackageInstruction(code, mode) + wantRegister, _ := decodedX86GeneralRegister(register) + if err != nil || !ok || length != len(code) || ins.Op != form.op || + len(ins.Args) != 1 || ins.Args[0].Reg != wantRegister { + t.Fatalf("mode=%d bytes=%x: %+v, length=%d, ok=%v, err=%v", mode, code, ins, length, ok, err) + } + if form.op == "UMONITOR" && ins.x86AddressBits != mode/2 { + t.Fatalf("UMONITOR address size = %d, want %d", ins.x86AddressBits, mode/2) + } + } + } + } + } + for _, other := range [][]byte{ + {0x0f, 0xae, 0xf0}, // MFENCE + {0x66, 0x0f, 0xae, 0x30}, // CLWB (AX) + {0xf3, 0x0f, 0xae, 0xc0}, // RDFSBASEL AX + {0xf3, 0x0f, 0xae, 0x30}, // a memory encoding, not WAITPKG + {0xf2, 0x0f, 0xae, 0x30}, // a memory encoding, not WAITPKG + } { + if _, _, ok, _ := decodedX86WaitPackageInstruction(other, 64); ok { + t.Fatalf("claimed another 0F AE family: %x", other) + } + } + for _, invalid := range [][]byte{ + {0xf3, 0x0f, 0xae}, + {0xf0, 0xf3, 0x0f, 0xae, 0xf0}, + append(bytes.Repeat([]byte{0x67}, 12), 0xf3, 0x0f, 0xae, 0xf0), + } { + if _, err := decodeX86RawDirectiveGroup(invalid, 64, 0, "invalid WAITPKG", map[string]bool{}); err == nil { + t.Fatalf("accepted invalid WAITPKG encoding %x", invalid) + } + } +} + +func TestX86RawWaitPackageMonitorAddressOverrides(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, target := range []struct { + arch, triple string + mode int + }{ + {"amd64", "x86_64-unknown-linux-gnu", 64}, + {"amd64", "x86_64-apple-darwin", 64}, + {"amd64", "x86_64-pc-windows-msvc", 64}, + {"386", "i386-unknown-linux-gnu", 32}, + {"386", "i686-pc-windows-msvc", 32}, + } { + for _, segment := range []byte{0x26, 0x2e, 0x36, 0x3e, 0x64, 0x65} { + t.Run(fmt.Sprintf("%s/%02x", target.triple, segment), func(t *testing.T) { + source := fmt.Sprintf(`TEXT monitor(SB),$0-0 + BYTE $0x%02x + BYTE $0x67; BYTE $0xf3; BYTE $0x0f; BYTE $0xae; BYTE $0xf3 + RET +`, segment) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: target.arch, TargetTriple: target.triple, + Sigs: map[string]FuncSig{"monitor": {Name: "monitor", Ret: Void}}, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ir, fmt.Sprintf(".byte 0x%02x; umonitor", segment)) || + !strings.Contains(ir, fmt.Sprintf("zext i%d", target.mode/2)) { + t.Fatal("lost monitor segment or narrow address semantics") + } + compileLLVMToObject(t, llc, target.triple, "monitor.ll", "monitor.o", ir) + }) + } + } +} diff --git a/amd64_raw_x87_control_test.go b/amd64_raw_x87_control_test.go new file mode 100644 index 000000000..dce6a37d5 --- /dev/null +++ b/amd64_raw_x87_control_test.go @@ -0,0 +1,412 @@ +package plan9asm + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" +) + +func TestX86RawX87AliasFamilyMatchesGoEncoder(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + wantTables := map[Op]string{"FCLEX": "ynone", "FINIT": "ynone", "FSAVE": "ysvrs_om", "FSTCW": "ysvrs_om", "FSTENV": "ysvrs_om", "FSTSW": "ystsw"} + if len(x86RawX87ControlAliases) != len(wantTables) { + t.Fatalf("raw x87 control grammar has %d aliases, want %d", len(x86RawX87ControlAliases), len(wantTables)) + } + for decoded, goOp := range x86RawX87ControlAliases { + if decoded.String() != "FN"+string(goOp)[1:] || !isX87Op(goOp) { + t.Fatalf("raw %s does not resolve to the Go x87 lowerer for %s", decoded, goOp) + } + row := regexp.MustCompile(`\{A` + string(goOp) + `,\s*(\w+),`).FindSubmatch(data) + if len(row) != 2 || string(row[1]) != wantTables[goOp] { + t.Fatalf("Go %s row = %q, want operand table %s", goOp, row, wantTables[goOp]) + } + } +} + +func TestX86RawX87EnvironmentAddressSpaces(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, arch := range []string{"386", "amd64"} { + var source strings.Builder + source.WriteString("TEXT x87segments(SB),4,$0-0\n") + for _, opcode := range [][]byte{{0xdd, 0x30}, {0xd9, 0x38}, {0xd9, 0x30}, {0xdd, 0x38}} { + for _, segment := range []byte{0x64, 0x65} { + for _, b := range append([]byte{segment}, opcode...) { + fmt.Fprintf(&source, "BYTE $%#02x\n", b) + } + } + } + source.WriteString("RET\n") + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + triple := "x86_64-unknown-linux-gnu" + if arch == "386" { + triple = "i386-unknown-linux-gnu" + } + ir, err := Translate(file, Options{Goarch: arch, TargetTriple: triple, Sigs: map[string]FuncSig{"x87segments": {Name: "x87segments", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "x87-segments.ll", "x87-segments.o", ir) + } +} + +func TestX86RawX87DoesNotWidenLegacyEnvironmentFormat(t *testing.T) { + for _, code := range [][]byte{{0x66, 0xdd, 0x30}, {0x66, 0xd9, 0x30}} { + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Fatal(err) + } + if _, err := decodedX86GoSyntax(inst, code); err == nil { + t.Fatalf("silently treated 16-bit environment %x as the Go 32-bit environment form", code) + } + } +} + +// asm6.go's FCLEX/FINIT/FSAVE/FSTCW/FSTENV/FSTSW rows encode the +// no-wait forms, despite omitting Intel's N in their Go mnemonic. +func x87NoWaitFixtures() []struct { + code []byte + text string +} { + return []struct { + code []byte + text string + }{ + {[]byte{0xdb, 0xe2}, "FCLEX"}, + {[]byte{0xdb, 0xe3}, "FINIT"}, + {[]byte{0xdd, 0x30}, "FSAVE (AX)"}, + {[]byte{0xd9, 0x38}, "FSTCW (AX)"}, + {[]byte{0xd9, 0x30}, "FSTENV (AX)"}, + {[]byte{0xdd, 0x38}, "FSTSW (AX)"}, + {[]byte{0xdf, 0xe0}, "FSTSW AX"}, + {[]byte{0xdd, 0x74, 0x8b, 0xf9}, "FSAVE -7(BX)(CX*4)"}, + {[]byte{0xd9, 0x7c, 0x8b, 0xf9}, "FSTCW -7(BX)(CX*4)"}, + {[]byte{0xd9, 0x74, 0x8b, 0xf9}, "FSTENV -7(BX)(CX*4)"}, + {[]byte{0xdd, 0x7c, 0x8b, 0xf9}, "FSTSW -7(BX)(CX*4)"}, + } +} + +func TestX86RawX87NoWaitCompleteGoForms(t *testing.T) { + for _, goarch := range []string{"386", "amd64"} { + t.Run(goarch, func(t *testing.T) { + var named, raw strings.Builder + named.WriteString("TEXT x87control(SB),4,$0-0\n") + raw.WriteString("TEXT x87control(SB),4,$0-0\n") + var expectedBytes []byte + for _, test := range x87NoWaitFixtures() { + fmt.Fprintln(&named, test.text) + expectedBytes = append(expectedBytes, test.code...) + for _, b := range test.code { + fmt.Fprintf(&raw, "BYTE $%#02x\n", b) + } + decoded, err := decodeX86RawDirectives(rawX86Function(test.code), goarch) + if err != nil { + t.Fatal(err) + } + want, err := Parse(ArchAMD64, "TEXT f(SB),$0-0\n"+test.text+"\n") + if err != nil { + t.Fatal(err) + } + if len(decoded.Instrs) != 1 || decoded.Instrs[0].Op != want.Funcs[0].Instrs[1].Op || !reflect.DeepEqual(decoded.Instrs[0].Args, want.Funcs[0].Instrs[1].Args) { + t.Fatalf("decoded %x as %+v, want %s", test.code, decoded.Instrs, test.text) + } + } + named.WriteString("RET\n") + raw.WriteString("RET\n") + expectedBytes = append(expectedBytes, 0xc3) + code := assembleX87ControlBytes(t, goarch, named.String()) + if !bytes.HasPrefix(code, expectedBytes) { + t.Fatalf("Go %s encoded no-wait family as %x, want prefix %x", goarch, code, expectedBytes) + } + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc"} { + if (goarch == "386") != strings.HasPrefix(triple, "i") { + continue + } + for _, mode := range []X87Mode{X87Auto, X87Software} { + var irs []string + for _, source := range []string{named.String(), raw.String()} { + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ir, err := Translate(file, Options{Goarch: goarch, TargetTriple: triple, X87Mode: mode, Sigs: map[string]FuncSig{"x87control": {Name: "x87control", Ret: Void}}}) + if err != nil { + t.Fatal(err) + } + irs = append(irs, ir) + } + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc not found") + } + for _, ir := range irs { + compileLLVMToObject(t, llc, triple, "x87-control.ll", "x87-control.o", ir) + } + } + } + }) + } +} + +func assembleX87ControlBytes(t *testing.T, goarch, source string) []byte { + t.Helper() + dir := t.TempDir() + name := filepath.Join(dir, "control.s") + if err := os.WriteFile(name, []byte(source), 0600); err != nil { + t.Fatal(err) + } + cmd := exec.Command("go", "tool", "asm", "-S", "-o", filepath.Join(dir, "control.o"), name) + cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH="+goarch) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Go assembler: %v\n%s", err, out) + } + var code []byte + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || !strings.HasPrefix(fields[0], "0x") || len(fields[1]) != 2 { + continue + } + offset, err := strconv.ParseUint(fields[0][2:], 16, 32) + if err != nil || int(offset) != len(code) { + t.Fatalf("noncontiguous assembler bytes: %s", line) + } + for i, field := range fields[1:] { + if i >= 16 || len(field) != 2 { + break + } + value, err := strconv.ParseUint(field, 16, 8) + if err != nil { + break + } + code = append(code, byte(value)) + } + } + return code +} + +func TestX86RawX87ClearExceptionRuntimeSemantics(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("execution requires amd64 or Rosetta; all-host x86 object tests remain required") + } + const source = `TEXT rawClear(SB),$0-0 +BYTE $0xdb; BYTE $0xe2 +RET +TEXT namedClear(SB),$0-0 +FCLEX +RET +TEXT rawInit(SB),$0-0 +BYTE $0xdb; BYTE $0xe3 +RET +TEXT namedInit(SB),$0-0 +FINIT +RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + sigs := make(map[string]FuncSig) + for _, name := range []string{"rawClear", "namedClear", "rawInit", "namedInit"} { + sigs[name] = FuncSig{Name: name, Ret: Void} + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple, prefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +extern void rawClear(void), namedClear(void), rawInit(void), namedInit(void); +static void seed(void) { + uint16_t cw=0x077f; + __asm__ volatile("fninit; fldcw %0; fldz; fldz; fdivp; fstp %%st(0)" : : "m"(cw) : "st"); +} +static uint16_t sw(void) { uint16_t v; __asm__ volatile("fnstsw %0":"=a"(v)); return v; } +static uint16_t cw(void) { uint16_t v; __asm__ volatile("fnstcw %0":"=m"(v)); return v; } +int main(void) { + void (*f[])(void)={rawClear,namedClear,rawInit,namedInit}; + for (int i=0;i<4;i++) { + seed(); + if (!(sw()&1) || cw()!=0x077f) return 20; + f[i](); + if ((sw()&0x80ff) || cw()!=(i<2 ? 0x077f : 0x037f)) { + fprintf(stderr,"x87 operation %d did not update hardware: sw=%x cw=%x\n",i,sw(),cw()); + __asm__ volatile("fninit"); return 1; + } + } + __asm__ volatile("fninit"); return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "raw_x87_clear", triple, ir, mainC, prefix) +} + +func TestX86RawX87SoftwareClearPreservesAllStatusFields(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable() + if runtime.GOARCH != "amd64" && !crossRosetta { + t.Skip("hardware status oracle requires amd64 or Rosetta") + } + const source = `TEXT clearModeledStatus(SB),4,$0-16 +MOVQ env+0(FP),BX +FLDENV (BX) +BYTE $0xdb; BYTE $0xe2 +XORL AX,AX +FSTSW AX +MOVQ AX,ret+8(FP) +RET +` + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + triple := testTargetTriple(runtime.GOOS, runtime.GOARCH) + var prefix []string + if crossRosetta { + triple, prefix = "x86_64-apple-macosx", []string{"/usr/bin/arch", "-x86_64"} + } + ir, err := Translate(file, Options{Goarch: "amd64", TargetTriple: triple, X87Mode: X87Software, Sigs: map[string]FuncSig{ + "clearModeledStatus": {Name: "clearModeledStatus", Args: []LLVMType{Ptr}, Ret: I64, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I64, Index: 0, Field: -1}}, + }}, + }}) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +#include +#include +extern uint64_t clearModeledStatus(const void *); +int main(void) { + struct { unsigned char bytes[28]; } env={{0}}; + uint16_t control=0x037f, tags=0xffff; + memcpy(env.bytes,&control,2); memcpy(env.bytes+8,&tags,2); + for (unsigned v=0;v<=0xffff;v++) { + uint16_t status=v, expected; + memcpy(env.bytes+4,&status,2); + __asm__ volatile("fldenv %1; fnclex; fnstsw %0" : "=a"(expected) : "m"(env) : "memory"); + uint64_t got=clearModeledStatus(&env); + if (got!=expected) { + fprintf(stderr,"status %x: got %llx, hardware %x\n",v,(unsigned long long)got,expected); + __asm__ volatile("fninit"); return 1; + } + } + __asm__ volatile("fninit"); return 0; +} +` + compileAndRunRuntimeTestForTarget(t, llc, clang, "x87_software_clear", triple, ir, mainC, prefix) +} + +func TestX86X87WordStoresToFPParametersAndResults(t *testing.T) { + llc, clang, ok := findLlcAndClang(t) + if !ok { + t.Fatal("LLVM 22 llc/clang not found") + } + var source strings.Builder + sigs := make(map[string]FuncSig) + for _, op := range []string{"FSTCW", "FSTSW"} { + for _, kind := range []string{"param", "result"} { + name := op + kind + fmt.Fprintf(&source, "TEXT %s(SB),4,$0-10\nFINIT\n", name) + if kind == "param" { + fmt.Fprintf(&source, "%s input+0(FP)\nMOVW input+0(FP),AX\nMOVW AX,ret+8(FP)\n", op) + } else { + fmt.Fprintf(&source, "%s ret+8(FP)\n", op) + } + source.WriteString("RET\n") + sigs[name] = FuncSig{Name: name, Args: []LLVMType{I16}, Ret: I16, Frame: FrameLayout{ + Params: []FrameSlot{{Offset: 0, Type: I16, Index: 0, Field: -1}}, Results: []FrameSlot{{Offset: 8, Type: I16, Index: 0, Field: -1}}, + }} + } + } + file, err := Parse(ArchAMD64, source.String()) + if err != nil { + t.Fatal(err) + } + const mainC = ` +#include +extern uint16_t FSTCWparam(uint16_t),FSTCWresult(uint16_t),FSTSWparam(uint16_t),FSTSWresult(uint16_t); +int main(void) { + return FSTCWparam(0xffff)!=0x037f || FSTCWresult(0xffff)!=0x037f || + FSTSWparam(0xffff)!=0 || FSTSWresult(0xffff)!=0; +} +` + for _, triple := range []string{"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc", "i386-unknown-linux-gnu", "i686-pc-windows-msvc", "x86_64-w64-windows-gnu"} { + goarch := "amd64" + if strings.HasPrefix(triple, "i") { + goarch = "386" + } + for _, mode := range []X87Mode{X87Auto, X87Software} { + t.Run(fmt.Sprintf("%s/%d", triple, mode), func(t *testing.T) { + ir, err := Translate(file, Options{Goarch: goarch, TargetTriple: triple, X87Mode: mode, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, triple, "x87-fp.ll", "x87-fp.o", ir) + crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && triple == "x86_64-apple-darwin" && rosettaAvailable() + hostOS := map[string]string{"darwin": "apple-darwin", "linux": "linux-gnu", "windows": "windows-msvc"}[runtime.GOOS] + if crossRosetta || (goarch == runtime.GOARCH && hostOS != "" && strings.Contains(triple, hostOS)) { + var prefix []string + if crossRosetta { + prefix = []string{"/usr/bin/arch", "-x86_64"} + } + runtimeTriple := testTargetTriple(runtime.GOOS, goarch) + // Windows CI provides MinGW, not the MSVC CRT. Retain the + // MSVC object check above, but regenerate the runtime module + // for the actual linker ABI instead of requesting MSVC libs. + runtimeIR, err := Translate(file, Options{Goarch: goarch, TargetTriple: runtimeTriple, X87Mode: mode, Sigs: sigs}) + if err != nil { + t.Fatal(err) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "x87_fp", runtimeTriple, runtimeIR, mainC, prefix) + } + }) + } + } +} + +func TestX86X87WordStoresRejectUnknownFPSlots(t *testing.T) { + for _, op := range []string{"FSTCW", "FSTSW"} { + file, err := Parse(ArchAMD64, "TEXT bad(SB),4,$0-0\n"+op+" unknown+24(FP)\nRET\n") + if err != nil { + t.Fatal(err) + } + for _, arch := range []string{"386", "amd64"} { + if _, err := Translate(file, Options{Goarch: arch, Sigs: map[string]FuncSig{"bad": {Name: "bad", Ret: Void}}}); err == nil { + t.Fatalf("%s accepted %s store to an undeclared FP slot", arch, op) + } + } + } +} diff --git a/amd64_register_result.go b/amd64_register_result.go new file mode 100644 index 000000000..b00865f95 --- /dev/null +++ b/amd64_register_result.go @@ -0,0 +1,79 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +// This lowering-only query observes the already modeled scalar register entry, +// not ABI0 FP fields. The source must explicitly select ABIInternal. It grants +// neither physical Go entry/closure/TLS state nor incoming spill-frame backing. +// See Go's cmd/compile/abi-internal.md, amd64 integer register assignment. +func checkAMD64RegisterResultInitialization(fn Func, sig FuncSig, transfer x86ResultSourceTransfer) error { + fail := func(reason string) error { + return fmt.Errorf("%w: amd64 register result initialization: %s", ErrProbeNeedsContext, reason) + } + registers := [...]Reg{AX, BX, CX, DI, SI, "R8", "R9", "R10", "R11"} + if !strings.HasSuffix(fn.Sym, "") || len(sig.ArgRegs) != 0 || + sig.ARMEntry != nil || sig.ARM64GoRegisterABI != nil || sig.ARM64ClosureABI != nil || + sig.WASMContext != "" || sig.WASMNative || len(sig.Args) > len(registers) || + len(fn.X86RawText) != 0 || fn.x86SourceStack != nil || fn.x86FixedLocal != nil { + return fail("requires a complete ordinary scalar register entry") + } + for _, ins := range fn.Instrs { + if ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86RawNearReturn != nil || + ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || ins.x86SegmentPrefix != 0 { + return fail("raw/native source needs its separate entry contract") + } + } + width := func(typ LLVMType) int64 { + switch typ { + case I8: + return 1 + case I16: + return 2 + case I32: + return 4 + case I64: + return 8 + } + return 0 + } + resultWidth := width(sig.Ret) + if resultWidth == 0 || len(sig.Frame.Results) != 1 { + return fail("requires one complete integer register result") + } + result := sig.Frame.Results[0] + if result.Type != sig.Ret || result.Index != 0 || result.Field != -1 || len(result.Fields) != 0 { + return fail("result transport differs from the scalar signature") + } + initial := x86ResultState{} + for index, typ := range sig.Args { + bytes := width(typ) + if typ == Ptr { + bytes = 8 + } + if bytes == 0 { + return fail("aggregate, floating, boolean or spilled argument needs its own register contract") + } + reg, mask, valid := x86ResultGP(registers[index], bytes, "amd64") + if !valid { + return fail("invalid integer register assignment") + } + if typ == Ptr { + initial.pointerGP[reg] = true // Opaque identity, not numerical bytes or bounds. + } else { + initial.gp[reg] = mask // Narrow inputs do not initialize the unspecified upper bytes. + } + } + // No FP/AUTO ownership is imported from a classic signature or TEXT size. + // The register role must close separately before gaining any stack storage. + return checkX86ResultSourceFlow(fn, initial, nil, 0, "amd64", transfer, + func(state x86ResultState, pc int) error { + reg, mask, _ := x86ResultGP(AX, resultWidth, "amd64") + if state.gp[reg]&mask != mask { + return fail(fmt.Sprintf("AX result bytes are not initialized on every path to RET %d", pc)) + } + return nil + }) +} diff --git a/amd64_register_result_runtime_test.go b/amd64_register_result_runtime_test.go new file mode 100644 index 000000000..29a4408e9 --- /dev/null +++ b/amd64_register_result_runtime_test.go @@ -0,0 +1,129 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func amd64RegisterResultMatrix(t *testing.T) (source, declarations, checks string) { + t.Helper() + var asm, decl, calls strings.Builder + count := 0 + for _, family := range []struct { + suffix string + width int + }{{"B", 1}, {"W", 2}, {"D", 4}, {"Q", 8}} { + for _, prefix := range []string{"", "V"} { + for lane := 0; lane < 16/family.width; lane++ { + name := fmt.Sprint("lane", count) + asm.WriteString(strings.ReplaceAll(amd64RegisterLaneFixture(prefix+"PINSR"+family.suffix, lane), "·view", "·"+name)) + fmt.Fprintf(&decl, "func %s(value,seed uint64) uint64\n", name) + fmt.Fprintf(&calls, "if(check(%s,%d,%d)){fprintf(stderr,\"%s failed\\n\");return 1;}\n", name, family.width, lane, name) + count++ + } + } + } + if count != 60 { + t.Fatal("incomplete register-entry scalar insertion matrix", count) + } + return asm.String(), decl.String(), calls.String() +} + +func requireAMD64RegisterResultGoObject(t *testing.T, source string) { + t.Helper() + dir := t.TempDir() + input, object := filepath.Join(dir, "source.s"), filepath.Join(dir, "source.o") + if err := os.WriteFile(input, []byte(source), 0600); err != nil { + t.Fatal(err) + } + // The shared flag probe handles Go 1.20's explicit -compiling-runtime. + args := append(arm64ClosureGoAsmArgs(t), "-o", object, input) + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, "go", args...) + command.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOOS=linux", "GOARCH=amd64") + output, err := runtimeOracleCombinedOutput(ctx, command) + if failure := goAssemblerOracleOutcome(input, object, output, err, ctx.Err(), true, "amd64"); failure != nil { + t.Fatalf("original Go register-entry object: %v\n%s", failure, output) + } +} + +func TestAMD64RegisterResultThreeTargetObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + source, declarations, _ := amd64RegisterResultMatrix(t) + requireAMD64RegisterResultGoObject(t, source) + for _, triple := range []string{"x86_64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-pc-windows-msvc"} { + for _, api := range []string{"text", "module", "Go"} { + t.Run(triple+"/"+api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + x86ResultInitializationObject(t, llc, triple, text, "register-result") + } + }) + } + } +} + +func TestCrossLinuxRuntimeMatrixAMD64RegisterResultScalarLanes(t *testing.T) { + llc, clang, triple, prefix := x86FloatAddSubRuntimeTools(t) + source, declarations, checks := amd64RegisterResultMatrix(t) + var header strings.Builder + header.WriteString("#include \n#include \n#include \n") + for index := 0; index < 60; index++ { + fmt.Fprintf(&header, "extern uint64_t lane%d(uint64_t,uint64_t);\n", index) + } + const reference = ` +static int check(uint64_t (*fn)(uint64_t,uint64_t), unsigned width, unsigned lane) { + const uint64_t values[] = {0, UINT64_MAX, UINT64_C(0x8070605040302010), + UINT64_C(0xfedcba9876543210)}; + for (unsigned a = 0; a < 4; a++) { + for (unsigned b = 0; b < 4; b++) { + uint8_t bytes[16] = {0}; + uint64_t low, high; + memcpy(bytes, &values[b], 8); + memcpy(bytes + width * lane, &values[a], width); + memcpy(&low, bytes, 8); + memcpy(&high, bytes + 8, 8); + uint64_t got = fn(values[a], values[b]); + if (got != (low ^ high)) { + fprintf(stderr, "width=%u lane=%u a=%u b=%u got=%llx want=%llx\n", + width, lane, a, b, (unsigned long long)got, (unsigned long long)(low ^ high)); + return 1; + } + } + } + return 0; +} +` + main := header.String() + reference + "int main(void){\n" + checks + "return 0;}\n" + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source, declarations, "amd64", triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + compileAndRunRuntimeTestForTarget(t, llc, clang, "register-results", triple, text, main, prefix) + } + }) + } + t.Log("60 register-entry forms × 16 independent byte inputs × 3 APIs × O0/O2; portable lowering, not a physical Go call bridge") +} diff --git a/amd64_register_result_test.go b/amd64_register_result_test.go new file mode 100644 index 000000000..c360718bb --- /dev/null +++ b/amd64_register_result_test.go @@ -0,0 +1,120 @@ +package plan9asm + +import ( + "errors" + "fmt" + "go/types" + "strings" + "testing" +) + +func amd64RegisterLaneFixture(op string, lane int) string { + destination := "X0" + insert := fmt.Sprintf("%s $%d,AX,X0", op, lane) + if strings.HasPrefix(op, "V") { + destination = "X1" + insert += ",X1" + } + return fmt.Sprintf("TEXT ·view(SB),4,$0-24\nMOVQ BX,X0\n%s\nPEXTRQ $1,%s,CX\nMOVQ %s,AX\nXORQ CX,AX\nRET\n", + insert, destination, destination) +} + +// The explicit register entry has no initialized ABI0 FP fields. This is the +// same seed/insert pattern as the SDK hash routines, without AES or globals. +func TestAMD64RegisterResultScalarLaneThreeAPIs(t *testing.T) { + for _, op := range []string{"PINSRB", "PINSRW", "PINSRD", "PINSRQ", "VPINSRB", "VPINSRW", "VPINSRD", "VPINSRQ"} { + t.Run(op, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, amd64RegisterLaneFixture(op, 255), + "func view(value,seed uint64) uint64\n", "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil || ir == "" { + t.Errorf("%s: register result incorrectly requires ABI0 result storage: %v", api, err) + } + } + }) + } +} + +func TestAMD64RegisterResultKeepsUnknownInputsAndSourceRoles(t *testing.T) { + const declaration = "func view(value,seed uint64) uint64\n" + source := amd64RegisterLaneFixture("PINSRQ", 1) + for name, bad := range map[string]string{ + "ABI0": strings.ReplaceAll(source, "", ""), + "unknown input": strings.ReplaceAll(source, "$1,AX,X0", "$1,R12,X0"), + "unknown return": strings.ReplaceAll(source, "RET", "MOVQ R12,AX\nRET"), + "ABI0 input": strings.ReplaceAll(source, "MOVQ BX,X0", "MOVQ seed+8(FP),X0"), + "ABI0 result": strings.ReplaceAll(source, "RET", "MOVQ AX,ret+16(FP)\nRET"), + "hidden g": strings.ReplaceAll(source, "$1,AX,X0", "$1,R14,X0"), + "hidden closure": strings.ReplaceAll(source, "$1,AX,X0", "$1,DX,X0"), + "physical stack": strings.ReplaceAll(source, "MOVQ BX,X0", "MOVQ (SP),X0"), + "unknown effect": strings.ReplaceAll(source, "RET", "RDTSC\nRET"), + "path loses value": strings.ReplaceAll(source, "RET", "CMPQ BX,$0\nJE done\nMOVQ R12,AX\ndone:\nRET"), + } { + t.Run(name, func(t *testing.T) { + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, bad, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Errorf("%s: unbound entry/input/result was accepted: %v", api, err) + } + } + }) + } +} + +func TestAMD64RegisterResultIntegerWidthsAndEveryArgumentRegister(t *testing.T) { + registers := []string{"AX", "BX", "CX", "DI", "SI", "R8", "R9", "R10", "R11"} + for _, width := range []struct{ typ, insert, extract string }{ + {"uint8", "PINSRB", "PEXTRB"}, {"uint16", "PINSRW", "PEXTRW"}, + {"uint32", "PINSRD", "PEXTRD"}, {"uint64", "PINSRQ", "PEXTRQ"}, + } { + for index, reg := range registers { + t.Run(fmt.Sprintf("%s/%s", width.typ, reg), func(t *testing.T) { + parameters := strings.TrimSuffix(strings.Repeat(width.typ+",", index+1), ",") + declaration := fmt.Sprintf("func view(%s) %s\n", parameters, width.typ) + source := fmt.Sprintf("TEXT ·view(SB),4,$0-80\n%s $0,%s,X0\n%s $0,X0,AX\nRET\n", + width.insert, reg, width.extract) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", api); err != nil { + t.Errorf("%s: initialized selected scalar bytes rejected: %v", api, err) + } + } + }) + } + } +} + +func TestAMD64RegisterResultRejectsUnmodeledTransport(t *testing.T) { + source := amd64RegisterLaneFixture("PINSRQ", 1) + for _, declaration := range []string{ + "func view(value uint8,seed uint64) uint64", // Unknown upper argument bytes. + "func view(value uint16,seed uint64) uint64", + "func view(value uint32,seed uint64) uint64", + "func view(value *uint64,seed uint64) uint64", // Pointer is not numeric. + "func view(value [2]uint64,seed uint64) uint64", + "func view(value float64,seed uint64) uint64", + "func view(value bool,seed uint64) uint64", + "func view(value,seed uint64) (uint64,uint64)", + "func view(value,seed uint64) *uint64", + "func view(value,seed uint64) bool", + "func view(" + strings.TrimSuffix(strings.Repeat("uint64,", 10), ",") + ") uint64", + } { + t.Run(declaration, func(t *testing.T) { + pkg := mustGoPackage(t, "registerresult", "package registerresult\n"+declaration) + sig, err := DeriveGoFuncABI0Signature("view", pkg.Types.Scope().Lookup("view").(*types.Func), "amd64", types.SizesFor("gc", "amd64")) + if err != nil { + t.Fatal(err) + } + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + ctx := &amd64Ctx{fpScalarSource: file.Funcs[0], sig: sig, goarch: "amd64"} + if err := ctx.checkLoweredScalarFPResultInitialization(); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unmodeled transport gained register initialization:", err) + } + if err := CheckX86ABI0ResultInitialization(file.Funcs[0], sig, "amd64"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("register role escaped into the standalone ABI0 observer:", err) + } + }) + } +} diff --git a/amd64_scalar_arithmetic_native.go b/amd64_scalar_arithmetic_native.go new file mode 100644 index 000000000..182442541 --- /dev/null +++ b/amd64_scalar_arithmetic_native.go @@ -0,0 +1,73 @@ +package plan9asm + +import "fmt" + +// The existing whole Go-table grammar supplies operand width, encoded byte +// aliases and carry roles. This query selects only its memory-to-register +// add/subtract rows; it supplies no memory or entry capability. +func x86ScalarArithmeticNativeReadProperties(ins Instr, arch string) (x86LockScalarRMWForm, bool, error) { + form, known, err := x86LockScalarRMWProperties(ins.Op, ins, arch) + if !known { + return form, false, nil + } + if err != nil { + return form, true, err + } + switch form.spec.stem { + case "ADD", "SUB", "ADC", "SBB": + default: + return form, false, nil + } + if form.source.Kind != OpMem || form.source.Mem.Sym != "" || + form.destination.Kind != OpReg || !x86OrdinaryNativeMemoryOperand(form.source, arch) { + return form, false, nil + } + return form, true, nil +} + +func (c *amd64Ctx) evalScalarArithmeticSource(ins Instr, typ LLVMType) (string, Operand, error) { + form, known, err := x86ScalarArithmeticNativeReadProperties(ins, c.goarch) + if err != nil { + return "", Operand{}, err + } + if !known { + value, err := c.evalIntSized(ins.Args[0], typ) + return value, ins.Args[1], err + } + if ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + form.destination.Reg != ins.Args[1].Reg && (form.destination.Reg == SP || form.destination.Reg == BP) { + return "", Operand{}, fmt.Errorf("%w: scalar arithmetic needs its original native address/register contract", ErrProbeNeedsContext) + } + pointer, pointerType, _, err := c.packedVectorMemoryPointer(form.source) + if err != nil { + return "", Operand{}, err + } + value := c.newTmp() + // Preserve the fault and exact read even when all numerical results and + // flags are discarded. Do not assert alignment, bounds or pointee validity. + fmt.Fprintf(c.b, " %%%s = load volatile %s, %s %s, align 1\n", value, typ, pointerType, pointer) + return "%" + value, form.destination, nil +} + +// Only the lowering paired with the exact volatile source load may consume +// its numerical continuation. The destination and incoming carry still need +// every source byte/flag. Public declaration and caller observers stay strict. +func x86LoweredScalarArithmeticNativeRead(ins Instr, state *x86ResultState, owned []bool, resultOffset int64, arch string) (bool, bool) { + form, known, err := x86ScalarArithmeticNativeReadProperties(ins, arch) + if !known || err != nil { + return false, known + } + width := int64(form.spec.bits / 8) + index, _, valid := x86ResultGP(form.destination.Reg, width, arch) + if !valid || state.pointerGP[index] || + form.spec.carryInput && state.flags&x86ByteFlagCF == 0 || + !x86ResultRead(form.destination, width, state, owned, arch) || + !x86ResultReadWithNativeContinuation(ins, form.source, width, state, owned, arch, true) { + return false, true + } + if !x86ResultStore(form.destination, width, state, owned, resultOffset, arch) { + return false, true + } + state.flags = state.flags&^form.spec.flagsKilled | form.spec.flagsDefined + return true, true +} diff --git a/amd64_scalar_arithmetic_native_result_test.go b/amd64_scalar_arithmetic_native_result_test.go new file mode 100644 index 000000000..a30b29384 --- /dev/null +++ b/amd64_scalar_arithmetic_native_result_test.go @@ -0,0 +1,224 @@ +package plan9asm + +import ( + "errors" + "fmt" + "reflect" + "strings" + "testing" +) + +// Keep this inventory independent of the production descriptors. These are +// all four widths of Go's add/subtract and carry/borrow families. +var x86ScalarArithmeticNativeNames = []string{ + "ADDB", "ADDW", "ADDL", "ADDQ", + "SUBB", "SUBW", "SUBL", "SUBQ", + "ADCB", "ADCW", "ADCL", "ADCQ", + "SBBB", "SBBW", "SBBL", "SBBQ", +} + +func TestX86ScalarArithmeticNativeSparseBytesFlagsAndOrigins(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86ScalarArithmeticNativeNames { + width := map[byte]int64{'B': 1, 'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + if arch == "386" && width == 8 { + continue + } + t.Run(arch+"/"+op, func(t *testing.T) { + ins := x86ScalarLaneInstruction(t, op+" -1(DX)(BX*1),CX") + base, _, _ := x86ResultGP(DX, 4, arch) + index, _, _ := x86ResultGP(BX, 4, arch) + destination, mask, _ := x86ResultGP(CX, width, arch) + for seed := 0; seed < 256; seed++ { + for flags := uint8(0); flags <= x86ByteFlagsAll; flags++ { + state := x86ResultState{flags: flags} + state.pointerGP[base], state.gp[index], state.gp[destination] = true, 0xff, uint8(seed) + want := state.clone() + accepted := uint8(seed)&mask == mask && + (!strings.HasPrefix(op, "ADC") && !strings.HasPrefix(op, "SBB") || flags&x86ByteFlagCF != 0) + if accepted { + want.flags = x86ByteFlagsAll + if width == 4 { + want.gp[destination] = 0xff + } + } + if got, known := x86LoweredScalarArithmeticNativeRead(ins, &state, nil, 0, arch); !known || got != accepted || !reflect.DeepEqual(state, want) { + t.Fatal("arithmetic changed byte/FLAGS roles", seed, flags, got, accepted, state, want) + } + } + } + for _, origin := range []string{"decoded", "raw", "raw return", "address", "unbound address", "segment", "pointer destination"} { + changed := ins + state := x86ResultState{flags: x86ByteFlagsAll} + state.pointerGP[base], state.gp[index], state.gp[destination] = true, 0xff, 0xff + switch origin { + case "decoded": + changed.x86Encoded = true + case "raw": + changed.x86SourceRaw = new(x86SourceRawInstruction) + case "raw return": + changed.x86RawNearReturn = new(x86RawNearReturnForm) + case "address": + changed.x86AddressBits = 32 + case "unbound address": + changed.x86UnboundAddressOverride = true + case "segment": + changed.x86SegmentPrefix = 0x64 + case "pointer destination": + state.pointerGP[destination] = true + } + before := state.clone() + if accepted, known := x86LoweredScalarArithmeticNativeRead(changed, &state, nil, 0, arch); !known || accepted || !reflect.DeepEqual(state, before) { + t.Fatal("unbound origin acquired native arithmetic", origin, state) + } + } + }) + } + } +} + +func TestX86ScalarArithmeticNativeOriginalREXByteAliases(t *testing.T) { + for _, row := range []struct { + op string + byte byte + }{{"ADDB", 0x02}, {"SUBB", 0x2a}, {"ADCB", 0x12}, {"SBBB", 0x1a}} { + for ordinal, reg := range []string{"AH", "CH", "DH", "BH"} { + for _, address := range []string{"(DX)", "(R8)", "(DX)(R9*2)"} { + instruction := row.op + " " + address + "," + reg + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+instruction+"\nRET\n") + want := []byte{row.byte, byte(0x22 + ordinal*8), 0xc3} + effective := Reg(reg) + if address == "(R8)" { + want = []byte{0x41, row.byte, byte(0x20 + ordinal*8), 0xc3} + } else if address == "(DX)(R9*2)" { + want = []byte{0x42, row.byte, byte(0x24 + ordinal*8), 0x4a, 0xc3} + } + if address != "(DX)" { + effective = []Reg{SP, BP, SI, DI}[ordinal] + } + if !reflect.DeepEqual(code, want) { + t.Fatalf("original REX/ModRM view changed: %s: %x != %x", instruction, code, want) + } + ins := x86ScalarLaneInstruction(t, instruction) + form, known, err := x86ScalarArithmeticNativeReadProperties(ins, "amd64") + if !known || err != nil || form.destination.Reg != effective { + t.Fatal("typed destination differs from actual Go encoding", instruction, form, err) + } + if address != "(DX)" && ordinal < 2 { + c, output := newAMD64CtxWithFuncForTest(t, Func{}, FuncSig{Name: "example.Value", Ret: Void}, nil) + before := output.String() + if _, _, err := c.evalScalarArithmeticSource(ins, I8); !errors.Is(err, ErrProbeNeedsContext) || output.String() != before { + t.Fatal("arithmetic manufactured a physical SP/BP alias", err) + } + } + } + } + } +} + +func TestX86ScalarArithmeticNativeResultsThreeAPIFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + count := 0 + for _, op := range x86ScalarArithmeticNativeNames { + if target.arch == "386" && strings.HasSuffix(op, "Q") { + goAssemblerOracleOutput(t, target.arch, "TEXT f(SB),4,$0-0\n"+op+" (DX),CX\nRET\n", "", false) + continue + } + for _, address := range []string{"(DX)", "-1(DX)(BX*1)"} { + body := "CMPL BX,BX\n" + op + " " + address + ",CX" + asm, decl := x86ExtensionNativeFixture(fmt.Sprintf("F%d", count), target.arch, body, false) + source.WriteString(asm) + declarations.WriteString(strings.ReplaceAll(decl, "*[4]byte", "*[16]byte")) + count++ + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("native arithmetic lost its paired numerical continuation", api, err) + } + for _, optimized := range []bool{false, true} { + text := ir + if optimized { + text = x86MulDivRoundOptimize(t, text) + } + if strings.Count(text, "load volatile i") != count { + t.Fatal("native arithmetic lost its exact observable reads", api, optimized, count) + } + compileLLVMToObject(t, llc, target.triple, "arithmetic-native.ll", "arithmetic-native.o", text) + } + } + }) + } +} + +func TestX86ScalarArithmeticDeadNativeReadsSurviveOptimization(t *testing.T) { + for _, op := range x86ScalarArithmeticNativeNames { + t.Run(op, func(t *testing.T) { + source, declaration := x86ExtensionNativeFixture("Value", "amd64", "CMPL BX,BX\n"+op+" (DX),CX", true) + goAssemblerOracleOutput(t, "amd64", source, "", true) + width := map[byte]int{'B': 8, 'W': 16, 'L': 32, 'Q': 64}[op[len(op)-1]] + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source, declaration, "amd64", "x86_64-unknown-linux-gnu", api) + if err != nil { + t.Fatal(err) + } + for _, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if strings.Count(text, fmt.Sprintf("load volatile i%d,", width)) != 1 { + t.Errorf("%s dead native read must retain exactly %d bits", api, width) + } + } + } + }) + } +} + +func TestX86ScalarArithmeticNativeRetainsPublicAndSourceBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + word, move, triple := 8, "MOVQ", "x86_64-unknown-linux-gnu" + if arch == "386" { + word, move, triple = 4, "MOVL", "i386-unknown-linux-gnu" + } + for _, op := range x86ScalarArithmeticNativeNames { + if arch == "386" && strings.HasSuffix(op, "Q") { + continue + } + t.Run(arch+"/"+op, func(t *testing.T) { + body := fmt.Sprintf("%s p+0(FP),DX\nMOVL $0,CX\nCMPL CX,CX\n%s (DX),CX\nMOVL CX,ret0+%d(FP)\nRET", move, op, word) + _, file, sig, _ := x86ResultInitializationFixture(t, arch, "func Value(p *[16]byte) uint32", body) + for _, check := range []func(Func, FuncSig, string) error{CheckX86ABI0ResultInitialization, checkX86ScalarFPResultInitialization} { + if err := check(file.Funcs[0], sig, arch); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("unpaired declaration check acquired native storage", err) + } + } + for name, body := range map[string]string{ + "unknown base": op + " (DI),CX", + "numeric base": "MOVL $42,DX\n" + op + " (DX),CX", + "unknown index": op + " (DX)(SI*1),CX", + "pointer index": op + " (DX)(DX*1),CX", + "unknown result": op + " (DX),DI\n" + move + " DI,CX", + "pointer result": op + " (DX),DX\n" + move + " DX,CX", + "native store": op + " CX,(DX)", + "destroyed pointer": "MOVL $42,DX\n" + op + " (DX),CX", + } { + t.Run(name, func(t *testing.T) { + source, declaration := x86ExtensionNativeFixture("Value", arch, "CMPL BX,BX\n"+body, false) + goAssemblerOracleOutput(t, arch, source, "", true) + for _, api := range []string{"text", "module", "Go"} { + if _, err := x86ScalarFPBitViewAPI(t, source, declaration, arch, triple, api); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatal("native arithmetic acquired an unproved source effect", api, err) + } + } + }) + } + }) + } + } +} diff --git a/amd64_scalar_arithmetic_native_runtime_unix_test.go b/amd64_scalar_arithmetic_native_runtime_unix_test.go new file mode 100644 index 000000000..7f139e6bc --- /dev/null +++ b/amd64_scalar_arithmetic_native_runtime_unix_test.go @@ -0,0 +1,192 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ScalarArithmeticNativeResultsAndFaults(t *testing.T) { + x86ScalarArithmeticNativeRuntime(t, false) +} + +func TestCrossLinuxRuntimeMatrixX86ScalarArithmeticNativeFlagsAndAliases(t *testing.T) { + x86ScalarArithmeticNativeRuntime(t, true) +} + +func x86ScalarArithmeticNativeRuntime(t *testing.T, flagsAndAliases bool) { + t.Helper() + execution := x86FloatCompareExecutionTools(t) + var source, declarations, prototypes, originals, rows strings.Builder + count := 0 + appendKernel := func(op string, width, carry int, dead, flags bool, alias string) { + name, original := fmt.Sprintf("F%d", count), fmt.Sprintf("Original%d", count) + comparison := "CMPL BX,BX\n" + nativeComparison := "MOVL $1,R10\nCMPL R10,R10\n" + if carry != 0 { + comparison = "CMPL BX,$2\n" + nativeComparison = "MOVL $1,R10\nCMPL R10,$2\n" + } + body := comparison + op + " -1(DX)(BX*1),CX" + native := "MOVQ $0x87654321fedcba98,AX\n" + nativeComparison + op + " (DI),AX\n" + mode := 0 // value result + if flags { + body += "\n" + x86ScalarArithmeticNativeFlagObserver + "\nMOVQ AX,CX" + native += x86ScalarArithmeticNativeFlagObserver + "\n" + mode = 1 + } + if alias != "" { + effective := "SI" + if alias == "BH" { + effective = "DI" + } + body = "MOVQ DX,R8\nMOVQ $0x87654321fedcba98," + effective + "\n" + comparison + op + " (R8)," + alias + "\nMOVQ " + effective + ",CX" + native = "MOVQ DI,R8\nMOVQ $0x87654321fedcba98," + effective + "\n" + nativeComparison + op + " (R8)," + alias + "\nMOVQ " + effective + ",AX\n" + } + if dead { + body += "\nMOVQ $0x87654321fedcba98,CX" + native += "MOVQ $0x87654321fedcba98,AX\n" + } + assembly, declaration := x86ExtensionNativeFixture(name, "amd64", body, false) + source.WriteString(assembly) + declarations.WriteString(strings.ReplaceAll(declaration, "*[4]byte", "*[16]byte")) + code := x86OrderingHintGoBytes(t, "amd64", "TEXT native(SB),4,$0-0\n"+native+"RET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&originals, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(const uint8_t*);\nextern uint64_t %s(const uint8_t*);\n", name, original) + subtract, discarded := 0, 0 + if strings.HasPrefix(op, "SUB") || strings.HasPrefix(op, "SBB") { + subtract = 1 + } + if dead { + discarded = 1 + } + effectiveCarry := 0 + if strings.HasPrefix(op, "ADC") || strings.HasPrefix(op, "SBB") { + effectiveCarry = carry + } + fmt.Fprintf(&rows, "{%s,%s,%d,%d,%d,%d,%d},\n", name, original, width, subtract, effectiveCarry, discarded, mode) + count++ + } + for _, op := range x86ScalarArithmeticNativeNames { + width := map[byte]int{'B': 1, 'W': 2, 'L': 4, 'Q': 8}[op[len(op)-1]] + for carry := 0; carry < 2; carry++ { + if flagsAndAliases { + appendKernel(op, width, carry, false, true, "") + if width == 1 { + for _, alias := range []string{"DH", "BH"} { + appendKernel(op, width, carry, false, false, alias) + } + } + } else { + for _, dead := range []bool{false, true} { + appendKernel(op, width, carry, dead, false, "") + } + } + } + } + wantCount := 64 + if flagsAndAliases { + wantCount = 48 + } + if count != wantCount { + t.Fatal("incomplete arithmetic/native state matrix", count, wantCount) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + // Reuse only the independent process/guard-page machinery. The arithmetic + // reference below does not consume a production descriptor or lowering. + start := strings.Index(x86ExtensionNativeRuntimeC, "static uint64_t reference(") + end := strings.Index(x86ExtensionNativeRuntimeC, "static int fault(") + if start < 0 || end <= start { + t.Fatal("missing complete fault-harness boundary") + } + driverTemplate := x86ExtensionNativeRuntimeC[:start] + x86ScalarArithmeticNativeReferenceC + "\n" + x86ExtensionNativeRuntimeC[end:] + driverTemplate = strings.NewReplacer( + "unsigned width, destination, sign, dead;", "unsigned width, subtract, carry, dead, flags;", + "uint8_t bytes[8];", "uint8_t bytes[16];", + "const uint8_t *p=bytes+(pattern&1);", ` + uint8_t *p=bytes+(pattern&1); + uint64_t mask=k.width==8?UINT64_MAX:(UINT64_C(1)<<(k.width*8))-1; + uint64_t sign=UINT64_C(1)<<(k.width*8-1), a=seed&mask; + const uint64_t edges[]={0,1,mask,mask-1,sign,sign-1,a,a-1,a+1,~a,-a,-a-1}; + if(pattern>(8*b)); + }`, + "28 extension kernels", fmt.Sprintf("%d scalar arithmetic kernels", count), + ).Replace(driverTemplate) + driver := fmt.Sprintf(driverTemplate, prototypes.String(), originals.String(), rows.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), "amd64", execution.triple, api) + if err != nil { + t.Fatal(err) + } + for _, optimized := range []bool{false, true} { + t.Run(fmt.Sprintf("O2=%t", optimized), func(t *testing.T) { + text, level := ir, "-O0" + if optimized { + text, level = x86MulDivRoundOptimize(t, ir), "-O2" + } + output := execution.compileAndRunLLVM(t, "scalar-arithmetic-native", text, driver, level) + want := fmt.Sprintf("%d scalar arithmetic kernels: numerical %d, fault %d, tripwires 2; failures 0\n", count, count*128, count*12) + if string(output) != want { + t.Fatal("original Go/LLVM arithmetic differs", string(output), want) + } + t.Log(strings.TrimSpace(string(output))) + }) + } + }) + } +} + +// Capture every SETcc-observable status bit before combining them with +// flag-writing arithmetic. MOV/SETcc preserve the original flags. +const x86ScalarArithmeticNativeFlagObserver = `MOVL $0,AX +MOVL $0,SI +MOVL $0,DI +MOVL $0,R10 +MOVL $0,R11 +SETCS AL +SETEQ SIB +SETMI DIB +SETOS R10B +SETPS R11B +SHLL $1,SI +SHLL $2,DI +SHLL $3,R10 +SHLL $4,R11 +ORL SI,AX +ORL DI,AX +ORL R10,AX +ORL R11,AX` + +const x86ScalarArithmeticNativeReferenceC = ` +static uint64_t reference(Kernel k, const uint8_t *p) { + if(k.dead) return seed; + uint64_t input=0; + for(unsigned b=0;b>bit)&1; + return cf|((value==0)<<1)|(((value&sign)!=0)<<2)|(of<<3)|((ones%%2==0)<<4); + } + if(k.width<4) return (seed&~mask)|value; + return value; +} +` diff --git a/amd64_scalar_bitcount.go b/amd64_scalar_bitcount.go new file mode 100644 index 000000000..9ee00dddd --- /dev/null +++ b/amd64_scalar_bitcount.go @@ -0,0 +1,112 @@ +package plan9asm + +import ( + "fmt" + "strings" +) + +type x86ScalarBitCountKind uint8 + +const ( + x86PopulationCount x86ScalarBitCountKind = iota + x86LeadingZeroCount + x86TrailingZeroCount +) + +type x86ScalarBitCountSpec struct { + bits int + kind x86ScalarBitCountKind +} + +// Complete Go yml_rl/ycrc32l scalar count rows. Shape, lowering and numerical +// byte effects share this table; it grants no native memory/caller permission. +var x86ScalarBitCountSpecs = map[Op]x86ScalarBitCountSpec{ + "POPCNTW": {16, x86PopulationCount}, "POPCNTL": {32, x86PopulationCount}, "POPCNTQ": {64, x86PopulationCount}, + "LZCNTW": {16, x86LeadingZeroCount}, "LZCNTL": {32, x86LeadingZeroCount}, "LZCNTQ": {64, x86LeadingZeroCount}, + "TZCNTW": {16, x86TrailingZeroCount}, "TZCNTL": {32, x86TrailingZeroCount}, "TZCNTQ": {64, x86TrailingZeroCount}, +} + +func x86ScalarBitCountProperties(op Op, ins Instr, arch string) (x86ScalarBitCountSpec, bool, error) { + name, _, dotted := strings.Cut(strings.ToUpper(string(op)), ".") + spec, known := x86ScalarBitCountSpecs[Op(name)] + if !known { + return spec, false, nil + } + fail := func() (x86ScalarBitCountSpec, bool, error) { + return spec, true, fmt.Errorf("%s %s outside Go scalar bit-count grammar: %q", arch, op, ins.Raw) + } + if arch != "amd64" && arch != "386" || arch == "386" && spec.bits == 64 || dotted || + len(ins.Args) != 2 || ins.Op != "" && !strings.EqualFold(string(ins.Op), name) { + return fail() + } + source, destination := ins.Args[0], ins.Args[1] + if destination.Kind != OpReg || !isX86YrlRegisterForArch(destination.Reg, arch) { + return fail() + } + if source.Kind == OpReg { + if !isX86YrlRegisterForArch(source.Reg, arch) { + return fail() + } + } else if !isAMD64MemoryOperand(source) || + source.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(source.Sym), "$") { + return fail() + } + if source.Kind == OpMem && !x86MemoryRegistersValidForArch(source.Mem, arch) { + return fail() + } + return spec, true, nil +} + +func x86TypedScalarBitCountByteEffect(op Op, ins Instr, arch string) (x86TypedByteEffect, bool, error) { + spec, known, err := x86ScalarBitCountProperties(op, ins, arch) + if !known || err != nil { + return x86TypedByteEffect{}, known, err + } + defined := x86ByteFlagCF | x86ByteFlagZF + if spec.kind == x86PopulationCount { + defined = x86ByteFlagsAll + } + width := int64(spec.bits / 8) + return x86TypedByteEffect{ + operands: []x86TypedByteOperand{ + {index: 0, operand: ins.Args[0], width: width, read: true}, + {index: 1, operand: ins.Args[1], width: width, write: true}, + }, + flagsDefined: defined, + flagsKilled: x86ByteFlagsAll, + }, true, nil +} + +func (c *amd64Ctx) lowerScalarBitCount(op Op, ins Instr) (bool, bool, error) { + spec, known, err := x86ScalarBitCountProperties(op, ins, c.goarch) + if !known || err != nil { + return known, false, err + } + typ := amd64IntegerTypeForBits(spec.bits) + value, err := c.evalIntSized(ins.Args[0], typ) + if err != nil { + return true, false, err + } + count := c.newTmp() + if spec.kind == x86PopulationCount { + fmt.Fprintf(c.b, " %%%s = call %s @llvm.ctpop.%s(%s %s)\n", count, typ, typ, typ, value) + } else { + intrinsic := "cttz" + if spec.kind == x86LeadingZeroCount { + intrinsic = "ctlz" + } + fmt.Fprintf(c.b, " %%%s = call %s @llvm.%s.%s(%s %s, i1 false)\n", count, typ, intrinsic, typ, typ, value) + } + inputZero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %s, 0\n", inputZero, typ, value) + if spec.kind == x86PopulationCount { + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", inputZero, c.flagsZSlot) + c.clearX86StatusFlags(x86StatusAll &^ x86StatusZF) + } else { + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", inputZero, c.flagsCFSlot) + zero := c.newTmp() + fmt.Fprintf(c.b, " %%%s = icmp eq %s %%%s, 0\n", zero, typ, count) + fmt.Fprintf(c.b, " store i1 %%%s, ptr %s\n", zero, c.flagsZSlot) + } + return true, false, c.storeRegSized(ins.Args[1].Reg, typ, "%"+count) +} diff --git a/amd64_scalar_bitcount_test.go b/amd64_scalar_bitcount_test.go new file mode 100644 index 000000000..02b8db39c --- /dev/null +++ b/amd64_scalar_bitcount_test.go @@ -0,0 +1,105 @@ +package plan9asm + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestX86ScalarBitCountCompleteGoRows(t *testing.T) { + table, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + rows := []string{ + "{ALZCNTL, yml_rl, Pf3, opBytes{0xbd}}", + "{ALZCNTQ, yml_rl, Pfw, opBytes{0xbd}}", + "{ALZCNTW, yml_rl, Pef3, opBytes{0xbd}}", + "{APOPCNTW, yml_rl, Pef3, opBytes{0xb8}}", + "{APOPCNTL, yml_rl, Pf3, opBytes{0xb8}}", + "{APOPCNTQ, yml_rl, Pfw, opBytes{0xb8}}", + "{ATZCNTW, ycrc32l, Pe, opBytes{0xf3, 0x0f, 0xbc, 0}}", + "{ATZCNTL, ycrc32l, Px, opBytes{0xf3, 0x0f, 0xbc, 0}}", + "{ATZCNTQ, ycrc32l, Pw, opBytes{0xf3, 0x0f, 0xbc, 0}}", + } + if len(x86ScalarBitCountSpecs) != len(rows) { + t.Fatal("incomplete scalar count family") + } + for _, row := range append(rows, + "var yml_rl = []ytab{\n\t{Zm_r, 1, argList{Yml, Yrl}},\n}", + "var ycrc32l = []ytab{\n\t{Zlitm_r, 0, argList{Yml, Yrl}},\n}") { + if !strings.Contains(string(table), row) { + t.Fatal("complete original Go encoding row changed:", row) + } + } + for _, family := range []string{"POPCNT", "LZCNT", "TZCNT"} { + for _, suffix := range []string{"W", "L", "Q"} { + op := Op(family + suffix) + for _, arch := range []string{"amd64", "386"} { + for _, input := range []string{"AX", "AL", "X0", "$3", "$table(SB)", "4(AX)(CX*2)", "table(SB)"} { + for _, output := range []string{"BX", "AH", "X1", "4(BX)"} { + source := fmt.Sprintf("TEXT probe(SB),4,$0-0\n%s %s,%s\nRET\n", op, input, output) + valid := !(arch == "386" && suffix == "Q") && output == "BX" && + (input == "AX" || input == "4(AX)(CX*2)" || input == "table(SB)") + goAssemblerOracleOutput(t, arch, source, "", valid) + file, err := Parse(ArchAMD64, source) + if err != nil { + t.Fatal(err) + } + _, known, err := x86ScalarBitCountProperties(op, file.Funcs[0].Instrs[1], arch) + if !known || (err == nil) != valid { + t.Fatalf("%s %s %s,%s: known=%t error=%v", arch, op, input, output, known, err) + } + } + } + } + } + } +} + +func TestX86ScalarBitCountFiveTargetThreeAPIObjects(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86OrderingHintTargets { + word, typ, move := 8, "uint64", "MOVQ" + if target.arch == "386" { + word, typ, move = 4, "uint32", "MOVL" + } + var declarations, source strings.Builder + declarations.WriteString("package ordering\n") + for _, family := range []string{"POPCNT", "LZCNT", "TZCNT"} { + for _, suffix := range []string{"W", "L", "Q"} { + if target.arch == "386" && suffix == "Q" { + continue // Independently Go-rejected above, not a skipped test. + } + op := family + suffix + name := strings.ToLower(op) + fmt.Fprintf(&declarations, "func %s(x %s) %s\n", name, typ, typ) + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-%d\n%s x+0(FP),AX\n%s $0,BX\n%s AX,BX\n%s BX,ret0+%d(FP)\nRET\n", + name, 2*word, move, move, op, move, word) + } + } + for _, api := range []string{"text", "module", "Go"} { + t.Run(target.triple+"/"+api, func(t *testing.T) { + ir, err := x86OrderingHintTranslateAPI(t, api, source.String(), declarations.String(), target.arch, target.triple) + if err != nil { + t.Fatal(err) + } + for _, level := range []string{"-O0", "-O2"} { + dir := t.TempDir() + input, object := filepath.Join(dir, "count.ll"), filepath.Join(dir, "count.o") + if err := os.WriteFile(input, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + x86FloatCompareCaptureCommand(t, exec.Command(llc, level, "-mtriple="+target.triple, "-filetype=obj", input, "-o", object)) + } + }) + } + } +} diff --git a/amd64_scalar_compare_fault_runtime_unix_test.go b/amd64_scalar_compare_fault_runtime_unix_test.go new file mode 100644 index 000000000..bdde6b432 --- /dev/null +++ b/amd64_scalar_compare_fault_runtime_unix_test.go @@ -0,0 +1,100 @@ +//go:build unix + +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ScalarCompareDeadReadFaults(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + for index, op := range x86ScalarCompareNames("amd64") { + name, original := fmt.Sprintf("dead%d", index), fmt.Sprintf("original%d", index) + // TEST $0 still reads its entire original memory operand. Neither its + // tautological flags nor a later independent result may suppress faults. + instruction := x86ScalarCompareInstruction(op, "(AX)", "$0") + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-8\nMOVQ p+0(FP),AX\n%s\nRET\n", name, instruction) + fmt.Fprintf(&declarations, "func %s(p *byte)\n", name) + code := x86PackedShiftActualGoBytes(t, "amd64", "TEXT f(SB),4,$0-0\n"+x86ScalarCompareInstruction(op, "(DI)", "$0")+"\nRET\n") + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern void %s(uint8_t*),%s(uint8_t*);\n", name, original) + fmt.Fprintf(&pairs, "{%s,%s,%d},\n", original, name, []int{1, 2, 4, 8}[index%4]) + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +#include +#include +#include +#include +` + prototypes.String() + native.String() + ` +typedef void (*program)(uint8_t*); +static const struct { program original,translated; int width; } pairs[]={ +` + pairs.String() + `}; +static int outcome(program fn,uint8_t *p) { + pid_t child=fork(); + if(child<0) return -1; + if(!child) { + struct rlimit no_core={0,0}; + if(setrlimit(RLIMIT_CORE,&no_core)) _exit(120); + fn(p); _exit(0); + } + int status; + if(waitpid(child,&status,0)!=child) return -1; + if(WIFEXITED(status) && WEXITSTATUS(status)==0) return 0; + if(WIFSIGNALED(status) && (WTERMSIG(status)==SIGSEGV || WTERMSIG(status)==SIGBUS)) return 1; + return -1; +} +int main(void) { + long page=sysconf(_SC_PAGESIZE); + uint8_t *data=mmap(0,2*page,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0); + if(data==MAP_FAILED || mprotect(data+page,page,PROT_NONE)) return 2; + for(unsigned i=0;i 1<<32-1) { + return fail("requires a resolved Yi32 integer in its original operand position") + } + form.args[index].Imm = amd64ScalarAddSubImmediateInt64(operand.Imm, form.bits) + immediates++ + case OpReg: + allowed := isX86YrlRegisterForArch(operand.Reg, arch) + if form.bits == 8 { + allowed = isGoYmbRegisterForArch(operand.Reg, arch) + } + if !allowed { + return fail("uses a register outside its Go Yrb/Yrl class") + } + registers++ + default: + if operand.Kind == OpSym && strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + // Non-shared 386's Yi32 includes relocated symbol addresses, + // but Go can emit the address relocation only for the L width. + // This syntax does not make the address numerical in a result proof. + if arch != "386" || index != immediatePosition { + return fail("has no relocated Yi32 row in this mode/position") + } + address := Operand{Kind: OpSym, Sym: strings.TrimPrefix(strings.TrimSpace(operand.Sym), "$")} + if _, _, valid := x86ScalarCompareSBAddress(address, arch); !valid { + return fail("has an unresolved relocated immediate") + } + if form.bits != 32 { + return form, true, fmt.Errorf("%w: Go %s short relocated immediate has no original relocation sink: %q", ErrProbeNeedsContext, op, ins.Raw) + } + immediates++ + continue + } + tls := x86SourceTLSMemoryRole(operand, arch) + if form.memory >= 0 || !tls && !x86ScalarCompareMemoryValid(operand, arch) { + return fail("requires at most one encodable Ymb/Yml memory operand") + } + form.memory = index + needsTLS = needsTLS || tls + } + } + if registers == 0 && (form.memory < 0 || immediates != 1) { + return fail("does not match any complete Go compare/test row") + } + if needsTLS { + // Check the complete row first: a TLS annotation cannot make a bad + // register, immediate direction, or second memory operand valid. + return form, true, fmt.Errorf("%w: %s %s TLS memory needs its actual Go OS/arch/shared progedit, relocation and native TLS address contract: %q", ErrProbeNeedsContext, arch, op, ins.Raw) + } + if arch == "amd64" && form.bits == 8 && x86CompareExchangeHasREXByteView(x86CompareExchangeForm{ + source: form.args[0], destination: form.args[1], + }) { + for index, operand := range form.args { + if operand.Kind == OpReg && (operand.Reg == AH || operand.Reg == CH || operand.Reg == DH || operand.Reg == BH) { + form.args[index].Reg = amd64YmbEffectiveRegister(operand.Reg, 16) + } + } + } + return form, true, nil +} + +func x86ScalarCompareMemoryValid(operand Operand, arch string) bool { + if operand.Kind == OpFP { + return true // Its separate logical frame consumer validates the byte span. + } + if operand.Kind == OpSym { + if strings.HasPrefix(strings.TrimSpace(operand.Sym), "$") { + return false + } + if _, _, valid := x86ScalarCompareSBAddress(operand, arch); valid { + return true + } + address, err := parseInt(strings.TrimSpace(operand.Sym)) + return err == nil && (address == int64(int32(address)) || arch == "386" && address == int64(uint32(address))) + } + if operand.Kind != OpMem { + return false + } + memory := operand.Mem + if x86SymbolIndexedMemoryError(memory, arch) != nil || unresolvedX86MemoryOffsetError(memory) != nil || + memory.IndexExt != "" || memory.Segment != "" && memory.Segment != FS && memory.Segment != GS { + return false + } + if memory.Sym != "" { + var valid bool + _, memory, valid = x86ScalarCompareSBAddress(operand, arch) + if !valid { + return false + } + } + return x86MemoryRegistersValidForArch(memory, arch) && + (memory.Off == int64(int32(memory.Off)) || arch == "386" && memory.Off == int64(uint32(memory.Off))) && + (memory.Index == "" && memory.Scale == 0 || memory.Index != "" && memory.Index != SP && + (memory.Scale == 1 || memory.Scale == 2 || memory.Scale == 4 || memory.Scale == 8)) +} + +// Unlike the same-file DATA reader this recognizes an address, including a +// negative displacement, without claiming an allocated or initialized extent. +func x86ScalarCompareSBAddress(operand Operand, arch string) (string, MemRef, bool) { + memory, valid := x86SBMemoryAddressRole(operand, arch) + if !valid || memory.OffRaw != "" || memory.IndexExt != "" { + return "", MemRef{}, false + } + base, offset, valid := parseSBRef(memory.Sym) + if !valid || base == "" || strings.IndexAny(base, "$+(),[] \t") >= 0 || + memory.Off > 0 && offset > math.MaxInt64-memory.Off || + memory.Off < 0 && offset < math.MinInt64-memory.Off { + return "", MemRef{}, false + } + offset += memory.Off + if offset != int64(int32(offset)) && !(arch == "386" && offset == int64(uint32(offset))) { + return "", MemRef{}, false + } + return base, memory, true +} diff --git a/amd64_scalar_compare_forms_test.go b/amd64_scalar_compare_forms_test.go new file mode 100644 index 000000000..c5e57c1eb --- /dev/null +++ b/amd64_scalar_compare_forms_test.go @@ -0,0 +1,267 @@ +package plan9asm + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestX86ScalarCompareCompleteActualGoTables(t *testing.T) { + data, err := os.ReadFile(filepath.Join(runtime.GOROOT(), "src/cmd/internal/obj/x86/asm6.go")) + if err != nil { + t.Fatal(err) + } + for table, body := range map[string]string{ + "ycmpb": `{Z_ib, 1, argList{Yal, Yi32}}, {Zm_ibo, 2, argList{Ymb, Yi32}}, +{Zm_r, 1, argList{Ymb, Yrb}}, {Zr_m, 1, argList{Yrb, Ymb}},`, + "ycmpl": `{Zm_ibo, 2, argList{Yml, Yi8}}, {Z_il, 1, argList{Yax, Yi32}}, +{Zm_ilo, 2, argList{Yml, Yi32}}, {Zm_r, 1, argList{Yml, Yrl}}, {Zr_m, 1, argList{Yrl, Yml}},`, + "yxorb": `{Zib_, 1, argList{Yi32, Yal}}, {Zibo_m, 2, argList{Yi32, Ymb}}, +{Zr_m, 1, argList{Yrb, Ymb}}, {Zm_r, 1, argList{Ymb, Yrb}},`, + "ytestl": `{Zil_, 1, argList{Yi32, Yax}}, {Zilo_m, 2, argList{Yi32, Yml}}, +{Zr_m, 1, argList{Yrl, Yml}}, {Zm_r, 1, argList{Yml, Yrl}},`, + } { + start := strings.Index(string(data), "var "+table+" = []ytab{") + if start < 0 { + t.Fatal("actual Go omitted complete comparison table", table) + } + actual := strings.SplitN(strings.SplitN(string(data)[start:], "\n}", 2)[0], "[]ytab{", 2)[1] + got, err := x86SignedPackedFloatGoOpcodeTokens(actual) + want, wantErr := x86SignedPackedFloatGoOpcodeTokens(body) + if err != nil || wantErr != nil || !reflect.DeepEqual(got, want) { + t.Fatal("actual Go comparison table changed", table) + } + } +} + +// Independent rows for ycmpb/ycmpl/yxorb/ytestl. Accumulator, general register, +// both register/memory directions and all immediate encoding transitions. +func x86ScalarCompareOracleRows(op, arch string) []string { + rows := []string{"AX,CX", "CX,AX", "AX,8(DX)", "8(DX),AX", "AX,data-3(SB)", "data+3(SB),AX", + "8(FS),AX", "AX,8(GS)", "-7(DX)(CX*2),AX", "0,AX"} + for _, immediate := range []string{"0", "1", "127", "128", "255", "256", "-128", "-129", "32768", "65535", "2147483647", "2147483648", "4294967295", "-2147483648"} { + for _, value := range []string{"AX", "CX", "8(DX)", "data(SB)"} { + operands := value + ",$" + immediate + if strings.HasPrefix(op, "TEST") { + operands = "$" + immediate + "," + value + } + rows = append(rows, operands) + } + } + if strings.HasSuffix(op, "B") { + rows = append(rows, "AH,BL", "CH,DH", "BP,SI", "DI,AX") + if arch == "amd64" { + rows = append(rows, "CH,R8B", "R9B,BH", "DIB,8(R12)") + } + } else { + rows = append(rows, "SP,BP", "DI,SI") + if arch == "amd64" { + rows = append(rows, "R8,R15", "16(R12)(R9*8),R11") + } + } + if arch == "386" { + rows = append(rows, "data(SB)(SP),AX", "AX,data(SB)(CX*4)") + if strings.HasSuffix(op, "L") { + rows = append(rows, strings.TrimPrefix(x86ScalarCompareInstruction(op, "AX", "$data(SB)"), op+" ")) + } + } + return rows +} + +func TestX86ScalarCompareCompleteGoForms(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + index := 0 + for _, op := range x86ScalarCompareNames(target.arch) { + for _, row := range x86ScalarCompareOracleRows(op, target.arch) { + body := op + " " + row + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86ScalarCompareProperties(ins.Op, ins, target.arch); !known || err != nil { + t.Fatalf("typed shape %s: %v", body, err) + } + fmt.Fprintf(&source, "TEXT ·form%d(SB),4,$0-0\n%s\nRET\n", index, body) + fmt.Fprintf(&declarations, "func form%d()\n", index) + index++ + } + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "compare-forms.ll", "compare-forms.o", ir) + }) + } + t.Logf("%d actual Go forms through three LLVM APIs", index) + }) + } +} + +func TestX86ScalarCompareMalformedForms(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + for _, op := range x86ScalarCompareNames(arch) { + bad := []string{"(AX),(CX)", "$1,$2", "X0,AX", "AX", "AX,CX,DX", "AX,K1"} + if strings.HasPrefix(op, "CMP") { + bad = append(bad, "$1,AX") + } else { + bad = append(bad, "AX,$1") + } + if strings.HasSuffix(op, "B") && arch == "386" { + bad = append(bad, "SP,AX") + } else if !strings.HasSuffix(op, "B") { + bad = append(bad, "AL,AX") + } + if arch == "amd64" { + bad = append(bad, strings.TrimPrefix(x86ScalarCompareInstruction(op, "AX", "$4294967296"), op+" ")) + } + for _, row := range bad { + body := op + " " + row + goAssemblerOracleOutput(t, arch, "TEXT f(SB),4,$0-0\n"+body+"\nRET\n", "", false) + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86ScalarCompareProperties(ins.Op, ins, arch); !known || err == nil { + t.Errorf("invalid Go form admitted: %s %s", arch, body) + } + } + } + } +} + +func TestX86ScalarCompareShortRelocationBackendDefect(t *testing.T) { + for _, op := range []string{"CMPB", "CMPW", "TESTB", "TESTW"} { + t.Run(op, func(t *testing.T) { + body := x86ScalarCompareInstruction(op, "AX", "$data(SB)") + original := "TEXT f(SB),4,$0-0\n" + body + "\nRET\n" + dir := t.TempDir() + source, object := filepath.Join(dir, "forms.s"), filepath.Join(dir, "forms.o") + if err := os.WriteFile(source, []byte(original), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := runtimeOracleStageContext(t) + defer cancel() + command := exec.CommandContext(ctx, filepath.Join(runtime.GOROOT(), "bin/go"), "tool", "asm", "-o", object, source) + command.Env = append(os.Environ(), "GOOS=linux", "GOARCH=386", "GOTOOLCHAIN=local") + output, err := runtimeOracleCombinedOutput(ctx, command) + exit, exited := err.(*exec.ExitError) + if ctx.Err() != nil || !exited || exit.ProcessState == nil || !exit.ProcessState.Exited() || + exit.ExitCode() != 1 || string(output) != "asm: need reloc for $data(SB)\nasm: reloc\n" { + t.Fatalf("unknown short relocation backend observation: %v; context %v\n%s", err, ctx.Err(), output) + } + if goAssemblerOracleOutcome(source, object, output, err, ctx.Err(), false, "386") == nil { + t.Fatal("backend defect became an illegal-source certificate") + } + if data, err := os.ReadFile(source); err != nil || string(data) != original { + t.Fatal("original source changed during backend observation", err) + } + ins := x86PureGPUnknownInstruction(t, body) + if _, known, err := x86ScalarCompareProperties(ins.Op, ins, "386"); !known || !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("missing original bytes became an ordinary numerical form: %v", err) + } + t.Log("GO_ASSEMBLER_DEFECT_OBSERVATION: short immediate relocation has no sink; not source rejection or a compiled Go object") + }) + } +} + +func TestX86ScalarCompareConditionalReadBoundaries(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + ctx := &amd64Ctx{goarch: arch, resolve: func(s string) string { return s }} + for _, op := range x86ScalarCompareNames(arch) { + ins := x86PureGPUnknownInstruction(t, x86ScalarCompareInstruction(op, "-7(DX)", "AX")) + form, _, err := x86ScalarCompareProperties(ins.Op, ins, arch) + if err != nil { + t.Fatal(err) + } + full := x86ResultState{flags: x86ByteFlagsAll} + full.gp[0] = 0xff + full.pointerGP[2] = true + before := full.clone() + if accepted, known := ctx.scalarCompareNativeResultTransfer(ins, &full, nil, arch); !known || !accepted { + t.Fatal("bounded-free external read rejected", arch, op) + } + before.flags = x86ByteFlagsAll + if form.test { + before.flags &^= x86ByteFlagAF + } + if !reflect.DeepEqual(full, before) { + t.Fatal("compare acquired more than its defined flags", arch, op) + } + for b := 0; b < form.bits/8; b++ { + state := before.clone() + state.gp[0] &^= 1 << uint(b) + original := state.clone() + if accepted, _ := ctx.scalarCompareNativeResultTransfer(ins, &state, nil, arch); accepted || !reflect.DeepEqual(state, original) { + t.Fatal("uninitialized register byte gained flags", arch, op, b) + } + } + for name, mutate := range map[string]func(*Instr){ + "address-size": func(i *Instr) { i.x86AddressBits = 32 }, + "unbound-size": func(i *Instr) { i.x86UnboundAddressOverride = true }, + "raw": func(i *Instr) { i.x86Encoded = true }, + "float": func(i *Instr) { i.Args[1-form.memory] = Operand{Kind: OpImm, ImmIsFloat: true} }, + "unresolved": func(i *Instr) { i.Args[1-form.memory] = Operand{Kind: OpImm, ImmRaw: "not_defined"} }, + } { + changed := ins + changed.Args = append([]Operand(nil), ins.Args...) + mutate(&changed) + state := before.clone() + if accepted, _ := ctx.scalarCompareNativeResultTransfer(changed, &state, nil, arch); accepted { + t.Fatal("unbound read accepted", arch, op, name) + } + } + } + } +} + +func TestX86ScalarCompareNativeSourceAddressRoles(t *testing.T) { + for _, arch := range []string{"amd64", "386"} { + ctx := &amd64Ctx{resolve: func(s string) string { return s }, sigs: map[string]FuncSig{"function": {Name: "aliasedFunction"}}, + continuations: map[string]x86Continuation{"continuation": {}}} + state := x86ResultState{} + state.pointerGP[0], state.gp[1] = true, 0xff + for _, address := range []string{"-7(AX)", "8(AX)(CX*2)"} { + operand := x86PureGPUnknownInstruction(t, "CMPL "+address+",$1").Args[0] + if !ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("valid address role rejected", arch, address) + } + } + for _, address := range []string{"external-7(SB)", "external+7(SB)", "function(SB)", "aliasedFunction(SB)", "continuation(SB)", "$external(SB)", "(FS)", "(GS)", "(DX)", "(AX)(DX*4)", "0", "(SP)"} { + operand := x86PureGPUnknownInstruction(t, "CMPL "+address+",$1").Args[0] + if ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("unbound address role accepted", arch, address) + } + } + if arch == "386" { + for _, address := range []string{"external(SB)(SP)", "external(SB)(FS)", "external(SB)(X15)", "external(SB)(CX*4)"} { + operand := x86PureGPUnknownInstruction(t, "CMPL "+address+",$1").Args[0] + if ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("named address acquired source-layout authority", address) + } + if !x86ScalarCompareMemoryValid(operand, arch) { + t.Fatal("Go ignored base or active index lost its syntax", address) + } + } + operand := x86PureGPUnknownInstruction(t, "CMPL external(SB)(DX*4),$1").Args[0] + if ctx.scalarCompareNativeAddress(operand, &state, arch) { + t.Fatal("active unknown named index treated as ignored base") + } + } + } + ins := x86PureGPUnknownInstruction(t, "CMPB AH,R8B") + var ir strings.Builder + ctx := &amd64Ctx{goarch: "amd64", b: &ir} + if _, _, err := ctx.lowerCmpBt(ins.Op, ins); !errors.Is(err, ErrProbeNeedsContext) || ir.Len() != 0 { + t.Fatalf("unbound REX SP alias emitted code: %v", err) + } +} diff --git a/amd64_scalar_compare_native_read.go b/amd64_scalar_compare_native_read.go new file mode 100644 index 000000000..77d63eec2 --- /dev/null +++ b/amd64_scalar_compare_native_read.go @@ -0,0 +1,56 @@ +package plan9asm + +// The paired compare/test emitter performs a volatile native read. If it +// continues, the comparison's inputs are numerical and its defined flags are +// available. This does not assert that an address is backed, in bounds, aligned +// or writable; no pointer identity, memory bytes or reusable proof is produced. +func (c *amd64Ctx) scalarCompareNativeResultTransfer(ins Instr, state *x86ResultState, owned []bool, arch string) (bool, bool) { + if ins.x86NumericalRead != nil { + return false, false // The immutable source-layout certificate owns this read. + } + form, known, err := x86ScalarCompareProperties(ins.Op, ins, arch) + if !known { + return false, false + } + if err != nil || ins.x86AddressBits != 0 || ins.x86UnboundAddressOverride || + ins.x86Encoded || ins.x86SourceRaw != nil || ins.x86RawNearReturn != nil { + return false, true + } + for index, operand := range form.args { + if index != form.memory || !x86OrdinaryNativeMemoryOperand(operand, arch) { + if operand.Kind == OpReg && !state.shapeOnly { + register, _, valid := x86ResultGP(operand.Reg, int64(form.bits/8), arch) + if valid && state.pointerGP[register] { + return false, true + } + } + if !x86ResultRead(operand, int64(form.bits/8), state, owned, arch) { + return false, true + } + continue + } + if !x86ResultRead(operand, int64(form.bits/8), state, owned, arch) && + !c.scalarCompareNativeAddress(operand, state, arch) { + return false, true + } + } + state.flags = x86ByteFlagsAll + if form.test { + state.flags &^= x86ByteFlagAF + } + return true, true +} + +func (c *amd64Ctx) scalarCompareNativeAddress(operand Operand, state *x86ResultState, arch string) bool { + // This rule deliberately excludes unbound TLS/segments, absolute numerical + // addresses, unknown GP bases and virtual frame addresses. External SB needs + // its existing actual-source numerical certificate: a volatile read alone + // cannot establish that an LLVM global preserves original Go field layout. + if operand.Kind == OpMem && operand.Mem.Index != "" && !state.shapeOnly { + index, _, valid := x86ResultGP(operand.Mem.Index, x86ResultPointerWord(arch), arch) + if valid && state.pointerGP[index] { + return false + } + } + return x86ResultIncomingPointerAddressIdentity(operand, state, arch) +} diff --git a/amd64_scalar_compare_native_test.go b/amd64_scalar_compare_native_test.go new file mode 100644 index 000000000..5a8f91a31 --- /dev/null +++ b/amd64_scalar_compare_native_test.go @@ -0,0 +1,188 @@ +package plan9asm + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func x86ScalarCompareNames(arch string) []string { + var names []string + for _, stem := range []string{"CMP", "TEST"} { + for _, suffix := range []string{"B", "W", "L", "Q"} { + if arch != "386" || suffix != "Q" { + names = append(names, stem+suffix) + } + } + } + return names +} + +func x86ScalarCompareInstruction(op, memory, other string) string { + if strings.HasPrefix(op, "TEST") { + return op + " " + other + "," + memory + } + return op + " " + memory + "," + other +} + +func x86ScalarCompareResultSource(name, body string) string { + // PEXTRD, unlike the older MOVL X0,AX compatibility path, actually invokes + // the private scalar-result observer. Both branches initialize every byte. + return fmt.Sprintf("TEXT ·%s(SB),4,$0-4\n%s\nJE equal\nMOVL $7,AX\nMOVL AX,X0\nJMP done\nequal:\nPXOR X0,X0\ndone:\nPEXTRD $0,X0,AX\nMOVL AX,ret+0(FP)\nRET\n", name, body) +} + +func TestX86ScalarCompareSourceDataResultFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc is required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + source.WriteString("GLOBL external(SB),8,$16\n") + for index, op := range x86ScalarCompareNames(target.arch) { + name := fmt.Sprintf("compare%d", index) + source.WriteString(x86ScalarCompareResultSource(name, x86ScalarCompareInstruction(op, "external+3(SB)", "$1"))) + fmt.Fprintf(&declarations, "func %s() uint32\n", name) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "compare.ll", "compare.o", ir) + }) + } + }) + } +} + +func TestX86ScalarCompareUnboundExternalResultStaysContext(t *testing.T) { + for _, target := range x86RawReturnTargets { + for _, op := range x86ScalarCompareNames(target.arch) { + for _, offset := range []int{-7, 0, 3} { + source := x86ScalarCompareResultSource("view", x86ScalarCompareInstruction(op, fmt.Sprintf("external%+d(SB)", offset), "$1")) + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "var external [32]byte\nfunc view() uint32", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("external source-layout guard bypassed: %s/%s/%s/%d: %v", target.triple, api, op, offset, err) + } + } + } + } + } +} + +func x86ScalarComparePointerSource(arch, name, setup, compare, tail string) string { + word, move := 8, "MOVQ" + if arch == "386" { + word, move = 4, "MOVL" + } + source := x86ScalarCompareResultSource(name, move+" p+0(FP),DX\n"+setup+compare+"\n"+tail) + source = strings.Replace(source, "$0-4", fmt.Sprintf("$0-%d", word+4), 1) + return strings.ReplaceAll(source, "ret+0(FP)", fmt.Sprintf("ret+%d(FP)", word)) +} + +func TestX86ScalarCompareIncomingPointerResultFiveTargets(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + for index, op := range x86ScalarCompareNames(target.arch) { + name := fmt.Sprintf("view%d", index) + compare := x86ScalarCompareInstruction(op, "-7(DX)(CX*2)", "$1") + source.WriteString(x86ScalarComparePointerSource(target.arch, name, "MOVL $4,CX\n", compare, "")) + fmt.Fprintf(&declarations, "func %s(p *byte) uint32\n", name) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + t.Run(api, func(t *testing.T) { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal(err) + } + compileLLVMToObject(t, llc, target.triple, "compare-pointer.ll", "compare-pointer.o", ir) + }) + } + }) + } +} + +func TestX86ScalarCompareIncomingPointerDoesNotGrantOtherEffects(t *testing.T) { + for _, target := range x86RawReturnTargets { + move := "MOVQ" + if target.arch == "386" { + move = "MOVL" + } + for _, op := range x86ScalarCompareNames(target.arch) { + for _, row := range []struct{ name, setup, address, partner, tail string }{ + {"unknown-index", "", "(DX)(CX*2)", "$1", ""}, + {"opaque-index", move + " p+0(FP),CX\n", "(DX)(CX*2)", "$1", ""}, + {"numeric-not-pointer", move + " $1234,DX\n", "(DX)", "$1", ""}, + {"unknown-numerical-input", "", "(DX)", "AX", ""}, + {"opaque-numerical-input", move + " p+0(FP),AX\n", "(DX)", "AX", ""}, + {"no-reusable-backing-credit", "", "(DX)", "$1", "MOVL (DX),DX\nMOVL (DX),AX\n"}, + {"no-write-credit", "", "(DX)", "$1", "MOVL $1,(DX)\n"}, + {"no-segment-credit", "", "(FS)", "$1", ""}, + } { + source := x86ScalarComparePointerSource(target.arch, "view", row.setup, + x86ScalarCompareInstruction(op, row.address, row.partner), row.tail) + for _, api := range []string{"text", "module", "Go"} { + _, err := x86ScalarFPBitViewAPI(t, source, "func view(p *byte) uint32", target.arch, target.triple, api) + if !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("conditional read expanded authority %s/%s/%s/%s: %v", target.triple, op, row.name, api, err) + } + } + } + if strings.HasPrefix(op, "TEST") { + source := x86ScalarComparePointerSource(target.arch, "view", "", + x86ScalarCompareInstruction(op, "(DX)", "$1"), "LAHF\n") + if _, err := x86ScalarFPBitViewAPI(t, source, "func view(p *byte) uint32", target.arch, target.triple, "text"); !errors.Is(err, ErrProbeNeedsContext) { + t.Fatalf("TEST undefined AF became observable: %s/%s: %v", target.triple, op, err) + } + } + } + } +} + +// The compare's continuation grants no credit to another read. Ordinary MOVL +// now has its own paired volatile load and numerical effect; both operations +// must survive even though the MOVL result is dead. Overwriting DX above still +// revokes the pointer before a later access. +func TestX86ScalarCompareAndMoveRetainIndependentNativeReads(t *testing.T) { + llc := findLLVM22Tool("llc") + if llc == "" { + t.Fatal("LLVM 22 llc required") + } + for _, target := range x86RawReturnTargets { + t.Run(target.triple, func(t *testing.T) { + var source, declarations strings.Builder + names := x86ScalarCompareNames(target.arch) + for index, op := range names { + name := fmt.Sprintf("CompareMove%d", index) + compare := x86ScalarCompareInstruction(op, "(DX)", "$1") + source.WriteString(x86ScalarComparePointerSource(target.arch, name, "", compare, "MOVL (DX),AX\n")) + fmt.Fprintf(&declarations, "func %s(p *byte) uint32\n", name) + } + x86LEAActualTargetGoObject(t, target.arch, target.triple, source.String()) + for _, api := range []string{"text", "module", "Go"} { + ir, err := x86ScalarFPBitViewAPI(t, source.String(), declarations.String(), target.arch, target.triple, api) + if err != nil { + t.Fatal("separately paired native reads rejected", api, err) + } + for level, text := range []string{ir, x86MulDivRoundOptimize(t, ir)} { + if got := strings.Count(text, "load volatile"); got < 2*len(names) { + t.Fatal("a native read borrowed earlier backing or was discarded", api, level, got) + } + x86ResultInitializationObject(t, llc, target.triple, text, fmt.Sprintf("compare-move-%s-O%d", api, level*2)) + } + } + }) + } +} diff --git a/amd64_scalar_compare_runtime_test.go b/amd64_scalar_compare_runtime_test.go new file mode 100644 index 000000000..5c37a865c --- /dev/null +++ b/amd64_scalar_compare_runtime_test.go @@ -0,0 +1,106 @@ +package plan9asm + +import ( + "fmt" + "strings" + "testing" +) + +func TestCrossLinuxRuntimeMatrixX86ScalarCompareFlags(t *testing.T) { + execution := x86FloatCompareExecutionTools(t) + var source, declarations, native, prototypes, pairs strings.Builder + count := 0 + for _, op := range x86ScalarCompareNames("amd64") { + rows := []string{"(DX),CX", "CX,(DX)"} + for _, immediate := range []string{"0", "127", "128", "255", "32768", "65535", "2147483648", "4294967295", "-2147483648"} { + rows = append(rows, strings.TrimPrefix(x86ScalarCompareInstruction(op, "(DX)", "$"+immediate), op+" ")) + } + if strings.HasSuffix(op, "B") { + rows = append(rows, "CH,(DX)", "(DX),CH", "CL,CH", "CH,CL", "BH,R8B", "R8B,DH") + } + for _, row := range rows { + name, original := fmt.Sprintf("compare%d", count), fmt.Sprintf("original%d", count) + mask := 0x8d5 // CF, PF, AF, ZF, SF, OF only. + if strings.HasPrefix(op, "TEST") { + mask &^= 0x10 // TEST leaves AF undefined; never invent an oracle value. + } + body := op + " " + row + setup, originalSetup := "", "" + if strings.Contains(row, "R8B") { + setup = "MOVQ x+8(FP),DI\nMOVQ x+8(FP),SI\nMOVQ x+8(FP),R8\nNOTQ R8\n" + originalSetup = "MOVQ SI,DI\nMOVQ SI,R8\nNOTQ R8\n" + } + fmt.Fprintf(&source, "TEXT ·%s(SB),4,$0-24\nMOVQ p+0(FP),DX\nMOVQ x+8(FP),CX\n%s%s\nPUSHFQ\nPOPQ AX\nANDQ $%d,AX\nMOVQ AX,ret+16(FP)\nRET\n", name, setup, body, mask) + fmt.Fprintf(&declarations, "func %s(p *byte,x uint64) uint64\n", name) + originalBody := strings.ReplaceAll(body, "DX", "DI") + code := x86PackedShiftActualGoBytes(t, "amd64", fmt.Sprintf("TEXT f(SB),4,$0-0\nMOVQ SI,CX\n%s%s\nPUSHFQ\nPOPQ AX\nANDQ $%d,AX\nRET\n", originalSetup, originalBody, mask)) + if len(code) < 9 || code[len(code)-1] != 0xc3 { + t.Fatal("original Go flags program is incomplete", body, code) + } + var bytes []string + for _, b := range code { + bytes = append(bytes, fmt.Sprintf("0x%02x", b)) + } + symbol := original + if strings.Contains(execution.triple, "apple") { + symbol = "_" + symbol + } + fmt.Fprintf(&native, "__asm__(%q);\n", ".text\n.globl "+symbol+"\n"+symbol+":\n.byte "+strings.Join(bytes, ",")+"\n") + fmt.Fprintf(&prototypes, "extern uint64_t %s(uint8_t*,uint64_t),%s(uint8_t*,uint64_t);\n", name, original) + fmt.Fprintf(&pairs, "{%s,%s},\n", original, name) + count++ + } + } + goAssemblerOracleOutput(t, "amd64", source.String(), "", true) + main := ` +#include +#include +#include +` + prototypes.String() + native.String() + ` +typedef uint64_t (*program)(uint8_t*,uint64_t); +static const struct { program original,translated; } pairs[]={ +` + pairs.String() + `}; +int main(void) { + const uint64_t patterns[]={0,1,15,16,127,128,255,256,32767,32768,65535,65536, + UINT64_C(0x7fffffff),UINT64_C(0x80000000),UINT64_C(0xffffffff), + UINT64_C(0x7fffffffffffffff),UINT64_C(0x8000000000000000), + UINT64_C(0xffffffffffffffff),UINT64_C(0x55aa0123cdef9876)}; + for(unsigned i=0;i